gss 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074
  1. #! /usr/bin/env sh
  2. #ProTeX: 1.14-AJS
  3. #
  4. #BOI
  5. #
  6. # !TITLE: GSS - General Storage System
  7. # !AUTHORS: Arjo Segers
  8. # !AFFILIATION: KNMI, The Netherlands
  9. # !DATE: \today
  10. #
  11. # !INTRODUCTION: Usage
  12. #
  13. # \bv
  14. # gss <task> [<options>] [<location> ...]
  15. # \ev
  16. #
  17. #
  18. # !INTRODUCTION: Description
  19. #
  20. # Shell script to access file systems, tape systems, ftp servers,
  21. # and other file archives in a uniform way.
  22. #
  23. #
  24. # !INTRODUCTION: TASKS
  25. #
  26. # \bv
  27. # list [-l,--long] [<location> ...]
  28. #
  29. # File or directory listing.
  30. # If no location is specified, the current directory is listed.
  31. # Use the '-l' option for long listing.
  32. #
  33. # exist [-e,--echo] <location>
  34. #
  35. # Returns with zero exit status (succeed) if the location exists,
  36. # and non-zero (fail) if not.
  37. # With the '-e' option, the csh status '1' (succeed) is echo'd
  38. # to standard output if the location exists and '0' (fail) if not;
  39. # the script always succeeds.
  40. #
  41. # copy [-f,--force] [-m,--mkdir] <source-location> <dest-location>
  42. #
  43. # Copy source file to destionation.
  44. # -f, --force : overwrite existing destination files if necessary
  45. # -m, --mkdir : create destination directories if necessary
  46. #
  47. #
  48. # link [-s,--soft] [-f,--force] [-n,--new] <remote-location> <localfile>
  49. # unlink <localfile>
  50. #
  51. # Create a symbolic link with name <localfile> to a so-called 'linkfile'
  52. # with name '.*.gsslink'.
  53. # The 'linkfile' is copy from or a symbolic link to the file specified
  54. # by <remote-location> .
  55. # If the <localfile> is unlinked, both the <localfile> and the
  56. # 'linkfile' are removed.
  57. # In case of a 'hard' link (default), the original location is written
  58. # to a file named '.*.gsslinksrc'; if a hard link is removed,
  59. # and the linkfile is a copy of a remote file,
  60. # the remote file is replaced by the (changed) local copy.
  61. #
  62. # local remote
  63. # -------------------------------------- ---------------------
  64. #
  65. # data-a.dat -> ._data_a_dat.gsslink
  66. # ._data_a.dat.gsslink -> /data/a.dat /data/a.dat
  67. #
  68. # data-b.dat -> .tape_b_dat.gsslink
  69. # .tape_b_dat.gsslink tape:b.dat
  70. # .tape_b_dat.gsslinksrc
  71. #
  72. # Options:
  73. # -s, --soft : create soft link
  74. # -f, --force : overwrite existing destination files if necessary
  75. # -n, --new : create new empty file if necessary
  76. #
  77. #
  78. # tar-extract <tarfile> [<tarred-file>]
  79. #
  80. # Extract the <tarred-file> from a tarfile or untar the tarfile completely.
  81. #
  82. # tar-update <tarfile> <input-file>
  83. #
  84. # Update the <input-file> in a (linked) tar file.
  85. # An old version of <input-file> is removed from the tarfile.
  86. #
  87. #
  88. # limit <max-usage> <directory>
  89. #
  90. # If the disk usage of the <directory-location> exceeds the <max-usage>,
  91. # files that have not been touched recently are removed until the
  92. # disk usage is small enought.
  93. # A valid <max-usage> is a number followed by a character
  94. # to specify kilo (default), mega, or giga bytes,
  95. # or 'Inf' to not have any limitation (case independent):
  96. # 100, 100k, 2m, 4g, Inf
  97. # \ev
  98. #
  99. #
  100. # !INTRODUCTION: Locations
  101. #
  102. # \bv
  103. # #
  104. # # standard files
  105. # #
  106. # file:path/file
  107. #
  108. # #
  109. # # KNMI's MOS tape system
  110. # #
  111. # mos:/fa/ks/file
  112. # mos:umask=002%/fa/ks/data/a.txt
  113. #
  114. # #
  115. # # CINECA cartridge
  116. # #
  117. # # NOTE: no subdirectories allowed, only a single volume
  118. # cart:myvolume/myfile.txt
  119. #
  120. # #
  121. # # ECMWF's ecfs
  122. # #
  123. # ecfs:/nl5/data/a.txt
  124. # ecfs:/TMP/nl5/dump.txt
  125. # ecfs:umask=002%/nlh/shared/a.txt
  126. #
  127. # #
  128. # # ECMWF file system via EcAccess gateway
  129. # #
  130. # ec:echome[nl5]:path/file
  131. # ec:ecscratch[nl5]:path/file
  132. # ec:ecfs[nl5]:path/file
  133. # ec:ectmp[nl5]:path/file
  134. #
  135. # #
  136. # # ftp server
  137. # #
  138. # ftp:ftp.somewhere.int:path/file
  139. # \ev
  140. #
  141. # !INTRODUCTION: General options
  142. #
  143. # \bv
  144. # -v, --verbose : display info on progress
  145. # \ev
  146. #
  147. #EOI
  148. # ----------------------------------------------------------
  149. # --- init
  150. # ----------------------------------------------------------
  151. # leave on error
  152. set -e
  153. # set program name and location:
  154. call="$0 $*"
  155. case $0 in
  156. /* ) script=$0 ;;
  157. * ) script="`pwd`/$0" ;;
  158. esac
  159. bindir=`dirname ${script}`
  160. prog=`basename ${script}`
  161. # ----------------------------------------------------------
  162. # --- help
  163. # ----------------------------------------------------------
  164. DisplayUsage ()
  165. {
  166. ${PAGER:-less} << EOF
  167. NAME
  168. ${prog} - General Storage System
  169. USAGE
  170. ${prog} <task> [<options>] [<location> ...]
  171. DESCRIPTION
  172. Shell script to access file systems, tape systems, ftp servers,
  173. and other file archives in a uniform way.
  174. TASKS
  175. list [-l,--long] [<location> ...]
  176. File or directory listing.
  177. If no location is specified, the current directory is listed.
  178. Use the '-l' option for long listing.
  179. exist [-e,--echo] <location>
  180. Returns with zero exit status (succeed) if the location exists,
  181. and non-zero (fail) if not.
  182. With the '-e' option, the csh status '1' (succeed) is echo'd
  183. to standard output if the location exists and '0' (fail) if not;
  184. the script always succeeds.
  185. copy [-f,--force] [-m,--mkdir] <source-location> <dest-location>
  186. Copy source file to destionation.
  187. -f, --force : overwrite existing destination files if necessary
  188. -m, --mkdir : create destination directories if necessary
  189. link [-s,--soft] [-f,--force] [-n,--new] <remote-location> <localfile>
  190. unlink <localfile>
  191. Create a symbolic link with name <localfile> to a so-called 'linkfile'
  192. with name '.*.gsslink'.
  193. The 'linkfile' is copy from or a symbolic link to the file specified
  194. by <remote-location> .
  195. If the <localfile> is unlinked, both the <localfile> and the
  196. 'linkfile' are removed.
  197. In case of a 'hard' link (default), the original location is written
  198. to a file named '.*.gsslinksrc'; if a hard link is removed,
  199. and the linkfile is a copy of a remote file,
  200. the remote file is replaced by the (changed) local copy.
  201. local remote
  202. -------------------------------------- ---------------------
  203. data-a.dat -> ._data_a_dat.gsslink
  204. ._data_a.dat.gsslink -> /data/a.dat /data/a.dat
  205. data-b.dat -> .tape_b_dat.gsslink
  206. .tape_b_dat.gsslink tape:b.dat
  207. .tape_b_dat.gsslinksrc
  208. Options:
  209. -s, --soft : create soft link
  210. -f, --force : overwrite existing destination files if necessary
  211. -n, --new : create new empty file if necessary
  212. tar-extract <tarfile> [<tarred-file>]
  213. Extract the <tarred-file> from a tarfile or untar the tarfile completely.
  214. tar-update [-k,--keep-old-files] <tarfile> <input-file>
  215. Update the <input-file> in a (linked) tar file.
  216. If '--keep-old-files' is not specified, an old version of <input-file>
  217. is removed from the tarfile.
  218. limit <max-usage> <directory>
  219. If the disk usage of the <directory-location> exceeds the <max-usage>,
  220. files that have not been touched recently are removed until the
  221. disk usage is small enought.
  222. A valid <max-usage> is a number followed by a character
  223. to specify kilo (default), mega, or giga bytes,
  224. or 'Inf' to not have any limitation (case independent):
  225. 100, 100k, 2m, 4g, Inf
  226. LOCATIONS
  227. #
  228. # standard files
  229. #
  230. file:path/file
  231. #
  232. # KNMI's MOS tape system
  233. #
  234. mos:/fa/ks/file
  235. mos:umask=002%/fa/ks/data/a.txt
  236. #
  237. # CINECA cartridge
  238. #
  239. # NOTE: no subdirectories allowed, only a single volume
  240. cart:myvolume/myfile.txt
  241. #
  242. # ECMWF's ecfs
  243. #
  244. ecfs:/nl5/data/a.txt
  245. ecfs:/TMP/nl5/dump.txt
  246. ecfs:umask=002%/nlh/shared/a.txt
  247. #
  248. # ECMWF file system via EcAccess gateway
  249. #
  250. ec:echome[nl5]:path/file
  251. ec:ecscratch[nl5]:path/file
  252. ec:ecfs[nl5]:path/file
  253. ec:ectmp[nl5]:path/file
  254. #
  255. # ftp server
  256. #
  257. ftp:ftp.somewhere.int:path/file
  258. GENERAL OPTIONS
  259. -v, --verbose : display info on progress
  260. EOF
  261. exit 0
  262. }
  263. # err 'help text'
  264. err ()
  265. {
  266. echo "$1" 1>&2
  267. }
  268. # errit <exit-status>
  269. errit ()
  270. {
  271. err "$prog - ERROR - from : ${call}"
  272. err "$prog - ERROR - Use '$prog --help' for more info."
  273. exit $1
  274. }
  275. # ----------------------------------------------------------
  276. # --- arguments
  277. # ----------------------------------------------------------
  278. options=''
  279. task=''
  280. locations=''
  281. verbose=''
  282. keep_old_files='false'
  283. # extract settings
  284. for arg in "$@" ; do
  285. case ${arg} in
  286. -h | --help ) DisplayUsage ;;
  287. -* ) # option
  288. # add to option list:
  289. options="${options} ${arg}"
  290. # special options:
  291. case ${arg} in
  292. -v | --verbose ) verbose=${arg} ;;
  293. -k | --keep-old-files ) keep_old_files='true' ;;
  294. esac
  295. ;;
  296. * ) # task or (one of the) location(s)
  297. if [ -z "${task}" ]; then
  298. task=${arg}
  299. else
  300. # default protocol is 'file:'
  301. if [[ ${arg} != *:* ]] ; then
  302. location="file:${arg}"
  303. else
  304. location=${arg}
  305. fi
  306. # add to location list:
  307. locations="${locations} ${location}"
  308. fi
  309. ;;
  310. esac
  311. done
  312. # not complete ?
  313. if [ -z "${task}" ]; then
  314. err "$prog - ERROR - no task specified"
  315. errit 1
  316. fi
  317. # ----------------------------------------------------------
  318. # --- settings
  319. # ----------------------------------------------------------
  320. # search gnu tar or use another recent tar version:
  321. gtar='/no/tar'
  322. test ! -x ${gtar} && gtar='/usr/local/bin/gtar' # linux
  323. test ! -x ${gtar} && gtar='/usr/freeware/bin/tar' # teras
  324. test ! -x ${gtar} && gtar='/usr/local/bin/tar-1.15' # hpcf
  325. test ! -x ${gtar} && gtar='tar' # default
  326. # ----------------------------------------------------------
  327. # --- begin
  328. # ----------------------------------------------------------
  329. test ${verbose} && echo "$prog - $prog $*"
  330. # test task:
  331. case ${task} in
  332. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  333. # list files
  334. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  335. 'list' )
  336. # empty ? then list current directory
  337. test -z "${locations}" && locations='file:.'
  338. # loop over locations
  339. for loc in ${locations} ; do
  340. # split in location protocol and location name:
  341. protocol=`echo ${loc} | cut -d ':' -f 1`
  342. filespec=`echo ${loc} | cut -d ':' -f 2-`
  343. # call protocol specific script:
  344. case ${protocol} in
  345. file ) ${bindir}/gss_file list ${options} ${filespec} ;;
  346. mos ) ${bindir}/gss_mos list ${options} ${filespec} ;;
  347. cart ) ${bindir}/gss_cart list ${options} ${filespec} ;;
  348. ecfs ) ${bindir}/gss_ecfs list ${options} ${filespec} ;;
  349. ec ) ${bindir}/gss_ec list ${options} ${filespec} ;;
  350. ftp ) ${bindir}/gss_ftp list ${options} ${filespec} ;;
  351. * ) err "$prog - ERROR - unsupported location protocol '${protocol}' for task '${task}'."
  352. errit 1;;
  353. esac # protocols
  354. done # loop over locations
  355. ;;
  356. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  357. # file exists ?
  358. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  359. 'exist' )
  360. # only one file specification:
  361. loc=''
  362. for aloc in ${locations} ; do
  363. if [ -z "${loc}" ]; then
  364. loc=${aloc}
  365. else
  366. err "$prog - ERROR - task '${task}' requires single location only."
  367. errit 1
  368. fi
  369. done
  370. if [ -z "${loc}" ]; then
  371. err "$prog - ERROR - task '${task}' requires location."
  372. errit 1
  373. fi
  374. # split in protocol and file specification:
  375. protocol=`echo ${loc} | cut -d ':' -f 1`
  376. filespec=`echo ${loc} | cut -d ':' -f 2-`
  377. # call protocol specific script:
  378. case ${protocol} in
  379. file | mos | cart | ecfs | ec | ftp )
  380. # extract options
  381. echo_status=''
  382. for option in ${options} ; do
  383. case ${option} in
  384. -e | --echo ) echo_status='true' ;;
  385. -v | --verbose ) ;;
  386. -* ) err "$prog - unknown option '${option}' for task '${task}'"
  387. errit 1 ;;
  388. esac
  389. done
  390. # list file; return status 0 if no error, non zero otherwise
  391. if ( ${script} list ${loc} > /dev/null 2>&1 ) ; then
  392. # found !
  393. if [ ${echo_status} ]; then echo '1' ; else exit 0 ; fi
  394. else
  395. # not found ...
  396. if [ ${echo_status} ]; then echo '0' ; else exit 1 ; fi
  397. fi
  398. ;;
  399. * ) err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
  400. errit 1 ;;
  401. esac # protocols
  402. ;;
  403. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  404. # copy files
  405. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  406. 'copy' )
  407. # extract source and destination location:
  408. source_loc=''
  409. dest_loc=''
  410. for loc in ${locations} ; do
  411. if [ -z "${source_loc}" ]; then
  412. source_loc=${loc}
  413. elif [ -z "${dest_loc}" ]; then
  414. dest_loc=${loc}
  415. else
  416. err "$prog - ERROR - task '${task}' requires source and destination locations only."
  417. errit 1
  418. fi
  419. done
  420. if [ -z "${dest_loc}" ]; then
  421. err "$prog - ERROR - task '${task}' requires source and destination locations."
  422. errit 1
  423. fi
  424. # split in protocol and file specification:
  425. source_protocol=`echo ${source_loc} | cut -d ':' -f 1`
  426. source_filespec=`echo ${source_loc} | cut -d ':' -f 2-`
  427. dest_protocol=`echo ${dest_loc} | cut -d ':' -f 1`
  428. dest_filespec=`echo ${dest_loc} | cut -d ':' -f 2-`
  429. # destination '.' ? set to filename
  430. test "${dest_filespec}" = "." && dest_filespec=`basename ${source_filespec}`
  431. # if either source or destination is a local file or directory,
  432. # the task specific scripts will handle it:
  433. # a copy between two non-file locations requires intermediate storage:
  434. if [ "${source_protocol}" = "file" ] ; then
  435. # call destination protocol specific script:
  436. case ${dest_protocol} in
  437. file ) ${bindir}/gss_file put ${options} ${source_filespec} ${dest_filespec} ;;
  438. mos ) ${bindir}/gss_mos put ${options} ${source_filespec} ${dest_filespec} ;;
  439. cart ) ${bindir}/gss_cart put ${options} ${source_filespec} ${dest_filespec} ;;
  440. ecfs ) ${bindir}/gss_ecfs put ${options} ${source_filespec} ${dest_filespec} ;;
  441. ec ) ${bindir}/gss_ec put ${options} ${source_filespec} ${dest_filespec} ;;
  442. ftp ) ${bindir}/gss_ftp put ${options} ${source_filespec} ${dest_filespec} ;;
  443. * )
  444. err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
  445. errit 1
  446. ;;
  447. esac # protocols
  448. elif [ "${dest_protocol}" = "file" ] ; then
  449. # call source protocol specific script:
  450. case ${source_protocol} in
  451. file ) ${bindir}/gss_file get ${options} ${source_filespec} ${dest_filespec} ;;
  452. mos ) ${bindir}/gss_mos get ${options} ${source_filespec} ${dest_filespec} ;;
  453. cart ) ${bindir}/gss_cart get ${options} ${source_filespec} ${dest_filespec} ;;
  454. ecfs ) ${bindir}/gss_ecfs get ${options} ${source_filespec} ${dest_filespec} ;;
  455. ec ) ${bindir}/gss_ec get ${options} ${source_filespec} ${dest_filespec} ;;
  456. ftp ) ${bindir}/gss_ftp get ${options} ${source_filespec} ${dest_filespec} ;;
  457. * )
  458. err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
  459. errit 1
  460. ;;
  461. esac # protocols
  462. else
  463. # get file from source under temporary name, put to destination, remove ...
  464. # name of temporary file, almost similar to source; remove if existing:
  465. tempfile=".`echo ${source_loc} | tr ':/' '__'`"
  466. rm -f ${tempfile}
  467. # get from source:
  468. ${script} copy ${options} ${source_loc} ${tempfile}
  469. # put to destination:
  470. ${script} copy ${options} ${tempfile} ${dest_loc}
  471. # remove temporary file:
  472. rm -f ${tempfile}
  473. fi
  474. ;;
  475. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  476. # link files
  477. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  478. 'link' )
  479. # extract options
  480. hardlink='true'
  481. force=''
  482. new=''
  483. for option in ${options} ; do
  484. case ${option} in
  485. -v | --verbose ) ;;
  486. -s | --soft ) hardlink='' ;;
  487. -f | --force ) force=${option} ;;
  488. -n | --new ) new='true' ;;
  489. -* )
  490. err "$prog - unknown option '${option}' for task '${task}'"
  491. errit 1
  492. ;;
  493. esac
  494. done
  495. # extract source and destination location:
  496. source_loc=''
  497. dest_loc=''
  498. for loc in ${locations} ; do
  499. if [ -z "${source_loc}" ]; then
  500. source_loc=${loc}
  501. elif [ -z "${dest_loc}" ]; then
  502. dest_loc=${loc}
  503. else
  504. err "$prog - ERROR - task '${task}' requires source and destination locations only."
  505. errit 1
  506. fi
  507. done
  508. if [ -z "${dest_loc}" ]; then
  509. err "$prog - ERROR - task '${task}' requires source and destination locations."
  510. errit 1
  511. fi
  512. # split in protocol and file specification:
  513. source_protocol=`echo ${source_loc} | cut -d ':' -f 1`
  514. source_filespec=`echo ${source_loc} | cut -d ':' -f 2-`
  515. # name of linkfile is same as source with strange characters replaced:
  516. linkfile=`echo ${source_loc} | tr ':/.@%=\[\]' '________'`
  517. linkfile=".${linkfile}.gsslink"
  518. # call protocol specific script:
  519. case ${source_protocol} in
  520. #
  521. # create symbolic link
  522. #
  523. file )
  524. if [ ! -f ${source_filespec} ]; then
  525. # file not present; remove existing link if neccessary:
  526. test -f ${linkfile} && rm -f ${linkfile}
  527. # create empty source file if necessary ?
  528. if [ ${new} ]; then
  529. # create empty target :
  530. mkdir -p `dirname ${source_filespec}`
  531. touch ${source_filespec}
  532. else
  533. err "$prog - source file not found:"
  534. err "$prog - ${filespec}"
  535. errit 1
  536. fi
  537. fi
  538. # target exists or new empty file; link:
  539. ln -s ${force} ${source_filespec} ${linkfile}
  540. ;;
  541. #
  542. # retrieve local copy
  543. #
  544. mos | cart | ecfs | ec | ftp )
  545. # source file present ?
  546. if ${bindir}/gss exist ${source_loc} ; then
  547. # file present on storage system; get local copy:
  548. ${bindir}/gss ${verbose} copy ${force} ${source_loc} ${linkfile}
  549. else
  550. # file not present on storage system; remove existing link if neccessary:
  551. test -f ${linkfile} && rm -f ${linkfile}
  552. # create empty source file if necessary ?
  553. if [ ${new} ]; then
  554. # create empty local 'copy' :
  555. touch ${linkfile}
  556. else
  557. err "$prog - source file not found:"
  558. err "$prog - ${source_loc}"
  559. errit 1
  560. fi
  561. fi
  562. # hard link ? then store original location:
  563. test "${hardlink}" && echo ${source_loc} > "${linkfile}source"
  564. ;;
  565. #
  566. # error ...
  567. #
  568. * ) err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
  569. errit 1 ;;
  570. esac # protocols
  571. # split in protocol and file specification:
  572. dest_protocol=`echo ${dest_loc} | cut -d ':' -f 1`
  573. dest_filespec=`echo ${dest_loc} | cut -d ':' -f 2-`
  574. # destination should be a file ...
  575. if [ "${dest_protocol}" != "file" ]; then
  576. err "$prog - ERROR - destination for task '${task}' should be a local file"
  577. errit 1
  578. fi
  579. # create symbolic link:
  580. ln -s -f ${linkfile} ${dest_filespec}
  581. ;;
  582. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  583. 'unlink' )
  584. # extract location:
  585. loc=`echo ${locations} | cut -d ' ' -f 1`
  586. if [ -z "${loc}" ]; then
  587. err "$prog - ERROR - task '${task}' requires location."
  588. err "$prog - $0 $*"
  589. errit 1
  590. fi
  591. # split in protocol and file specification:
  592. protocol=`echo ${loc} | cut -d ':' -f 1`
  593. filespec=`echo ${loc} | cut -d ':' -f 2-`
  594. # local file required ...
  595. if [ "${protocol}" != "file" ]; then
  596. err "$prog - ERROR - destination for task '${task}' should be a local file"
  597. errit 1
  598. fi
  599. # symbolic link required ...
  600. if [ ! -h ${filespec} ]; then
  601. err "$prog - ERROR - destination for task '${task}' should be a symbolic link"
  602. errit 1
  603. fi
  604. # linked file:
  605. #linkfile=`readlink ${filespec}`
  606. linkfile=`ls -l ${filespec} | cut -d '>' -f 2 | tr -d ' '`
  607. # linkfile should have special name ...
  608. case ${linkfile} in
  609. .*.gsslink ) ;;
  610. * ) err "$prog - ERROR - linked file should be named '.*.gsslink', not:"
  611. err "$prog - ERROR - ${linkfile}"
  612. errit 1 ;;
  613. esac
  614. # hard link ? then restore
  615. linkfile_source="${linkfile}source"
  616. if [ -f ${linkfile_source} ]; then
  617. # original location:
  618. linkfile_loc=`cat ${linkfile_source}`
  619. # restore:
  620. ${bindir}/gss ${verbose} copy --force ${linkfile} ${linkfile_loc}
  621. # remove source info:
  622. rm -f ${linkfile_source}
  623. fi
  624. # remove linkfile:
  625. rm -f ${linkfile}
  626. # remove local file:
  627. rm -f ${filespec}
  628. ;;
  629. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  630. # operations on linked tar files
  631. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  632. 'tar-extract' )
  633. # extract source and destination location:
  634. tarfile_loc=''
  635. outfile_loc=''
  636. for loc in ${locations} ; do
  637. if [ -z "${tarfile_loc}" ]; then
  638. tarfile_loc=${loc}
  639. elif [ -z "${outfile_loc}" ]; then
  640. outfile_loc=${loc}
  641. else
  642. err "$prog - ERROR - task '${task}' requires tarfile and tarred file locations only."
  643. errit 1
  644. fi
  645. done
  646. if [ -z "${tarfile_loc}" ]; then
  647. err "$prog - ERROR - task '${task}' requires tarfile and optional tarred file locations."
  648. errit 1
  649. fi
  650. # split tarfile location in protocol and file specification:
  651. tarfileprot=`echo ${tarfile_loc} | cut -d ':' -f 1`
  652. tarfilespec=`echo ${tarfile_loc} | cut -d ':' -f 2-`
  653. # local file required ...
  654. if [ "${tarfileprot}" != "file" ]; then
  655. err "$prog - ERROR - tarfile for task '${task}' should be a local file"
  656. errit 1
  657. fi
  658. # tarred file ?
  659. if [ -z "${outfile_loc}" ]; then
  660. outfile_filespec=''
  661. else
  662. # split output file location in protocol and specification:
  663. outfile_protocol=`echo ${outfile_loc} | cut -d ':' -f 1`
  664. outfile_filespec=`echo ${outfile_loc} | cut -d ':' -f 2-`
  665. # output file should have 'file:' protocol ...
  666. if [ "${outfile_protocol}" != "file" ]; then
  667. err "$prog - ERROR - task '${task}' requires 'file:' protocol for outfile"
  668. errit 1
  669. fi
  670. fi
  671. test ${verbose} && echo "$prog - extract ..."
  672. #
  673. # extract from tarfile:
  674. # x, --extract : extract files from an archive
  675. # -f, --file=F : use archive file or device F
  676. # by default, an extracted file remains the archived time;
  677. # prevent that the times of previously touched files are overwritten:
  678. # -k, --keep-old-files : keep existing files
  679. # (if a file already exists, an error message is issued)
  680. # ensure that new files are touched:
  681. # -m, --touch : don't extract file modified time
  682. #
  683. # redirection for standard output and error:
  684. stdout=".tmp.out.$$"
  685. # untar; trap error status:
  686. # pls: Remove the -k option => was crashing on c2a
  687. if ( ${gtar} xmf ${tarfilespec} ${outfile_filespec} > ${stdout} 2>&1 ); then
  688. # ok
  689. rm -f ${stdout}
  690. else
  691. # some errors; probably about overwriting existing files
  692. echo "$prog - WARNING : errors from tar (overwriting existing files?); written to: ${stdout}"
  693. echo "$prog- WARNING from command : ${gtar} xkmf ${tarfilespec} ${outfile_filespec}"
  694. fi
  695. ;;
  696. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  697. 'tar-update' )
  698. # extract source and destination location:
  699. tarfile_loc=''
  700. infile_loc=''
  701. for loc in ${locations} ; do
  702. if [ -z "${tarfile_loc}" ]; then
  703. tarfile_loc=${loc}
  704. elif [ -z "${infile_loc}" ]; then
  705. infile_loc=${loc}
  706. else
  707. err "$prog - ERROR - task '${task}' requires tarfile and input file locations only."
  708. errit 1
  709. fi
  710. done
  711. if [ -z "${infile_loc}" ]; then
  712. err "$prog - ERROR - task '${task}' requires tarfile and input file locations."
  713. errit 1
  714. fi
  715. # split tarfile location in protocol and file specification:
  716. tarfileprot=`echo ${tarfile_loc} | cut -d ':' -f 1`
  717. tarfilespec=`echo ${tarfile_loc} | cut -d ':' -f 2-`
  718. # local file required ...
  719. if [ "${tarfileprot}" != "file" ]; then
  720. err "$prog - ERROR - tarfile for task '${task}' should be a local file"
  721. errit 1
  722. fi
  723. # tarfile should be present:
  724. if [ ! -f ${tarfilespec} ]; then
  725. err "$prog - ERROR - tar file not found :"
  726. err "$prog - ERROR - ${tarfilespec}"
  727. errit 1
  728. fi
  729. # split input file in protocol and specification:
  730. inprotocol=`echo ${infile_loc} | cut -d ':' -f 1`
  731. infilespec=`echo ${infile_loc} | cut -d ':' -f 2-`
  732. # input file should have 'file:' protocol ...
  733. if [ "${inprotocol}" != "file" ]; then
  734. err "$prog - ERROR - task '${task}' requires 'file:' protocol for infile"
  735. errit 1
  736. fi
  737. # old version present in tarfile ?
  738. #if ( ${gtar} --list --file=${tarfilespec} ${infilespec} > /dev/null 2>&1 ) ; then
  739. if ( ${gtar} tf ${tarfilespec} ${infilespec} > /dev/null 2>&1 ) ; then
  740. # keep old version in tarfile ?
  741. if [ "${keep_old_files}" = "true" ]; then
  742. # keep old; do not add new file
  743. test ${verbose} && echo "$prog - keep old ..."
  744. else
  745. # pls: on c2a long options not available anymore. Previous:
  746. # # remove old file:
  747. # test ${verbose} && echo "$prog - remove old ..."
  748. # ${gtar} --delete --file=${tarfilespec} ${infilespec}
  749. # # add to tarfile:
  750. # test ${verbose} && echo "$prog - add ..."
  751. # #${gtar} --append --file=${tarfilespec} ${infilespec}
  752. # ${gtar} rf ${tarfilespec} ${infilespec}
  753. # so simply replace with update. Now:
  754. test ${verbose} && echo "$prog - add ..."
  755. ${gtar} uvf ${tarfilespec} ${infilespec}
  756. fi
  757. else
  758. # add to (eventually empty) tarfile:
  759. test ${verbose} && echo "$prog - add ..."
  760. #${gtar} --append --file=${tarfilespec} ${infilespec}
  761. #pls: tar does not work (although it was working before) with not valid
  762. # archive. So now check that the file has non-zero size (I assume that zero
  763. # size files have not been created by a tar command basically)
  764. if [ -s ${tarfilespec} ]; then
  765. ${gtar} rf ${tarfilespec} ${infilespec}
  766. else
  767. ${gtar} cf ${tarfilespec} ${infilespec}
  768. fi
  769. fi
  770. ;;
  771. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  772. # limit directory usage
  773. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  774. 'limit')
  775. # extract source and destination location:
  776. max_usage=''
  777. bufdir_loc=''
  778. for loc in ${locations} ; do
  779. if [ -z "${max_usage}" ]; then
  780. max_usage=`echo ${loc} | cut -d ':' -f 2-`
  781. elif [ -z "${bufdir_loc}" ]; then
  782. bufdir_loc=${loc}
  783. else
  784. err "$prog - ERROR - task '${task}' requires directory and max usage only."
  785. errit 1
  786. fi
  787. done
  788. if [ -z "${bufdir_loc}" ]; then
  789. err "$prog - ERROR - task '${task}' requires directory and max usage."
  790. errit 1
  791. fi
  792. # no infinite usage ? then remove files if necessary:
  793. if [[ ${max_usage} != [Ii][Nn][Ff] ]]; then
  794. # convert max usage to kB :
  795. mu=`echo ${max_usage} | tr -d 'kKmMgG'`
  796. case ${max_usage} in
  797. *g | *G ) max_usage_kB=`expr ${mu} \* 1000000` ;;
  798. *m | *M ) max_usage_kB=`expr ${mu} \* 1000` ;;
  799. * ) max_usage_kB=${mu} ;;
  800. esac
  801. test ${verbose} && echo "$prog - max usage : ${max_usage_kB} kB"
  802. # split in protocol and file specification:
  803. protocol=`echo ${bufdir_loc} | cut -d ':' -f 1`
  804. bufdir=`echo ${bufdir_loc} | cut -d ':' -f 2-`
  805. # local directory required ...
  806. if [ "${protocol}" != "file" ]; then
  807. err "$prog - ERROR - buffer directory for task '${task}' should be local"
  808. errit 1
  809. fi
  810. # buffer exists ?
  811. if [ ! -d ${bufdir} ]; then
  812. err "$prog - ERROR - directory to limit not found:"
  813. err "$prog - ERROR - ${bufdir_loc}"
  814. errit 1
  815. fi
  816. # current usage:
  817. curr_usage_kB=`du -sk ${bufdir} | cut -f 1`
  818. test ${verbose} && echo "$prog - curr usage : ${curr_usage_kB} kB"
  819. # disk usage exceeds maximum ? then remove some files:
  820. if [ ${curr_usage_kB} -gt ${max_usage_kB} ]; then
  821. # loop over files in buffer;
  822. # sort on time (-t), oldest first (-r);
  823. # for symbolic links, show times for linked files (-L)
  824. # (if a symbolic link is touched, only time of linked file changes)
  825. for file in `ls -t -r -L ${bufdir}` ; do
  826. # tmp directory ? must be old scratch ...
  827. test -d ${file} && rm -r -f ${file}
  828. # symbolic link to gsslink ? then remove gsslink too
  829. if [ -h ${file} ]; then
  830. #linkfile=`readlink ${file}`
  831. linkfile=`ls -l ${file} | cut -d '>' -f 2 | tr -d ' '`
  832. # gss link ?
  833. if [[ ${linkfile} = .*.gsslink ]]; then
  834. # update disk usages for linkfile:
  835. file_usage_kB=`du -k ${bufdir}/${linkfile} | cut -f 1`
  836. curr_usage_kB=`expr ${curr_usage_kB} - ${file_usage_kB}`
  837. # source file present ? update disk usage for this file:
  838. linkfile_source="${linkfile}src"
  839. if [ -f ${linkfile_source} ]; then
  840. file_usage_kB=`du -k ${bufdir}/${linkfile_source} | cut -f 1`
  841. curr_usage_kB=`expr ${curr_usage_kB} - ${file_usage_kB}`
  842. fi
  843. # unlink, eventually the file is restored:
  844. #${bindir}/gss ${verbose} unlink ${file}
  845. ${bindir}/gss unlink ${file}
  846. fi
  847. fi
  848. # remove file; update disk usage:
  849. if [ -f ${file} ]; then
  850. file_usage_kB=`du -k ${bufdir}/${file} | cut -f 1`
  851. curr_usage_kB=`expr ${curr_usage_kB} - ${file_usage_kB}`
  852. rm -f ${bufdir}/${file}
  853. fi
  854. test ${verbose} && echo "$prog - ${curr_usage_kB} ${file_usage_kB} ${file}"
  855. # current usage smaller than maximum ? then leave
  856. test ${curr_usage_kB} -le ${max_usage_kB} && break
  857. done # loop over old files
  858. fi # usage > max
  859. fi # max usage < Inf
  860. ;;
  861. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  862. # task ???
  863. # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  864. * )
  865. err "$prog - ERROR - unsupported task '${task}'"
  866. errit 1
  867. ;;
  868. esac # tasks
  869. # ----------------------------------------------------------
  870. # --- end
  871. # ----------------------------------------------------------
  872. test ${verbose} && echo "$prog - end"
  873. # ok
  874. exit 0