NASA
/
shift
mirror of https://github.com/pkolano/shift.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Shift 8.0

master
Paul Kolano 2021-09-16 17:26:08 -07:00
parent ffb7f5fa1f
commit 1fbeb53e3b
9 changed files with 623 additions and 780 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
CHANGES
View File

@ -341,3 +341,22 @@ CHANGES
- Fixed manager exception during read of corrupt/incomplete gzi files
- Fixed handling of manager disk exhaustion to prevent finalized metadata
- Fixed potential divide by zero exceptions during throttling
- Fixed lustre handling in shift-bin to current lustre API
* Shift 8.0 (09/16/21)
- Note that metadata is not backward compatible with previous versions
- Added plots by tool, subnet, and batch size
- Added detection of BeeGFS file systems
- Changed plots from line to heatmap for improved scalability
- Changed processing so batches only contain single operation type
- Changed monitoring to support manager hosts across shared file system
- Changed naming of built-in tools in --stats and new --plot by tool
- Fixed detection of DMF file systems when using xdsm mount option
- Fixed detection of GPFS file systems when mmlsmgr not user-accessible
- Fixed --wait status retrieval when integrated with Mesh framework
- Fixed shift-bin striping with lustre progressive layouts when --stripe=0
- Fixed use of XS Data::MessagePack when embedded pure perl version differs
- Fixed scalability of built-in yEnc encoding/decoding
- Fixed unnecessary metadata traversal when showing detailed status
- Fixed application of lustre striping expression to directories
- Removed bbcp and gridftp support since reported fatal bugs never fixed

6
INSTALL
View File

@ -63,15 +63,10 @@ Shift Installation and Configuration
(https://metacpan.org/pod/Data::MessagePack)
o IO::Socket::SSL - allows fish-tcp encryption with --secure
(https://metacpan.org/pod/IO::Socket::SSL)
o bbcp - high speed remote copy
(http://www.slac.stanford.edu/~abh/bbcp)
(note latest v17.12.00.00.0 fails when overwriting existing files)
o bbftp - high speed remote copy
(http://doc.in2p3.fr/bbftp)
o gnuplot >= 5.0 - allow display of --plot output
(http://gnuplot.info)
o gridftp - high speed remote copy
(http://toolkit.globus.org/toolkit/data/gridftp)
o mcp/msum >= 1.76.7 - high speed local copy/sum
(http://mutil.sf.net)
o mesh - lightweight single sign-on via ssh publickey authentication
@ -97,7 +92,6 @@ Shift Installation and Configuration
o su - become non-root process to access manager during root transfers
o sysctl - determine number of cpus on BSD
o touch - change symlink modification time
o unbuffer - interleave stdout/stderr when using gridftp
3. Build (optional - linux only)

3
README.md
View File

@ -59,8 +59,7 @@ Shift includes the following features, among others:
- fully self-contained besides perl core and ssh
- automatic detection and selection of higher performance transports and
hash utilities when available including bbcp, bbftp, gridftp, mcp, msum,
and rsync
hash utilities when available including bbftp, mcp, msum, and rsync
- automatic many-to-many parallelization of single and multi-file
transfers with file system equivalence detection and rewriting

11
c/shift-bin.c
View File

@ -1,5 +1,5 @@
//
// Copyright (C) 2012-2020 United States Government as represented by the
// Copyright (C) 2012-2021 United States Government as represented by the
// Administrator of the National Aeronautics and Space Administration
// (NASA). All Rights Reserved.
//
@ -252,6 +252,15 @@ int do_getstripe(char *file) {
//////////////////////
int do_setstripe(char *file, int scount, unsigned long long ssize, char *pool) {
#ifndef _NO_LUSTRE
if (scount == 0 && ssize == 0) {
FILE *rc = fopen(file, "w");
if (rc != NULL) {
fclose(rc);
return 0;
} else {
return -1;
}
}
return llapi_file_create_pool(file, ssize, -1, scount, 0, pool);
#else
return -1;

140
doc/shiftc.1
View File

@ -58,8 +58,7 @@ automatic striping of files transferred to Lustre file systems
fully self-contained besides perl core and ssh
.IP -
automatic detection and selection of higher performance transports and
hash utilities when available including bbcp, bbftp, gridftp, mcp,
msum, and rsync
hash utilities when available including bbftp, mcp, msum, and rsync
.IP -
automatic many-to-many parallelization of single and multi-file
transfers with file system equivalence detection and rewriting
@ -130,10 +129,11 @@ given in following sections.
\-\-mgr\-user=USER access manager host as USER
\-\-monitor[=FORMAT] monitor progress of running transfers
(FORMAT one of {color,csv,pad})
\-\-plot[=[BY:]LIST] plot detailed performance when piped to gnuplot
(BY one of {client,host,id,user})
(LIST subset of {chattr,cksum,cp,find,io,ln,meta,
mkdir,sum})
\-\-plot[=[BY[:/]]LIST] plot detailed performance when piped to gnuplot
(BY one of {client,fs,host,id,net,user})
(LIST subset of {bbftp,chattr,cksum,cp,find,fish,fish-tcp,
io,ln,mcp,meta, mkdir,msum,rsync,shift-cp,
shift-sum,sum,tool})
\-\-restart[=ignore] restart transfer with given \-\-id [ignoring errors]
\-\-search=REGEX show only status/history matching REGEX
\-\-state=STATE show status of only those operations in STATE
@ -153,12 +153,10 @@ given in following sections.
(use suffix {k,m,b/g,t} for 1E{3,6,9,12}) [1k]
\-\-interval=NUM adjust batches to run for around NUM seconds [30]
\-\-local=LIST set local transport mechanism to one of LIST
(LIST subset of {bbcp,bbftp,fish,fish-tcp,gridftp,
mcp,rsync,shift})
(LIST subset of {bbftp,fish,fish-tcp,mcp,rsync,shift})
\-\-preallocate=NUM preallocate files when sparsity under NUM percent
\-\-remote=LIST set remote transport mechanism to one of LIST
(LIST subset of {bbcp,bbftp,fish,fish-tcp,gridftp,
rsync,shift})
(LIST subset of {bbftp,fish,fish-tcp,rsync,shift})
\-\-retry=NUM retry failed operations up to NUM times [2]
\-\-size=SIZE process transfer in batches of at least SIZE bytes
(use suffix {k,m,g,t} for {KB,MB,GB,TB}) [4g]
@ -356,10 +354,9 @@ there is actually a need to process destination files during the
transfer.
.IP "\fB\-\-ports=NUM1:NUM2\fP"
Use ports from the range NUM1-NUM2 for the data streams of TCP-based
transports (currently, bbcp, bbftp, fish-tcp, and gridftp). All
connections originate from the client host so the given port range must
be allowed on the network path to the remote host and by the remote host
itself.
transports (currently, bbftp and fish-tcp). All connections
originate from the client host so the given port range must be allowed
on the network path to the remote host and by the remote host itself.
.IP "\fB\-R, \-r, \-\-recursive\fP"
Transfer directories recursively. This option implies
\fB\-\-no\-dereference\fP.Note that any symbolic links pointing
@ -521,24 +518,39 @@ coloring, respectively. When \fB\-\-id\fP is specified, only the given
transfer will be shown. When all transfers (or the one specified)
have completed, the command will exit. This option may be used with
\fB\-\-wait\fP to monitor progress while waiting.
.IP "\fB\-\-plot=[=[BY:]LIST]\fP"
.IP "\fB\-\-plot=[=[BY[:/]]LIST]\fP"
Produce output suitable for piping into gnuplot (version 5 or above)
that shows detailed performance over time across all transfers. The
\fB\-\-id\fP and \fB\-\-state\fP options may be used to plot only a
single transfer or transfers in a particular state, respectively. The
default plot will show the aggregate performance of each I/O operation
(i.e. cp, sum, and cksum) and the aggregate performance of each metadata
operation (i.e. find, mkdir, ln, and chattr). I/O operations are
plotted against the left y-axis while metadata operations are plotted
against the right y-axis. The list of plotted items may be changed by
giving a comma-separated list consisting of one of more of {chattr,
cksum, cp, find, io, ln, meta, mkdir, sum}. Note that "io" is a
shorthand for "cp,sum,cksum" and "meta" is a shorthand for
"find,mkdir,ln,chattr". The list of items may be grouped by any of
{host, id, user} by prefixing one of these terms to the list. For
example, \fB\-\-plot=id:cp\fP would show a curve for the copy
performance of each tranfer id. When a grouping is given without a
specific list of metrics (e.g. \fB\-\-plot=id\fP), "io" is assumed.
operation (i.e. find, mkdir, ln, and chattr) across all of the user's
transfers. Operations and/or additional groupings are shown on the
left y-axis axis across time on the x-axis with heat-based coloring
indicating MB/s for I/O operations or operations per second for metadata
operations. In addition, aggregate I/O and metadata performance will be
shown as an overlayed point plot with green and blue points,
respectively.
.IP
The list of plotted items may be changed by giving a comma-separated
list consisting of one or more of the stages {chattr, cksum, cp, find,
io, ln, meta, mkdir, sum} and/or one or more of the tools {bbftp, fish,
fish-tcp, mcp, msum, rsync, shift-cp, shift-sum}. Note that "io" is a
shorthand for "cp,sum,cksum", "meta" is a shorthand for
"find,mkdir,ln,chattr", and "tool" is a shorthand for
"bbftp,fish,fish-tcp,mcp,msum,rsync,shift-cp,shift-sum".
.IP
The list of items may be grouped by any of {client, fs, host, id, net,
user} by prefixing one of these terms to the list. For example,
\fB\-\-plot=id:cp\fP would show a plot of the copy performance achieved
by each transfer id. When a grouping is given without a specific list
of metrics (e.g. \fB\-\-plot=id\fP), "io" is assumed. When a slash "/"
is used instead of colon ":", a heatmap-based bubble plot will be
created with the size of each circle indicating the relative size of the
batch of operations. For example, \fB\-\-plot=fs/tool\fP would show a
plot of the performance that each tool achieved on each file system
with relative batch size.
.IP "\fB\-\-restart[=ignore]\fP"
Restart the transfer associated with the given \fB\-\-id\fP that was
stopped due to unrecoverable errors or stopped explicitly via
@ -623,12 +635,12 @@ Some advanced options are available to tune various aspects of shiftc
behavior. These options are not needed by most users.
.IP "\fB\-\-bandwidth=BITS\fP"
Choose the TCP window size and number of TCP streams of TCP-based
transports (currently, bbcp, bbftp, fish-tcp, and gridftp) based on
the given bits per second. The suffixes k, m, g, and t may be used for
Kb, Mb, Gb, and Tb, respectively. The default bandwidth is estimated to
be 10 Gb/s if a 10 GE adapter is found on the client host, 1 Gb/s if the
client host can be resolved to an organization domain (by default, one
of the six original generic top-level domains), and 100 Mb/s otherwise.
transports (currently, bbftp and fish-tcp) based on the given bits per
second. The suffixes k, m, g, and t may be used for Kb, Mb, Gb, and Tb,
respectively. The default bandwidth is estimated to be 10 Gb/s if a 10
GE adapter is found on the client host, 1 Gb/s if the client host can be
resolved to an organization domain (by default, one of the six original
generic top-level domains), and 100 Mb/s otherwise.
.IP "\fB\-\-buffer=SIZE\fP"
Use memory buffer(s) of the given size when configurable in the
underlying tranport being utilized (currently, all but rsync). The
@ -662,13 +674,13 @@ for manager locks. To make batch selection completely static, use
.IP "\fB\-\-local=LIST\fP"
Specify one or more local transports to be used for the transfer in
order of preference, separated by commas. Valid transports for this
option currently include bbcp, bbftp, cp, fish, fish-tcp, gridftp,
mcp, and rsync. Note that the given transport(s) will be given
priority, but may not be used in some cases (e.g. rsync is not capable
of transferring a specific portion of a file as needed by verification
mode). In such cases, the default transport based on File::Copy will be
used. The tool actually used for each file operation can be shown using
\fB\-\-status\fP with \fB\-\-id\fP set to the given transfer identifier.
option currently include bbftp, cp, fish, fish-tcp, mcp, and rsync.
Note that the given transport(s) will be given priority, but may not be
used in some cases (e.g. rsync is not capable of transferring a specific
portion of a file as needed by verification mode). In such cases, the
default transport based on File::Copy will be used. The tool actually
used for each file operation can be shown using \fB\-\-status\fP with
\fB\-\-id\fP set to the given transfer identifier.
.IP "\fB\-\-preallocate=NUM\fP"
Preallocate files when their sparsity is under the given percent, where
sparsity is defined as the number of bytes a file takes up on disk
@ -681,14 +693,13 @@ transport due to their use of temporary files.
.IP "\fB\-\-remote=LIST\fP"
Specify one or more remote transports to be used for the transfer in
order of preference, separated by commas. Valid transports for this
option currently include bbcp, bbftp, fish, fish-tcp, gridftp, rsync,
and sftp. Note that the given transport(s) will be given priority, but
may not be used in some cases (e.g. bbftp is not capable of transferring
files with spaces in their names and is also incompatible with
\fB\-\-secure\fP). In such cases, the default transport based on sftp
will be used. The tool actually used for each file operation can be
shown using \fB\-\-status\fP with \fB\-\-id\fP set to the given transfer
identifier.
option currently include bbftp, fish, fish-tcp, rsync, and sftp. Note
that the given transport(s) will be given priority, but may not be used
in some cases (e.g. bbftp is not capable of transferring files with
spaces in their names and is also incompatible with \fB\-\-secure\fP).
In such cases, the default transport based on sftp will be used. The
tool actually used for each file operation can be shown using
\fB\-\-status\fP with \fB\-\-id\fP set to the given transfer identifier.
.IP "\fB\-\-retry=NUM\fP"
Retry operations deemed recoverable up to the given number of attempts
per file. The default number of retries is 2. A value of zero disables
@ -733,13 +744,13 @@ resulting tar files may still be larger than specified when source files
exist that are larger than the given size.
.IP "\fB\-\-streams=NUM\fP"
Use the given number of TCP streams in TCP-based transports (currently,
bbcp, bbftp, fish-tcp, and gridftp). The default is the number of
streams necessary to fully utilize the specified/estimated bandwidth
using the maximum TCP window size. Note that it is usually preferable
to specify \fB\-\-bandwidth\fP, which allows an appropriate number of
streams to be set automatically. Increasing the number of streams can
increase performance when the maximum window size is set too low or
there is cross-traffic on the network, but too high a value can decrease
bbftp and fish-tcp). The default is the number of streams necessary
to fully utilize the specified/estimated bandwidth using the maximum TCP
window size. Note that it is usually preferable to specify
\fB\-\-bandwidth\fP, which allows an appropriate number of streams to be
set automatically. Increasing the number of streams can increase
performance when the maximum window size is set too low or there is
cross-traffic on the network, but too high a value can decrease
performance due to increased congestion and packet loss.
.IP "\fB\-\-stripe=[CEXP][::[SEXP][::PEXP]]\fP"
By default, a file transferred to a Lustre file system will be striped
@ -797,15 +808,14 @@ to 33%, but does not allow bits corrupted during the initial read to be
detected.
.IP "\fB\-\-window=SIZE\fP"
Use a TCP send/receive window of the given size in TCP-based transports
(currently, bbcp, bbftp, fish-tcp, and gridftp). The suffixes k, m,
g, and t may be used for KB, MB, GB, and TB, respectively. The default
is the product of the specified/estimated bandwidth and the round-trip
time between source and destination. Note that it is usually preferable
to specify \fB\-\-bandwidth\fP, which allows an appropriate window size
to be set automatically. Increasing the window size allows TCP to
operate more efficiently over high bandwidth and/or high latency
networks, but too high a value can overrun the receiver and cause packet
loss.
(currently, bbftp and fish-tcp). The suffixes k, m, g, and t may be
used for KB, MB, GB, and TB, respectively. The default is the product
of the specified/estimated bandwidth and the round-trip time between
source and destination. Note that it is usually preferable to specify
\fB\-\-bandwidth\fP, which allows an appropriate window size to be set
automatically. Increasing the window size allows TCP to operate more
efficiently over high bandwidth and/or high latency networks, but too
high a value can overrun the receiver and cause packet loss.
./"################################################################
.SH "TRANSFER THROTTLING"
./"################################################################
@ -1142,5 +1152,5 @@ shiftc was written by Paul Kolano.
./"################################################################
.SH "SEE ALSO"
./"################################################################
bbcp(1), bbftp(1), cp(1), Date::Parse(3), globus-url-copy(1), mcp(1),
msum(1), perlre(1), perlsyn(1), rsync(1), scp(1), sftp(1)
bbftp(1), cp(1), Date::Parse(3), mcp(1), msum(1), perlre(1),
perlsyn(1), rsync(1), scp(1), sftp(1)

8
etc/shiftrc
View File

@ -78,20 +78,12 @@ user_dir /home/%u/.shift
# (example: mcp,shift,fish,fish-tcp,rsync,bbftp)
#local_small shift,fish,fish-tcp
# command-line options that will be used by bbcp on client hosts
# (example: opts_bbcp -s 4 -w 4194304)
#opts_bbcp nodefault
# behavior commands that will be used by bbftp on client hosts
# (see the "behavior commands" section of bbftp man page for details)
# (options must be separated by "\n" as shown in example below)
# (example: opts_bbftp setnbstream 4\nsetrecvwinsize 4096\nsetsendwinsize 4096)
#opts_bbftp nodefault
# command-line options that will be used by globus-url-copy on client hosts
# (example: opts_gridftp -p 4 -tcp-bs 4194304)
#opts_gridftp nodefault
# command-line options that will be used by mcp on client hosts
# (if mcp >= 1.822.1, a --preallocate setting is recommended for DMF on CXFS)
#opts_mcp --double-buffer

26
perl/shift-aux
View File

@ -66,7 +66,7 @@ use Symbol qw(gensym);
use Sys::Hostname;
use Text::ParseWords;
our $VERSION = 7.05;
our $VERSION = 8.0;
# do not die when receiving sigpipe
$SIG{PIPE} = 'IGNORE';
@ -1260,7 +1260,7 @@ sub mount {
$mnt{opts} = /[\(,]ro[\),]/ ? "ro" : "rw";
# acl support is the default unless explicitly disabled
$mnt{opts} .= ",acl" if (/[\(,]acl[\),]/ || $acl && !/[\(,]noacl[\),]/);
$mnt{opts} .= ",dmf" if (/[\(,]dm(ap)?i[\),]/);
$mnt{opts} .= ",dmf" if (/[\(,](dmapi|dmi|xdsm)[\),]/);
$mnt{opts} .= ",xattr" if (/[\(,]user_xattr[\),]/);
#TODO: need to escape local and remote?
(my $dev, $mnt{local}, my $type) = ($1, $2, $3)
@ -1291,17 +1291,19 @@ sub mount {
$mnt{servers} =~ s/@\w*//g;
$mnt{servers} = join("|", map {$_ = fqdn($_)} split(/:/, $mnt{servers}));
} elsif ($type eq 'gpfs') {
# gpfs servers do not appear in mount output so call mmlsmgr
my $srv = open3_get([-1, undef, -1], "mmlsmgr $dev");
# try a default location if not in path
$srv = open3_get([-1, undef, -1],
"/usr/lpp/mmfs/bin/mmlsmgr $dev") if (!$srv);
next if (!defined $srv);
# output is file system then server ip address
if ($srv =~ /^(\w+)\s+(\d+\.\d+\.\d+\.\d+)/m) {
$mnt{remote} = "/$1";
$mnt{servers} = fqdn($2);
# gpfs servers do not appear in mount output so read config
if (open(FILE, "/var/mmfs/gen/mmfs.cfg")) {
while (<FILE>) {
s/^\s+|\s+$//g;
if (/^clustername\s+(\S+)/i) {
$mnt{servers} = $1;
$mnt{remote} = "/" . $mnt{servers};
last;
}
}
close FILE;
}
next if (!$mnt{servers});
} elsif ($mnt{opts} =~ /,dmf/) {
# always report dmf file systems even if local
$mnt{servers} = $mnt{host};

807
perl/shift-mgr
View File

File diff suppressed because it is too large Load Diff

383
perl/shiftc
View File

@ -83,7 +83,7 @@ use constant SFTP_TRUNC => 0x10;
use constant SFTP_WRITE => 0x02;
use constant SFTP_EXCL => 0x20;
our $VERSION = 7.08;
our $VERSION = 8.0;
$Data::Dumper::Indent = 0;
$Data::Dumper::Purity = 1;
@ -233,7 +233,7 @@ if ($opts{h}) {
if (scalar(@ARGV) > 0) {
# ignore command if key generation forced and no arguments given
my $cmd = shift @ARGV;
if ($cmd !~ /(?:^|\W)(?:bbcp|bbftp|bbscp|globus-url-copy|mesh-keykill|mesh-keytime|pcp\+|rm|rsync|scp|sftp|shiftc?|ssh|ssh-balance)$/ && $< != 0) {
if ($cmd !~ /(?:^|\W)(?:bbcp|bbftp|bbscp|mesh-keykill|mesh-keytime|pcp\+|rm|rsync|scp|sftp|shiftc?|ssh|ssh-balance)$/ && $< != 0) {
# resolve all symlinks to support links to host:/path in VFS
# (exclude rm so linked targets are not removed)
# (exclude root to prevent unintended exposure/modification)
@ -245,7 +245,7 @@ if (scalar(@ARGV) > 0) {
if ($cmd =~ /(?:^|\W)pwd$/) {
print "$ENV{PWD}\n";
exit;
} elsif ($cmd !~ /(?:^|\W)(?:bbcp|bbftp|bbscp|globus-url-copy|mesh-keykill|mesh-keytime|pcp\+|rsync|scp|sftp|ssh|ssh-balance)$/ &&
} elsif ($cmd !~ /(?:^|\W)(?:bbcp|bbftp|bbscp|mesh-keykill|mesh-keytime|pcp\+|rsync|scp|sftp|ssh|ssh-balance)$/ &&
!$argv_hostpath && (!hostpath($ENV{PWD}) ||
grep(!/^[-\/]/, @ARGV) == 0 || $cmd =~ /(?:^|\W)complete$/ &&
$ARGV[1] =~ /^\//)) {
@ -532,16 +532,6 @@ if ($ARGV[0] =~ /(?:^|\W)(?:scp|sftp)$/) {
splice(@ARGV, 1, 0, ("-S", "$opts{ssh} %H bbcp", "-T", "$opts{ssh} %H bbcp"));
} elsif ($ARGV[0] =~ /(?:^|\W)(?:bbftp|bbscp)$/) {
splice(@ARGV, 1, 0, ("-L", $opts{ssh}));
} elsif ($ARGV[0] =~ /(?:^|\W)globus-url-copy$/) {
my $dir = glob("~/.globus");
mkdir $dir if (! -d $dir);
my $file = "$dir/gridftp-ssh";
open(FILE, '>', $file);
print FILE "#!/bin/sh\n$opts{ssh} \$2 sshftp";
close FILE;
chmod(0700, $file);
# reduce $argc since no additional args are spliced onto @ARGV
$argc--;
} elsif ($ARGV[0] =~ /(?:^|\W)pcp\+$/) {
splice(@ARGV, 1, 0, ("-s", "$opts{ssh_l} $opts{p}"));
} elsif ($ARGV[0] =~ /(?:^|\W)rsync$/) {
@ -1372,10 +1362,11 @@ sub shift_ {
print " --mgr-user=USER access manager host as USER\n";
print " --monitor[=FORMAT] monitor progress of running transfers\n";
print " (FORMAT one of {color,csv,pad})\n";
print " --plot[=[BY:]LIST] plot detailed performance when piped to gnuplot\n";
print " (BY one of {client,host,id,user})\n";
print " (LIST subset of {chattr,cksum,cp,find,io,ln,meta,\n";
print " mkdir,sum})\n";
print " --plot[=[BY[:/]]LIST] plot detailed performance when piped to gnuplot\n";
print " (BY one of {client,fs,host,id,net,user})\n";
print " (LIST subset of {bbftp,chattr,cksum,cp,find,fish,\n";
print " fish-tcp,io,ln,mcp,meta,mkdir,msum,\n";
print " rsync,shift-cp,shift-sum,sum,tool})\n";
print " --restart[=ignore] restart transfer with given --id [ignoring errors]\n";
print " --search=REGEX show only status/history matching REGEX\n";
print " --state=STATE show status of only those operations in STATE\n";
@ -1395,12 +1386,10 @@ sub shift_ {
print " (use suffix {k,m,b/g,t} for 1E{3,6,9,12}) [1k]\n";
print " --interval=NUM adjust batches to run for around NUM seconds [30]\n";
print " --local=LIST set local transport mechanism to one of LIST\n";
print " (LIST subset of {bbcp,bbftp,fish,fish-tcp,gridftp,\n";
print " mcp,rsync,shift})\n";
print " (LIST subset of {bbftp,fish,fish-tcp,mcp,rsync,shift})\n";
print " --preallocate=NUM preallocate files when sparsity under NUM percent\n";
print " --remote=LIST set remote transport mechanism to one of LIST\n";
print " (LIST subset of {bbcp,bbftp,fish,fish-tcp,gridftp,\n";
print " rsync,shift})\n";
print " (LIST subset of {bbftp,fish,fish-tcp,rsync,shift})\n";
print " --retry=NUM retry failed operations up to NUM times [2]\n";
print " --size=SIZE process transfer in batches of at least SIZE bytes\n";
print " (use suffix {k,m,g,t} for {KB,MB,GB,TB}) [4g]\n";
@ -1569,10 +1558,22 @@ sub shift_ {
}
waitpid($pid, 0);
=for mesh
# use agent key directly since agent will have been killed by child
$opts{sshmp} =~ s/^(ssh)/$1 -i $agent_key/;
# use new agent since initial agent will have been killed by child
$agent_sock = open3_get([-1, undef, -1], "ssh-agent -c");
if ($agent_sock =~ /SSH_AGENT_PID\s+(\d+);/) {
$opts{k} = $1;
}
$agent_sock = $1 if ($agent_sock =~ /SSH_AUTH_SOCK\s+([^;]+);/);
$ENV{SSH_AUTH_SOCK} = $agent_sock;
open3_get([-1, undef], "ssh-add $agent_key");
=cut mesh
my $out = shift_mgr("--status --state=none --id=$opts{id}");
=for mesh
if ($opts{k}) {
# kill new agent
kill(SIGTERM, $opts{k}) && waitpid($opts{k}, 0);
}
=cut mesh
if (defined $opts{monitor}) {
print "\e[1A\e[K" foreach (1 .. 5);
}
@ -2201,7 +2202,6 @@ sub shift_loop {
next if ($t =~ /^(shift|fish(-tcp)?)$/);
foreach my $path (split(/:/, $ENV{PATH})) {
if (-x "$path/$t" ||
$t eq 'gridftp' && -x "$path/globus-url-copy" ||
$t =~ /^fish(-tcp)$/ && -x "$path/$opts{caux}") {
$have{$t} = 1;
last;
@ -2230,7 +2230,7 @@ sub shift_loop {
$rtthost->{$1} = 1;
} elsif ($args[0] =~ /^(?:exclude|include)$/) {
$opts{$args[0]} = thaw(unescape($op{text}));
} elsif ($args[0] =~ /^(?:bandwidth|buffer|create-tar|cron|dereference|extract-tar|find-files|force|get_host|ignore-times|index-tar|newer|offline|older|opts_bbcp|opts_bbftp|opts_gridftp|opts_mcp|opts_msum|opts_ssh|ports|preallocate|preserve|recall|sanity|secure|streams|stripe|stripe-pool|stripe-size|sum_split|sum_type|sync|sync_host|threads|window|verify|verify-fast)$/) {
} elsif ($args[0] =~ /^(?:bandwidth|buffer|create-tar|cron|dereference|extract-tar|find-files|force|get_host|ignore-times|index-tar|newer|offline|older|opts_bbftp|opts_mcp|opts_msum|opts_ssh|ports|preallocate|preserve|recall|sanity|secure|streams|stripe|stripe-pool|stripe-size|sum_split|sum_type|sync|sync_host|threads|window|verify|verify-fast)$/) {
$opts{$args[0]} = defined $op{text} ?
unescape($op{text}) : 1;
}
@ -2549,7 +2549,7 @@ sub shift_mounts {
$mnt{opts} = /[\(,]ro[\),]/ ? "ro" : "rw";
# acl support is the default unless explicitly disabled
$mnt{opts} .= ",acl" if (/[\(,]acl[\),]/ || $acl && !/[\(,]noacl[\),]/);
$mnt{opts} .= ",dmf" if (/[\(,]dm(ap)?i[\),]/);
$mnt{opts} .= ",dmf" if (/[\(,](dmapi|dmi|xdsm)[\),]/);
$mnt{opts} .= ",xattr" if (/[\(,]user_xattr[\),]/);
#TODO: need to escape local and remote?
(my $dev, $mnt{local}, my $type) = ($1, $2, $3)
@ -2590,18 +2590,32 @@ sub shift_mounts {
# lustre may have extra @id and multiple colon-separated servers
$mnt{servers} =~ s/@\w*//g;
$mnt{servers} = join("|", map {$_ = fqdn($_)} split(/:/, $mnt{servers}));
} elsif ($type eq 'gpfs') {
# gpfs servers do not appear in mount output so call mmlsmgr
my $srv = open3_get([-1, undef, -1], "mmlsmgr $dev");
# try a default location if not in path
$srv = open3_get([-1, undef, -1],
"/usr/lpp/mmfs/bin/mmlsmgr $dev") if (!$srv);
} elsif ($type eq 'beegfs') {
# beegfs servers do not appear in mount output so call beegfs-ctl
my $srv = open3_get([-1, undef, -1],
"beegfs-ctl --listnodes --nodetype=management --mount=$mnt{local}");
next if (!defined $srv);
# output is file system then server ip address
if ($srv =~ /^(\w+)\s+(\d+\.\d+\.\d+\.\d+)/m) {
$mnt{remote} = "/$1";
$mnt{servers} = fqdn($2);
chomp $srv;
# output is host name then id
my @hosts;
push(@hosts, fqdn($1)) while ($srv =~ /^([\w-.]+)(\s|$)/mg);
next if (!scalar(@hosts));
$mnt{servers} = join("|", @hosts);
$mnt{remote} = "/" . $mnt{servers};
} elsif ($type eq 'gpfs') {
# gpfs servers do not appear in mount output so read config
if (open(FILE, "/var/mmfs/gen/mmfs.cfg")) {
while (<FILE>) {
s/^\s+|\s+$//g;
if (/^clustername\s+(\S+)/i) {
$mnt{servers} = $1;
$mnt{remote} = "/" . $mnt{servers};
last;
}
}
close FILE;
}
next if (!$mnt{servers});
} elsif ($mnt{opts} =~ /,dmf/) {
# always report dmf file systems even if local
$mnt{servers} = $mnt{host};
@ -3254,16 +3268,10 @@ sub transport {
my $tool = $opts{$type}->[
($i + $ref->{try}) % scalar(@{$opts{$type}})];
next if (
# bbcp does not encrypt and cannot handle partial transfers or
# (using --infiles) colon/ff/cr/lf/tab/vt in file names
$tool eq 'bbcp' && ($opts{secure} || $ref->{bytes} ||
"$src$dst" =~ /[:\f\n\r\t\x0b]/) ||
# bbftp does not encrypt and cannot handle partial transfers or
# whitespace/vt in file names
$tool eq 'bbftp' && ($opts{secure} || $ref->{bytes} ||
"$src$dst" =~ /[\s\x0b]/) ||
# gridftp cannot handle tar ops due to differing src/dst offsets
$tool eq 'gridftp' && defined $ref->{tar_bytes} ||
# rsync cannot handle partial transfers, and
# (using --files-from) cannot handle cr/lf in file names
$tool eq 'rsync' && ($ref->{bytes} || "$src$dst" =~ /[\n\r]/));
@ -3298,16 +3306,12 @@ sub transport {
foreach my $tool (keys %tools) {
next if (!scalar(@{$tools{$tool}}));
if ($tool eq 'bbcp') {
transport_bbcp($host, $tools{$tool});
} elsif ($tool eq 'bbftp') {
if ($tool eq 'bbftp') {
transport_bbftp($host, $tools{$tool});
} elsif ($tool eq 'fish') {
transport_fish($host, $tools{$tool});
} elsif ($tool eq 'fish-tcp') {
transport_fish($host, $tools{$tool}, 1);
} elsif ($tool eq 'gridftp') {
transport_gridftp($host, $tools{$tool});
} elsif ($tool eq 'mcp') {
transport_mcp($host, $tools{$tool});
} elsif ($tool eq 'rsync') {
@ -3330,139 +3334,6 @@ sub transport {
return $rsize;
}
########################
#### transport_bbcp ####
########################
sub transport_bbcp {
my ($host, $tcmds) = @_;
my %errs;
my ($fh, $tmp) = sftp_tmp();
my $sep = chr(0);
my ($shost, $spath, $dhost, $dpath, $args);
if ($host eq 'localhost') {
$shost = "";
# bbcp assumes host name instead of localhost when host not given
$dhost = "localhost:";
$args = " -S bbcp -T bbcp";
} else {
my $ssh_l;
if ($host =~ /@/) {
($ssh_l, $host) = split(/@/, $host);
$ssh_l = " -l " . $ssh_l if ($ssh_l);
}
$args = " -S '$opts{ssh}$ssh_l %H bbcp' -T '$opts{ssh}$ssh_l %H bbcp'";
}
foreach my $cmd (@{$tcmds}) {
my ($op, $src, $dst, $ref) = @{$cmd};
$ref->{tool} = "bbcp";
if (!defined $shost) {
$shost = $op eq 'get' ? $host . ":" : "";
$dhost = $op eq 'put' ? $host . ":" : "";
}
# find longest common suffix starting with "/"
if ("$src$sep$dst" =~ /^.*?(\/.*)$sep.*\1$/) {
my $lcs = $1;
# bbcp batch mode does not use leading slashes like rsync
$lcs =~ s/^\/+//;
if ($spath && $src eq "$spath/$lcs" && $dst eq "$dpath/$lcs") {
print $fh ($shost ? $shost . ":" : ""), "$lcs\n";
$errs{"$spath/$lcs"}->{$ref} = $ref;
$errs{"$dpath/$lcs"}->{$ref} = $ref;
next;
} elsif ($spath) {
# next file has different prefix so process current batch
close $fh;
transport_bbcp_batch($args . ($shost ? " -z" : ""), $tmp,
"$shost$spath", "$dhost$dpath", \%errs, $host);
%errs = ();
open($fh, '>', $tmp);
}
print $fh ($shost ? $shost . ":" : ""), "$lcs\n";
$spath = $src;
# escape lcs in case it contains regex characters
$spath =~ s/\/\Q$lcs\E$//;
$dpath = $dst;
$dpath =~ s/\/\Q$lcs\E$//;
$errs{"$spath/$lcs"}->{$ref} = $ref;
$errs{"$dpath/$lcs"}->{$ref} = $ref;
} else {
# no common suffix implies single file copy with rename
# or symlink dereference
my %errs_tmp;
# use different hash as other files may already be in there
$errs_tmp{$src}->{$ref} = $ref;
$errs_tmp{$dst}->{$ref} = $ref;
transport_bbcp_batch($args . ($shost ? " -z" : ""), "",
"$shost$src", "$dhost$dst", \%errs_tmp, $host);
}
}
close $fh;
if ($spath) {
transport_bbcp_batch($args . ($shost ? " -z" : ""), $tmp,
"$shost$spath", "$dhost$dpath", \%errs, $host);
}
unlink $tmp;
}
##############################
#### transport_bbcp_batch ####
##############################
sub transport_bbcp_batch {
my ($args, $from, $src, $dst, $errs, $host) = @_;
my ($pid, $in, $out, $size);
$from = " --infiles $from -d" if ($from);
eval {
local $SIG{__WARN__} = sub {die};
# escape remote src/dst metacharacters since interpreted by remote shell
my ($esrc, $edst) = ($src, $dst);
$esrc =~ s/([^A-Za-z0-9\-_.:+\/])/\\$1/g if ($esrc =~ /^[^\/]/);
$edst =~ s/([^A-Za-z0-9\-_.:+\/])/\\$1/g if ($edst =~ /^[^\/]/);
my $nstream = $host eq 'localhost' ? $opts{threads} : $opts{streams};
my $extra;
$extra .= " -B " . $opts{buffer} if ($opts{buffer});
$extra .= " -s " . $nstream if ($nstream);
$extra .= " -w " . $opts{window} if ($opts{window});
$extra .= " -Z " . $opts{ports} if ($opts{ports});
# apply opts_bbcp last to override other settings
$extra .= " " . $opts{opts_bbcp};
# use open3 to avoid executing a shell command based on the name
# of a file being copied (which may contain metacharacters, etc.)
# must keep write access to handle warnings/corruption
$pid = IPC::Open3::open3($in, $out, $out,
# make sure quotewords string does not end in space
quotewords('\s+', 0, "bbcp $extra -AfKv -m 0600$args$from"),
$esrc, $edst);
};
if (!$@) {
while (my $line = <$out>) {
$line =~ s/\s+$//;
if ($line =~ /^File (.*) created(?!.*created)/) {
my $file = $1;
foreach my $key (grep(/^\Q$file\E$/, keys(%{$errs}))) {
$_->{text} = 0 foreach (values %{$errs->{$key}});
}
} elsif ($line =~ /^bbcp: [^\/]*(\/.*)/) {
my $file = $1;
foreach my $key (grep(/^\Q$file\E$/, keys(%{$errs}))) {
sftp_error($_, $line) foreach (values %{$errs->{$key}});
}
}
}
}
close $in;
close $out;
waitpid($pid, 0) if ($pid);
foreach my $key (keys %{$errs}) {
foreach (values %{$errs->{$key}}) {
sftp_error($_, "unknown bbcp failure") if (!defined $_->{text});
}
}
}
#########################
#### transport_bbftp ####
#########################
@ -3564,7 +3435,8 @@ sub transport_chattr {
foreach my $cmd (@{$tcmds}) {
my ($op, $src, $dst, $ref) = @{$cmd};
# lfs setstripe (must be done before fallocate)
if ((!$opts{'create-tar'} && $op =~ /^(?:get|mkdir|put)$/ ||
if ((!$opts{'create-tar'} && ($op =~ /^(?:get|put)$/ ||
$op eq 'mkdir' && $ref->{lustre_attrs}) ||
$op eq 'chattr' && $ref->{tar_creat}) &&
$ref->{dstfs} && $ref->{dstfs} =~ /^lustre/ && !$ref->{ln} &&
($opts{stripe} ne '0' || defined $opts{'stripe-size'} ||
@ -3573,23 +3445,28 @@ sub transport_chattr {
my @stripe = (0, 0);
# preserve existing striping when available
@stripe = split(/,/, $ref->{lustre_attrs}) if ($ref->{lustre_attrs});
my @attrs = split(/,/, $ref->{attrs});
# define variables that are allowed in striping expressions
my ($nm, $sz, $sc, $ss) = ($dst, $attrs[7], @stripe);
# base striping on tar size instead of file size during tar creation
$sz = $ref->{tar_creat} if ($ref->{tar_creat});
my @evals = ($opts{stripe}, $opts{'stripe-size'});
push(@evals, $opts{'stripe-pool'})
if ($opts{'stripe-pool'} !~ /^[\w.-]+$/);
# evaluate all striping expressions
foreach my $i (0 .. 2) {
my $eval = $evals[$i];
next if (!$eval);
$eval =~ s/(NM|SZ|SC|SS)/q($).lc($1)/eg;
$stripe[$i] = eval $eval;
# preserve but do not apply striping expressions to directories
if ($op ne 'mkdir') {
my @attrs = split(/,/, $ref->{attrs});
# define variables that are allowed in striping expressions
my ($nm, $sz, $sc, $ss) = ($dst, $attrs[7], @stripe);
# base striping on tar size instead of file size during tar creation
$sz = $ref->{tar_creat} if ($ref->{tar_creat});
my @evals = ($opts{stripe}, $opts{'stripe-size'});
push(@evals, $opts{'stripe-pool'})
if ($opts{'stripe-pool'} !~ /^[\w.-]+$/);
# evaluate all striping expressions
foreach my $i (0 .. 2) {
my $eval = $evals[$i];
next if (!$eval);
$eval =~ s/(NM|SZ|SC|SS)/q($).lc($1)/eg;
$stripe[$i] = eval $eval;
}
# count >= 64k indicates a size per stripe
$stripe[0] = ceil($sz / $stripe[0]) if ($stripe[0] >= 65536);
}
# count >= 64k indicates a size per stripe
$stripe[0] = ceil($sz / $stripe[0]) if ($stripe[0] >= 65536);
# size should never be < 64k (llapi inexplicably returns 2 for dirs)
$stripe[1] = 0 if ($stripe[1] < 65536);
my @args = ("setstripe", escape($dst) . ($op eq 'mkdir' ? "/" : ""),
join(" ", @stripe));
push(@chattrs, \@args);
@ -4722,116 +4599,6 @@ sub transport_fish_return {
return {error => "Invalid protocol return ($msg)"};
}
###########################
#### transport_gridftp ####
###########################
sub transport_gridftp {
my ($host, $tcmds) = @_;
my $ssh_l;
if ($host =~ /@/) {
($ssh_l, $host) = split(/@/, $host);
$ssh_l = " -l " . $ssh_l if ($ssh_l);
}
# make sure gridftp-ssh is set up properly
my $prefix = $host ne 'localhost' ? "sshftp://$host" : "file://";
my $dir = glob("~/.globus");
mkdir $dir if (! -d $dir);
my $file = "$dir/gridftp-ssh";
open(FILE, '>', $file);
# note that sshftp must exist in path (normally resides in .globus/sshftp)
print FILE "#!/bin/sh\n$opts{ssh}$ssh_l \$2 sshftp";
close FILE;
chmod(0700, $file);
my %errs;
my ($fh, $tmp);
foreach my $cmd (@{$tcmds}) {
my ($op, $src, $dst, $ref) = @{$cmd};
($fh, $tmp) = sftp_tmp() if (!$tmp);
if ($op eq 'put') {
$src = "file://" . escape($src);
$dst = $prefix . escape($dst);
$errs{"$src $dst"} = $ref;
} else {
$src = $prefix . escape($src);
$dst = "file://" . escape($dst);
$errs{"$src $dst"} = $ref;
}
if ($ref->{bytes}) {
my @ranges = split(/,/, $ref->{bytes});
foreach my $range (@ranges) {
my ($x1, $x2) = split(/-/, $range);
print $fh "$src $dst $x1,", $x2 - $x1, "\n";
}
} else {
print $fh "$src $dst\n";
}
$ref->{tool} = "gridftp";
}
return if (!$tmp);
close $fh;
my $nstream = $host eq 'localhost' ? $opts{threads} : $opts{streams};
my $extra;
$extra .= " -bs " . $opts{buffer} if ($opts{buffer});
$extra .= " -p " . $nstream if ($nstream);
$extra .= " -tcp-bs " . $opts{window} if ($opts{window});
# encrypt data channel during secure transfers
$extra .= " -dcpriv" if ($opts{secure});
# apply opts_gridftp last to override other settings
$extra .= " " . $opts{opts_gridftp};
if ($opts{ports}) {
#TODO: test that this really works (both on open3 side and globus side)
my $ports = $opts{ports};
$ports =~ s/:/,/;
$ENV{GLOBUS_TCP_RANGE} = $ports;
$ENV{GLOBUS_TCP_PORT_RANGE} = $ports;
$ENV{GLOBUS_TCP_SOURCE_RANGE} = $ports;
$ENV{GLOBUS_UDP_PORT_RANGE} = $ports;
$ENV{GLOBUS_UDP_SOURCE_RANGE} = $ports;
}
if (open(OUT, '-|',
# unbuffer must be used to interleave stdout/stderr
"unbuffer globus-url-copy $extra -c -cd -r -v -f $tmp 2>&1")) {
my ($src, $dst, $text);
while (my $line = <OUT>) {
$line =~ s/\s+$//;
if ($line =~ /^Source:\s*(\S+)/) {
if ($dst && $text && $errs{"$src $dst"}) {
sftp_error($errs{"$src $dst"}, $text);
} elsif ($dst && $errs{"$src $dst"}) {
$errs{"$src $dst"}->{text} = 0;
}
$text = undef;
$src = $1;
} elsif ($line =~ /^Dest:\s*(\S+)/) {
$dst = $1;
} elsif ($line =~ /^\s*(\S+)\s*->\s*(\S+)$/) {
$src .= $1;
$dst .= $2;
} elsif ($line =~ /^\s*(\S+)$/) {
$src .= $1;
$dst .= $1;
} elsif ($line && $line !~ /^error: There was an error with/) {
$text .= $line . " ";
}
}
if ($dst && $text && $errs{"$src $dst"}) {
sftp_error($errs{"$src $dst"}, $text);
} elsif ($dst && $errs{"$src $dst"}) {
$errs{"$src $dst"}->{text} = 0;
}
}
close OUT;
foreach my $key (keys %errs) {
if (!defined $errs{$key}->{text}) {
sftp_error($errs{$key}, "unknown gridftp failure");
}
}
unlink $tmp;
}
#######################
#### transport_mcp ####
#######################
@ -5167,6 +4934,8 @@ sub transport_shift {
push(@scmds, [$x == $x1 ? undef : $x . "-" . $x2, $cmd]);
}
}
# return when no work (mainly when single-threaded)
return if (!scalar(@scmds));
foreach my $o (keys %opts) {
# must kill existing sftp connections or various things can hang