3 # This file is released under the MIT license.
4 # Copyright (c) Famundo LLC, 2007. http://www.famundo.com
5 # Author: Guy Naor - http://devblog.famundo.com
13 opts = OptionParser.new do |opts|
14 opts.banner = "Usage: cp2s3.rb [options] FILE_SPEC"
15 opts.separator "Copy files and directories from the local machine into Amazon's S3. Keep the directory structure intact."
16 opts.separator "Empty directories will be skipped."
18 opts.separator "FILE_SPEC List of files/directories. Accepts wildcards."
19 opts.separator " If given the -g option, interpret FILE_SPEC as a Ruby Dir::Glob style regular expressions."
20 opts.separator " With -g option, '' needed around the pattern to protect it from shell parsing."
22 opts.separator "Required:"
23 opts.on("-k", "--key ACCESS_KEY" , "Your S3 access key. You can also set the environment variable AWS_ACCESS_KEY_ID instead") { |o| @access_key = o }
24 opts.on("-s", "--secret SECRET_KEY" , "Your S3 secret key. You can also set the environment variable AWS_SECRET_ACCESS_KEY instead") { |o| @secret_key = o }
25 opts.on("-b", "--bucket BUCKET_NAME", "The S3 bucket you want the files to go into. Repeat for multiple buckets.") { |o| @buckets << o }
28 opts.separator "Optional:"
30 opts.on("-x", "--remote-prefix PREFIX", "A prefix to add to each file as it's uploaded") { |o| @prefix = o }
31 opts.on("-v", "--verbose", "Print the file names as they are being copied. Repeat for more details") { |o| @verbose += 1 }
32 opts.on("-p", "--public-read", "Set the copied files permission to be public readable.") { |o| @public = true }
33 opts.on("-c", "--compress EXT", "Compress files with given EXT before uploading (ususally css and js),", "setting the HTTP headers for delivery accordingly. Repeat for multiple extensions") { |o| @compress << ".#{o}" }
34 opts.on("-d", "--digest", "Save the sha1 digest of the file, to the S3 metadata. Require sha1sum to be installed") { |o| @save_hash = true }
35 opts.on("-t", "--time", "Save modified time of the file, to the S3 metadata") { |o| @save_time = true }
36 opts.on("-z", "--size", "Save size of the file, to the S3 metadata ") { |o| @save_size = true }
37 opts.on("-r", "--recursive", "If using file system based FILE_SPEC, recurse into sub-directories") { |o| @fs_recurse = true }
38 opts.on("-g", "--glob-ruby", "Interpret FILE_SPEC as a Ruby Dir::Glob. Make sure to put it in ''") { |o| @ruby_glob = true }
39 opts.on("-m", "--modified-only", "Only upload files that were modified must have need uploaded with the digest option.", "Will force digest, size and time modes on") { |o| @modified_only = @save_hash = @save_time = @save_size = true; }
40 opts.on("-y", "--dry-run", "Simulate only - do not upload any file to S3") { |o| @dry_run = true }
41 opts.on("-h", "--help", "Show this instructions") { |o| @help_exit = true }
43 opts.banner = "Copyright(c) Famundo LLC, 2007 (www.famundo.com). Released under the MIT license."
46 @file_spec = opts.parse!(ARGV)
48 @access_key ||= ENV['AWS_ACCESS_KEY_ID']
49 @secret_key ||= ENV['AWS_SECRET_ACCESS_KEY']
52 if @help_exit || !@access_key || !@secret_key || @buckets.empty? || !@file_spec || @file_spec.empty?
57 # Now we start working for real
65 # Log to stderr according to verbosity
66 def log message, for_level
67 puts(message) if @verbose >= for_level
72 log "Connecting to S3", 3
73 AWS::S3::Base.establish_connection!(:access_key_id => @access_key, :secret_access_key => @secret_key)
76 # Copy one file to amazon, compressing and setting metadata as needed
77 def copy_one_file file, fstat
79 content_encoding = nil
84 options[:access] = :public_read if @public
85 options["x-amz-meta-sha1_hash"] = `sha1sum #{file}`.split[0] if @save_hash
86 options["x-amz-meta-mtime"] = fstat.mtime.getutc.to_i if @save_time
87 options["x-amz-meta-size"] = fstat.size if @save_size
89 sent_it = !@modified_only
91 # Check if it was modified
94 if S3Object.find("#{@prefix}#{file}", b).metadata["x-amz-meta-sha1_hash"] == options["x-amz-meta-sha1_hash"]
96 log("Skipping: #{file} in #{b}", 3)
99 rescue AWS::S3::NoSuchKey => ex
100 # This file isn't there yet, so we need to send it
104 # We compress only if we need to compredd and we didn't compress yet
105 if !@compress.empty? && compressed.nil?
106 if @compress.include?(File.extname(file))
108 log "Compressing #{file}", 3
109 strio = StringIO.open('', 'w')
110 gz = Zlib::GzipWriter.new(strio)
111 gz.write(open(file).read)
113 compressed = strio.string
114 options["Content-Encoding"] = 'gzip'
115 log_prefix = '[c] ' if @verbose == 2 # Mark as compressed
117 log_prefix = '[-] ' # So the file names align...
121 log("Sending #{file} to #{b}...", 3)
122 S3Object.store("#{@prefix}#{file}", compressed.nil? ? open(file) : compressed, b, options) unless @dry_run
125 log("#{log_prefix}#{file}", 1) if sent_it
128 # Copy one file/dir from the system, recurssing if needed. Used for non-Ruby style globs
129 def copy_one_file_or_dir name, base_dir
130 return if name[0,1] == '.'
131 file_name = "#{base_dir}#{name}"
132 fstat = File.stat(file_name)
133 copy_one_file(file_name, fstat) if fstat.file? || fstat.symlink?
134 # See if we need to recurse...
135 if @fs_recurse && fstat.directory?
136 my_base = file_name + '/'
137 Dir.foreach(my_base) { |e| copy_one_file_or_dir(e, my_base) }
142 # Glob all the dirs for the files to upload - we expect a ruby like glob format or file system list from the command line
143 @file_spec.each do |spec|
146 Dir.glob(spec) do |file|
147 fstat = File.stat(file)
148 copy_one_file(file, fstat) if fstat.file? || fstat.symlink?
152 copy_one_file_or_dir(spec, '')