[ { "path": ".document", "content": "README.rdoc\nlib/**/*.rb\nbin/*\nfeatures/**/*.feature\nLICENSE\n" }, { "path": ".gitignore", "content": "*.gem\n*.rbc\n/.config\n/coverage/\n/InstalledFiles\n/pkg/\n/spec/reports/\n/spec/examples.txt\n/test/tmp/\n/test/version_tmp/\n/tmp/\n\n## Specific to RubyMotion:\n.dat*\n.repl_history\nbuild/\n\n## Documentation cache and generated files:\n/.yardoc/\n/_yardoc/\n/doc/\n/rdoc/\n\n## Environment normalisation:\n/.bundle/\n/vendor/bundle\n/lib/bundler/man/\n\n# for a library or gem, you might want to ignore these files since the code is\n# intended to run in multiple environments; otherwise, check them in:\nGemfile.lock\n.ruby-version\n.ruby-gemset\n\n# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:\n.rvmrc\n" }, { "path": "Gemfile", "content": "source 'https://rubygems.org'\r\n\r\ngem 'trollop'\r\ngem 's3'\r\ngem 'ruby-atmos-pure'\r\ngem 'right_aws'\r\ngem 'net_dav', :git => 'https://github.com/devrandom/net_dav.git', :require => 'net/dav'\r\ngem 'boxr'\r\n\r\ngem 'netrc'\r\n#gem 'curb', :require => false\r\n" }, { "path": "LICENSE", "content": "Copyright (c) 2009 Scott Chacon\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "# git-media\n\nGitMedia extension allows you to use Git with large media files\nwithout storing the media in Git itself.\n\n## Configuration\n\nSetup the attributes filter settings.\n\n\t(once after install)\n\t$ git config filter.media.clean \"git-media filter-clean\"\n\t$ git config filter.media.smudge \"git-media filter-smudge\"\n\nSetup the `.gitattributes` file to map extensions to the filter.\n\n\t(in repo - once)\n\t$ echo \"*.mov filter=media -crlf\" > .gitattributes\n\nStaging files with those extensions will automatically copy them to the\nmedia buffer area (.git/media) until you run 'git media sync' wherein they\nare uploaded. Checkouts that reference media you don't have yet will try to\nbe automatically downloaded, otherwise they are downloaded when you sync.\n\nNext you need to configure git to tell it where you want to store the large files.\nThere are five options:\n\n1. Storing remotely in Amazon's S3\n2. Storing locally in a filesystem path\n3. Storing remotely via SCP (should work with any SSH server)\n4. Storing remotely in atmos\n5. Storing remotely via WebDav\n\nHere are the relevant sections that should go either in `~/.gitconfig` (for global settings)\nor in `clone/.git/config` (for per-repo settings).\n\n```ini\n[git-media]\n\ttransport = \n\tautodownload = \n\n\t# settings for scp transport\n\tscpuser = \n\tscphost = \n\tscppath = \n\n\t# settings for local transport\n\tlocalpath = \n\n\t# settings for s3 transport\n\ts3bucket = \n\ts3key = \n\ts3secret = \n\n\t# settings for atmos transport\n\tendpoint = \n\tuid = \n\tsecret = \n\ttag = \n\n\t# settings for webdav transport\n\twebdavurl = \n\t# user and password are taken from netrc if omitted\n\twebdavuser = \n\twebdavpassword = \n\n\twebdavverifyserver = \n\twebdavbinarytransfer = \n\n```\n\n\n## Usage\n\n\t(in repo - repeatedly)\n\t$ (hack, stage, commit)\n\t$ git media sync\n\nYou can also check the status of your media files via\n\n\t$ git media status\n\nWhich will show you files that are waiting to be uploaded and how much data\nthat is. If you want to upload & delete the local cache of media files, run:\n\n\t$ git media clear\n\nIf you want to replace file in git-media with changed version (for example, video file has been edited),\nyou need to explicitly tell git that some media files has changed:\n\n $ git update-index --really-refresh\n\n\n## Config Settings\n\nIf autodownload is set to true, required files will automatically be\ndownloaded when checking out or pulling. Default is false\n\n\t$ git config --global media.autodownload true\n\n\n## Installing\n\n $ git clone git@github.com:alebedev/git-media.git\n $ cd git-media\n $ sudo gem install bundler\n $ bundle install\n $ gem build git-media.gemspec\n $ sudo gem install git-media-*.gem\n\n## Notes for Windows\n\nIt is important to switch off git smart newline character support for media files.\nUse `-crlf` switch in `.gitattributes` (for example `*.mov filter=media -crlf`) or config option `core.autocrlf = false`.\n\nIf installing on windows, you might run into a problem verifying certificates\nfor S3 or something. If that happens, see the [instructions in this Gist for how\nto update your RubyGems to the proper certificates](https://gist.github.com/luislavena/f064211759ee0f806c88).\n\n## Copyright\n\nCopyright (c) 2009 Scott Chacon. See LICENSE for details.\n" }, { "path": "Rakefile", "content": "require 'rubygems'\nrequire 'rake'\n\nbegin\n require 'jeweler'\n Jeweler::Tasks.new do |gem|\n gem.name = \"git-media\"\n gem.summary = %Q{git-media}\n gem.email = \"schacon@gmail.com\"\n gem.homepage = \"http://github.com/schacon/git-media\"\n gem.authors = [\"Scott Chacon\"]\n # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings\n end\n\nrescue LoadError\n puts \"Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler\"\nend\n\nrequire 'spec/rake/spectask'\nSpec::Rake::SpecTask.new(:spec) do |spec|\n spec.libs << 'lib' << 'spec'\n spec.spec_files = FileList['spec/**/*_spec.rb']\nend\n\nSpec::Rake::SpecTask.new(:rcov) do |spec|\n spec.libs << 'lib' << 'spec'\n spec.pattern = 'spec/**/*_spec.rb'\n spec.rcov = true\nend\n\n\ntask :default => :spec\n\nrequire 'rake/rdoctask'\nRake::RDocTask.new do |rdoc|\n if File.exist?('VERSION.yml')\n config = YAML.load(File.read('VERSION.yml'))\n version = \"#{config[:major]}.#{config[:minor]}.#{config[:patch]}\"\n else\n version = \"\"\n end\n\n rdoc.rdoc_dir = 'rdoc'\n rdoc.title = \"git-media #{version}\"\n rdoc.rdoc_files.include('README*')\n rdoc.rdoc_files.include('lib/**/*.rb')\nend\n\n" }, { "path": "TODO", "content": "== Tools\n\n\t* tool to clean large files out of existing repo (filter-branch?)\n\t - can also just re-do the last commit with a new filter\n\n\t* git media add (file) - adds it to the .gitattributes file\n\n== Transports\n\n\t* Local\n\t* Amazon S3\n\t* SCP\n\t* SFTP\n\t* FTP\n" }, { "path": "VERSION", "content": "0.1.4\n" }, { "path": "bin/git-media", "content": "#!/usr/bin/env ruby\nrequire 'optparse'\n\n$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')\nrequire 'git-media'\n\nGitMedia::Application.run!\n" }, { "path": "git-media.gemspec", "content": "# Generated by jeweler\r\n# DO NOT EDIT THIS FILE DIRECTLY\r\n# Instead, edit Jeweler::Tasks in rakefile, and run 'rake gemspec'\r\n# -*- encoding: utf-8 -*-\r\n\r\nGem::Specification.new do |s|\r\n s.name = \"git-media\"\r\n s.version = \"0.1.5\"\r\n\r\n s.required_rubygems_version = Gem::Requirement.new(\">= 0\") if s.respond_to? :required_rubygems_version=\r\n s.authors = [\"Scott Chacon\", \"Alexander Lebedev\"]\r\n s.date = \"2014-10-20\"\r\n s.email = \"alexander.lebedev@gmail.com\"\r\n s.executables = [\"git-media\"]\r\n s.extra_rdoc_files = [\r\n \"LICENSE\",\r\n \"README.md\",\r\n \"TODO\"\r\n ]\r\n s.files = [\r\n \".document\",\r\n \"Gemfile\",\r\n \"Gemfile.lock\",\r\n \"LICENSE\",\r\n \"README.md\",\r\n \"Rakefile\",\r\n \"TODO\",\r\n \"VERSION\",\r\n \"bin/git-media\",\r\n \"git-media.gemspec\",\r\n \"lib/git-media.rb\",\r\n \"lib/git-media/clear.rb\",\r\n \"lib/git-media/filter-clean.rb\",\r\n \"lib/git-media/filter-smudge.rb\",\r\n \"lib/git-media/filter-branch.rb\",\r\n \"lib/git-media/status.rb\",\r\n \"lib/git-media/sync.rb\",\r\n \"lib/git-media/transport.rb\",\r\n \"lib/git-media/transport/atmos_client.rb\",\r\n \"lib/git-media/transport/box.rb\",\r\n \"lib/git-media/transport/local.rb\",\r\n \"lib/git-media/transport/s3.rb\",\r\n \"lib/git-media/transport/scp.rb\",\r\n \"lib/git-media/transport/webdav.rb\",\r\n \"spec/media_spec.rb\",\r\n \"spec/spec_helper.rb\"\r\n ]\r\n s.homepage = \"http://github.com/alebedev/git-media\"\r\n s.require_paths = [\"lib\"]\r\n s.rubygems_version = \"1.8.28\"\r\n s.summary = \"git-media\"\r\n\r\n if s.respond_to? :specification_version then\r\n s.specification_version = 3\r\n\r\n if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then\r\n s.add_runtime_dependency(%q, [\">= 0\"])\r\n else\r\n s.add_dependency(%q, [\">= 0\"])\r\n end\r\n else\r\n s.add_dependency(%q, [\">= 0\"])\r\n end\r\nend\r\n\r\n" }, { "path": "lib/git-media/clear.rb", "content": "require 'git-media/status'\n\nmodule GitMedia\n module Clear\n\n def self.run!\n @push = GitMedia.get_push_transport\n self.clear_local_cache\n end\n \n def self.clear_local_cache\n # find files in media buffer and delete all pushed files\n all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }\n unpushed_files = @push.get_unpushed(all_cache)\n pushed_files = all_cache - unpushed_files\n pushed_files.each do |sha|\n puts \"Removing \" + sha[0, 8]\n File.unlink(File.join(GitMedia.get_media_buffer, sha))\n end\n end\n \n end\nend" }, { "path": "lib/git-media/filter-branch.rb", "content": "require 'set'\nrequire 'git-media/filter-clean'\nrequire 'fileutils'\n\ninclude Process\n\nmodule GitMedia\n module FilterBranch\n\n def self.get_temp_buffer\n @@git_dir ||= `git rev-parse --git-dir`.chomp\n temp_buffer = File.join(@@git_dir, 'media/filter-branch')\n FileUtils.mkdir_p(temp_buffer) if !File.exist?(temp_buffer)\n return temp_buffer\n end\n\n def self.clean!\n tmp_buffer = get_temp_buffer\n FileUtils.rm_r (tmp_buffer)\n FileUtils.rmdir (tmp_buffer)\n end\n\n def self.run!\n # Rewriting of history\n # Inspired by how git-fat does it\n\n inputfiles = ARGF.read.split(\"\\n\").map { |s| s.downcase }.to_set\n all_files = `git ls-files -s`.split(\"\\n\")\n filecount = all_files.length.to_s\n\n # determine and initialize our media buffer directory\n media_buffer = GitMedia.get_media_buffer\n\n tmp_buffer = get_temp_buffer\n\n STDOUT.write (\" \")\n\n index = 0\n prevLength = 0\n\n\n fileLists = [[],[],[],[]]\n all_files.each_with_index do |f, i|\n fileLists[i % fileLists.length].push (f)\n end\n\n update_index_reader, update_index_writer = IO.pipe\n update_index_pid = spawn(\"git update-index --index-info\", :in=>update_index_reader)\n update_index_reader.close\n mutex = Mutex.new\n \n threads = []\n fileLists.each_with_index do |files, thread_index|\n\n fls = files\n thread = Thread.new do\n\n fls.each do |line|\n index += 1\n\n head, filepath = line.split(\"\\t\")\n filepath.strip!\n\n if not inputfiles.include? (filepath.downcase)\n next\n end\n\n mode, blob, stagenumber = head.split()\n\n # Skip symlinks\n if mode == \"120000\"\n next\n end\n\n # 1 Find cached git-hash of the media stub\n # 1.2 If not found, calculate it\n # 1.3 store object in media buffer\n # 1.4 save the hash in the cache\n # 2 Replace object with git-hash of the stub\n \n #1\n hash_file_path = File.join(tmp_buffer, blob)\n\n hash_of_stub = nil\n if File.exists?(hash_file_path)\n\n File.open(hash_file_path, \"rb\") do |f|\n hash_of_stub = f.read.strip()\n end\n else\n\n # Only show progress output for thread 0 because otherwise the thread\n # output might get messed up by multiple threads writing at the same time\n if thread_index == 0\n # Erase previous output text\n # \\b is backspace\n prevLength.times {\n STDOUT.write(\"\\b\")\n STDOUT.write(\" \")\n STDOUT.write(\"\\b\")\n }\n\n line = \"Filtering \" + index.to_s + \" of \" + filecount + \" : \" + filepath\n prevLength = line.length\n STDOUT.write (line)\n end\n\n # pipes roughly equivalent to\n # cat-file | clean | hash | update-index\n # 1.2, 1.3\n\n gitcat_reader, gitcat_writer= IO.pipe\n gitcat_pid = spawn(\"git cat-file blob \" + blob, :out=>gitcat_writer, :close_others=>true)\n\n # We are not using it, so close it\n gitcat_writer.close\n\n githash_reader, githash_writer= IO.pipe\n githash_output_reader, githash_output_writer= IO.pipe\n githash_pid = spawn(\"git hash-object -w --stdin\", :in=>githash_reader, :out=>githash_output_writer)\n githash_output_writer.close\n githash_reader.close\n\n GitMedia::FilterClean.run!(gitcat_reader, githash_writer, false)\n \n gitcat_reader.close\n githash_writer.close\n\n hash_of_stub = githash_output_reader.read().strip()\n\n # 1.4\n cache = File.new(hash_file_path, File::CREAT|File::RDWR|File::BINARY)\n cache.write(hash_of_stub)\n cache.close\n\n wait (githash_pid)\n wait (gitcat_pid)\n end\n\n # 2\n update = mode + \" \" + hash_of_stub + \" \" + stagenumber + \"\\t\" + filepath + \"\\n\"\n\n # Synchronize with a mutex to avoid multiple\n # threads writing to the pipe at the same time\n mutex.synchronize do\n update_index_writer.write(update)\n end\n end\n end\n\n threads.push(thread)\n end\n\n threads.each do |thread|\n thread.join\n end\n\n update_index_writer.close()\n wait(update_index_pid)\n end\n end\nend" }, { "path": "lib/git-media/filter-clean.rb", "content": "require 'digest/sha1'\nrequire 'fileutils'\nrequire 'tempfile'\n\nmodule GitMedia\n module FilterClean\n\n def self.run!(input=STDIN, output=STDOUT, info_output=true)\n \n input.binmode\n # Read first 42 bytes\n # If the file is only 41 bytes long (as in the case of a stub)\n # it will only return a string with a length of 41\n data = input.read(42)\n output.binmode\n\n if data != nil && data.length == 41 && data.match(/^[0-9a-fA-F]+\\n$/)\n \n # Exactly 41 bytes long and matches the hex string regex\n # This is most likely a stub\n # TODO: Maybe add some additional marker in the files like\n # \"[hex string]:git-media\"\n # to really be able to say that a file is a stub\n\n output.write (data)\n\n if info_output\n STDERR.puts(\"Skipping unexpanded stub : \" + data[0, 8])\n end\n\n else\n\n # determine and initialize our media buffer directory\n media_buffer = GitMedia.get_media_buffer\n\n hashfunc = Digest::SHA1.new\n start = Time.now\n\n # read in buffered chunks of the data\n # calculating the SHA and copying to a tempfile\n tempfile = Tempfile.new('media', :binmode => true)\n\n # Write the first 42 bytes\n if data != nil\n hashfunc.update(data)\n tempfile.write(data)\n end\n\n while data = input.read(4096)\n hashfunc.update(data)\n tempfile.write(data)\n end\n tempfile.close\n\n # calculate and print the SHA of the data\n output.print hx = hashfunc.hexdigest \n output.write(\"\\n\")\n\n # move the tempfile to our media buffer area\n media_file = File.join(media_buffer, hx)\n FileUtils.mv(tempfile.path, media_file)\n\n elapsed = Time.now - start\n\n if info_output\n STDERR.puts('Saving media : ' + hx + ' : ' + elapsed.to_s)\n end\n end\n end\n\n end\nend" }, { "path": "lib/git-media/filter-smudge.rb", "content": "module GitMedia\n module FilterSmudge\n\n def self.print_stream(stream)\n # create a binary stream to write to stdout\n # this avoids messing up line endings on windows\n outstream = IO.try_convert(STDOUT)\n outstream.binmode\n\n while data = stream.read(4096) do\n print data\n end\n end\n\n def self.run!\n media_buffer = GitMedia.get_media_buffer\n \n # read checksum size\n STDIN.binmode\n STDOUT.binmode\n orig = STDIN.readline(64)\n sha = orig.strip # read no more than 64 bytes\n if STDIN.eof? && sha.length == 40 && sha.match(/^[0-9a-fA-F]+$/) != nil\n # this is a media file\n media_file = File.join(media_buffer, sha.chomp)\n if File.exists?(media_file)\n STDERR.puts('Recovering media : ' + sha)\n File.open(media_file, 'rb') do |f|\n print_stream(f)\n end\n else\n # Read key from config\n auto_download = `git config git-media.autodownload`.chomp.downcase == \"true\"\n\n if auto_download\n\n pull = GitMedia.get_pull_transport\n\n cache_file = GitMedia.media_path(sha)\n if !File.exist?(cache_file)\n STDERR.puts (\"Downloading : \" + sha[0,8])\n # Download the file from backend storage\n # We have no idea what the final file will be (therefore nil)\n pull.pull(nil, sha)\n end\n\n STDERR.puts (\"Expanding : \" + sha[0,8])\n\n if File.exist?(cache_file)\n File.open(media_file, 'rb') do |f|\n print_stream(f)\n end\n else\n STDERR.puts (\"Could not get media, saving placeholder : \" + sha)\n puts orig\n end\n\n else\n STDERR.puts('Media missing, saving placeholder : ' + sha)\n # Print orig and not sha to preserve eventual newlines at end of file\n # To avoid git thinking the file has changed\n puts orig\n end\n end\n else\n # if it is not a 40 character long hash, just output\n STDERR.puts('Unknown git-media file format')\n print orig\n print_stream(STDIN)\n end\n end\n\n end\nend\n" }, { "path": "lib/git-media/status.rb", "content": "require 'pp'\nEncoding.default_external = Encoding::UTF_8\n\nmodule GitMedia\n module Status\n\n def self.run!(opts)\n @push = GitMedia.get_push_transport\n r = self.find_references\n self.print_references(r, opts[:short])\n r = self.local_cache_status\n self.print_cache_status(r, opts[:short])\n end\n\n # find tree entries that are likely media references\n def self.find_references\n references = {:to_expand => [], :expanded => [], :deleted => []}\n files = `git ls-tree -l -r HEAD | tr \"\\\\000\" \\\\\\\\n`.split(\"\\n\")\n files = files.map { |f| s = f.split(\"\\t\"); [s[0].split(' ').last, s[1]] }\n files = files.select { |f| f[0] == '41' } # it's the right size\n files.each do |tree_size, fname|\n if File.exists?(fname)\n size = File.size(fname)\n\n # Windows newlines can offset file size by 1\n if size == tree_size.to_i or size == tree_size.to_i + 1\n # TODO: read in the data and verify that it's a sha + newline\n fname = fname.tr(\"\\\\\",\"\") #remove backslash\n sha = File.read(fname).strip\n if sha.length == 40 && sha =~ /^[0-9a-f]+$/\n references[:to_expand] << [fname, sha]\n end\n else\n references[:expanded] << fname\n end\n else\n # file was deleted\n references[:deleted] << fname\n end\n end\n references\n end\n\n def self.print_references(refs, short=false)\n\n if refs[:to_expand].size > 0\n puts \"== Unexpanded Media ==\"\n if short\n puts \"Count: \" + refs[:to_expand].size.to_s\n else\n refs[:to_expand].each do |file, sha|\n puts \" \" + sha[0, 8] + \" \" + file\n end\n puts\n end\n end\n if refs[:expanded].size > 0\n puts \"== Expanded Media ==\"\n if short\n puts \"Count: \" + refs[:expanded].size.to_s\n else\n refs[:expanded].each do |file|\n size = File.size(file)\n puts \" \" + \"(#{self.to_human(size)})\".ljust(8) + \" #{file}\"\n end\n puts\n end\n end\n if refs[:deleted].size > 0\n puts \"== Deleted Media ==\"\n if short\n puts \"Count: \" + refs[:deleted].size.to_s\n else\n refs[:deleted].each do |file|\n puts \" \" + \" #{file}\"\n end\n puts\n end\n end\n end\n\n def self.print_cache_status(refs, short)\n if refs[:unpushed].size > 0\n puts \"== Unpushed Media ==\"\n if short\n puts \"Count: \" + refs[:unpushed].size.to_s\n else\n refs[:unpushed].each do |sha|\n cache_file = GitMedia.media_path(sha)\n size = File.size(cache_file)\n puts \" \" + \"(#{self.to_human(size)})\".ljust(8) + ' ' + sha[0, 8]\n end\n puts\n end\n end\n if refs[:pushed].size > 0\n puts \"== Already Pushed Media ==\"\n if short\n puts \"Count: \" + refs[:pushed].size.to_s\n else\n refs[:pushed].each do |sha|\n cache_file = GitMedia.media_path(sha)\n size = File.size(cache_file)\n puts \" \" + \"(#{self.to_human(size)})\".ljust(8) + ' ' + sha[0, 8]\n end\n puts\n end\n end\n end\n\n def self.local_cache_status\n # find files in media buffer and upload them\n references = {:unpushed => [], :pushed => []}\n all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }\n unpushed_files = @push.get_unpushed(all_cache) || []\n references[:unpushed] = unpushed_files\n references[:pushed] = all_cache - unpushed_files rescue []\n references\n end\n\n\n def self.to_human(size)\n if size < 1024\n return size.to_s + 'b'\n elsif size < 1048576\n return (size / 1024).to_s + 'k'\n else\n return (size / 1048576).to_s + 'm'\n end\n end\n\n end\nend\n" }, { "path": "lib/git-media/sync.rb", "content": "# find files that are placeholders (41 char) and download them\n# upload files in media buffer that are not in offsite bin\nrequire 'git-media/status'\n\nmodule GitMedia\n module Sync\n\n def self.run!\n @push = GitMedia.get_push_transport\n @pull = GitMedia.get_pull_transport\n\n self.expand_references\n self.update_index\n self.upload_local_cache\n end\n\n def self.expand_references\n status = GitMedia::Status.find_references\n status[:to_expand].each_with_index do |tuple, index|\n file = tuple[0]\n sha = tuple[1]\n cache_file = GitMedia.media_path(sha)\n if !File.exist?(cache_file)\n puts \"Downloading \" + sha[0,8] + \" : \" + file\n @pull.pull(file, sha)\n end\n\n puts \"Expanding \" + (index+1).to_s + \" of \" + status[:to_expand].length.to_s + \" : \" + sha[0,8] + \" : \" + file\n\n if File.exist?(cache_file)\n FileUtils.cp(cache_file, file)\n else\n puts 'Could not get media from storage'\n end\n end\n end\n\n def self.update_index\n refs = GitMedia::Status.find_references\n\n # Split references up into lists of at most 500\n # because most OSs have limits on the size of the argument list\n # TODO: Could probably use the --stdin flag on git update-index to be\n # able to update it in a single call\n refLists = refs[:expanded].each_slice(500).to_a\n\n refLists.each {\n |refList|\n\n refList = refList.map { |v| \"\\\"\" + v + \"\\\"\"}\n\n `git update-index --assume-unchanged -- #{refList.join(' ')}`\n }\n \n puts \"Updated git index\"\n end\n\n def self.upload_local_cache\n # find files in media buffer and upload them\n all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }\n unpushed_files = @push.get_unpushed(all_cache)\n unpushed_files.each_with_index do |sha, index|\n puts \"Uploading \" + sha[0, 8] + \" \" + (index+1).to_s + \" of \" + unpushed_files.length.to_s\n @push.push(sha)\n end\n # TODO: if --clean, remove them\n end\n\n end\nend" }, { "path": "lib/git-media/transport/atmos_client.rb", "content": "require 'git-media/transport'\nrequire 'ruby-atmos-pure'\nrequire 'atmos'\n\n\n# git-media.transport atmos\n# git-media.endpoint\n# git-media.uid\n# git-media.secret\n# git-media.tag (optional)\n\nmodule GitMedia\n module Transport\n class AtmosClient < Base\n\n def initialize(endpoint, uid, secret, tag)\n atmos_options = {\n :url => endpoint,\n :uid => uid,\n :secret => secret\n }\n @tag = tag\n @atmos_client = Atmos::Store.new(atmos_options)\n end\n\n def read?\n reachable?\n end\n\n def get_file(sha, to_file)\n dst_file = File.new(to_file, File::CREAT|File::RDWR|File::BINARY)\n @atmos_client.get(:namespace => sha).data_as_stream do |chunck|\n dst_file.write(chunck)\n end\n end\n\n def write\n reachable?\n end\n\n def put_file(sha, from_file)\n src_file = File.open(from_file,\"rb\")\n obj_conf = {:data => src_file, :length => File.size(from_file), :namespace => sha}\n obj_conf[:listable_metadata] = {@tag => true} if @tag\n @atmos_client.create(obj_conf)\n end\n\n def get_unpushed(files)\n unpushed = []\n files.each do |file|\n begin\n @atmos_client.get(:namespace => file)\n rescue Atmos::Exceptions::AtmosException\n unpushed << file\n end\n end\n unpushed\n end\n\n private\n # dummy function to test connectivity to atmos\n def reachable?\n @atmos_client.server_version\n true\n rescue\n false\n end\n\n end\n end\nend\n\n\n" }, { "path": "lib/git-media/transport/box.rb", "content": "require 'git-media/transport'\nrequire 'boxr'\nrequire 'shellwords'\n\n# git-media.transport box\n# git-media.boxclientid\n# git-media.boxclientsecret\n# git-media.boxredirecturi\n# git-media.boxfolderid\n# git-media.boxaccesstoken\n# git-media.boxrefreshtoken\n\nmodule GitMedia\n module Transport\n class Box < Base\n\n def initialize(client_id, client_secret, redirect_uri, folder_id, access_token, refresh_token)\n if access_token == \"\" || refresh_token == \"\"\n uri = Boxr::oauth_url(redirect_uri, box_client_id: client_id)\n print \"(1) Paste following URL to your browser, and get your access code:\\n\\n#{uri}\\n\\n(2) Enter your access code: \"\n code = STDIN.gets.chomp\n token = Boxr::get_tokens(code, box_client_id: client_id, box_client_secret: client_secret)\n\n access_token = token.access_token\n refresh_token = token.refresh_token\n\n `git config git-media.boxaccesstoken #{access_token.shellescape}`\n `git config git-media.boxrefreshtoken #{refresh_token.shellescape}`\n end\n\n token_refresh_callback = lambda {|at, rt, id|\n `git config git-media.boxaccesstoken #{at.shellescape}`\n `git config git-media.boxrefreshtoken #{rt.shellescape}`\n }\n @box = Boxr::Client.new(access_token, refresh_token: refresh_token, box_client_id: client_id, box_client_secret: client_secret, &token_refresh_callback)\n @folder = @box.folder_from_id(folder_id)\n end\n\n def read?\n true\n end\n\n def get_file(sha, to_file)\n files = get_files(true)\n if files.has_key?(sha) == false\n files = get_files()\n end\n\n file_id = files[sha]\n if file_id == nil\n STDERR.puts(\"Storage backend (box) did not contain file : \"+sha+\", have you run 'git media sync' from all repos?\")\n return false\n end\n\n file = @box.file_from_id(file_id)\n content = @box.download_file(file)\n File::open(to_file, \"wb\") do |f|\n f.write(content)\n end\n end\n\n def write?\n true\n end\n\n def put_file(sha, from_file)\n @box.upload_file(from_file, @folder)\n end\n\n def get_unpushed(files)\n remote_files = get_files()\n\n files.select do |f|\n !remote_files.has_key?(f)\n end\n end\n\n def get_files(use_cache = false)\n media_buffer = GitMedia.get_media_buffer\n cache_file = File.join(media_buffer, \"cache\")\n files = {}\n\n if use_cache\n File::exists?(cache_file) && File::open(cache_file) do |f|\n f.each do |s|\n r = s.strip.split(\",\")\n files[r[0]] = r[1]\n end\n end\n\n return files if files.length > 0\n end\n\n offset = 0\n limit = 100\n while (items = @box.folder_items(@folder, fields: [:id, :name], offset: offset, limit: limit)).length > 0\n items.each do |f|\n files[f[:name]] = f[:id]\n end\n offset = offset + limit\n end\n\n # cache update\n f = File::open(cache_file, \"w\")\n files.each do |name, id|\n f.puts \"#{name},#{id}\"\n end\n f.close\n\n return files\n end\n end\n end\nend\n" }, { "path": "lib/git-media/transport/local.rb", "content": "require 'git-media/transport'\n\n# move large media to local bin\n\n# git-media.transport local\n# git-media.localpath /opt/media\n\nmodule GitMedia\n module Transport\n class Local < Base\n\n def initialize(path)\n @path = path\n end\n\n def read?\n File.exist?(@path)\n end\n\n def get_file(sha, to_file)\n from_file = File.join(@path, sha)\n if File.exists?(from_file)\n FileUtils.cp(from_file, to_file)\n return true\n end\n return false\n end\n\n def write?\n File.exist?(@path)\n end\n\n def put_file(sha, from_file)\n to_file = File.join(@path, sha)\n if File.exists?(from_file)\n FileUtils.cp(from_file, to_file)\n return true\n end\n return false\n end\n \n def get_unpushed(files)\n files.select do |f|\n !File.exist?(File.join(@path, f))\n end\n end\n \n end\n end\nend\n" }, { "path": "lib/git-media/transport/s3.rb", "content": "require 'git-media/transport'\nrequire 's3'\nrequire 'right_aws'\n\n# git-media.transport s3\n# git-media.s3bucket\n# git-media.s3key\n# git-media.s3secret\n\nmodule GitMedia\n module Transport\n class S3 < Base\n\n def initialize(bucket, access_key_id = nil, secret_access_key = nil) \n @s3 = RightAws::S3Interface.new(access_key_id, secret_access_key,\n {:multi_thread => true, :logger => Logger.new(File.expand_path('~/.git-media.s3.log'))})\n \n @bucket = bucket\n\n begin\n @buckets = @s3.list_all_my_buckets.map { |a| a[:name] }\n rescue RightAws::AwsError\n # Need to use STDERR because this might be called inside a filter\n STDERR.puts (\"Failed to connect to storage backend (S3)\")\n raise\n end\n\n if !@buckets.include?(bucket)\n # Need to use STDERR because this might be called inside a filter\n STDERR.puts (\"Creating New Bucket\")\n if @s3.create_bucket(bucket)\n @buckets << bucket\n end\n end\n end\n\n def read?\n @buckets.size > 0\n end\n\n def get_file(sha, to_file)\n to = File.new(to_file, File::CREAT|File::RDWR|File::BINARY)\n begin\n @s3.get(@bucket, sha) do |chunk|\n to.write(chunk)\n end\n to.close\n return true\n rescue RightAws::AwsError => e\n # Delete the file to make sure it is not expanded\n to.close\n File.delete(to_file)\n\n # Ugly, but AwsError does not seem to give me much choice\n if e.message.include?('NoSuchKey')\n STDERR.puts(\"Storage backend (S3) did not contain file : \"+sha+\", have you run 'git media sync' from all repos?\")\n return false\n else\n # Need to use STDERR because this might be called inside a filter\n STDERR.puts (\"Downloading file from S3 failed with error:\\n\" + e.message)\n return false\n end\n end\n end\n\n def write?\n @buckets.size > 0\n end\n\n def put_file(sha, from_file)\n @s3.put(@bucket, sha, File.open(from_file,\"rb\"))\n end\n\n def get_unpushed(files)\n # Using a set instead of a list improves performance a lot\n # since it reduces the complexity from O(n^2) to O(n)\n keys = Set.new()\n\n # Apparently the list_bucket method only returns the first 1000 elements\n # This method however will continue to give back results until all elements\n # have been listed\n @s3.incrementally_list_bucket(@bucket) { |contents| \n contents[:contents].each { |element|\n keys.add (element[:key])\n }\n }\n\n files.select do |f|\n !keys.include?(f)\n end\n end\n end\n end\nend" }, { "path": "lib/git-media/transport/scp.rb", "content": "require 'git-media/transport'\n\n# move large media to remote server via SCP\n\n# git-media.transport scp\n# git-media.scpuser someuser\n# git-media.scphost remoteserver.com\n# git-media.scppath /opt/media\n\nmodule GitMedia\n module Transport\n class Scp < Base\n\n def initialize(user, host, path, port)\n @user = user\n @host = host\n @path = path\n unless port === \"\"\n @sshport = \"-p#{port}\"\n end\n unless port === \"\"\n @scpport = \"-P#{port}\"\n end\n end\n\n def exist?(file)\n if `ssh #{@user}@#{@host} #{@sshport} [ -f \"#{file}\" ] && echo 1 || echo 0`.chomp == \"1\"\n puts file + \" exists\"\n return true\n else\n puts file + \" doesn't exists\"\n return false\n end\n end\n\n def read?\n return true\n end\n\n def get_file(sha, to_file)\n from_file = @user+\"@\"+@host+\":\"+File.join(@path, sha)\n `scp #{@scpport} \"#{from_file}\" \"#{to_file}\"`\n if $? == 0\n puts sha+\" downloaded\"\n return true\n end\n puts sha+\" download fail\"\n return false\n end\n\n def write?\n return true\n end\n\n def put_file(sha, from_file)\n to_file = @user+\"@\"+@host+\":\"+File.join(@path, sha)\n `scp #{@scpport} \"#{from_file}\" \"#{to_file}\"`\n if $? == 0\n puts sha+\" uploaded\"\n return true\n end\n puts sha+\" upload fail\"\n return false\n end\n \n def get_unpushed(files)\n files.select do |f|\n !self.exist?(File.join(@path, f))\n end\n end\n \n end\n end\nend\n" }, { "path": "lib/git-media/transport/webdav.rb", "content": "require 'git-media/transport'\n\nrequire 'uri'\nrequire 'net/dav'\n\n\nmodule GitMedia\n module Transport\n class WebDav < Base\n def initialize(url, user, password, verify_server=true, binary_transfer=false)\n @uri = URI(url)\n # Faster binary transport requires curb gem\n @dav = Net::DAV.new(url, :curl => (binary_transfer))\n @dav.verify_server = verify_server\n @dav.credentials(user, password)\n print 'checking connection... '\n @has_connection = @dav.exists?('.')\n puts (if @has_connection then 'ok' else 'failed' end)\n end\n\n def read?\n @has_connection\n end\n\n def write?\n @has_connection\n end\n\n def get_path(path)\n @uri.merge(path).path\n end\n\n def exists?(file)\n @dav.exists?(get_path(file))\n end\n\n def get_file(sha, to_file)\n to = File.new(to_file, File::CREAT|File::RDWR|File::BINARY)\n begin\n @dav.get(get_path(sha)) do |chunk|\n to.write(chunk)\n end\n true\n ensure\n to.close\n end\n end\n\n def put_file(sha, from_file)\n @dav.put(get_path(sha), File.open(from_file, \"rb\"), File.size(from_file))\n end\n\n def get_unpushed(files)\n files.select do |f|\n !self.exists?(f)\n end\n end\n\n end\n end\nend\n" }, { "path": "lib/git-media/transport.rb", "content": "module GitMedia\n module Transport\n class Base\n\n def pull(final_file, sha)\n to_file = GitMedia.media_path(sha)\n get_file(sha, to_file)\n end\n\n def push(sha)\n from_file = GitMedia.media_path(sha)\n put_file(sha, from_file)\n end\n\n ## OVERWRITE ##\n \n def read?\n false\n end\n\n def write?\n false\n end\n\n def get_file(sha, to_file)\n false\n end\n\n def put_file(sha, to_file)\n false\n end\n \n def get_unpushed(files)\n files\n end\n \n end\n end\nend" }, { "path": "lib/git-media.rb", "content": "require 'rubygems'\nrequire 'bundler/setup'\n\nrequire 'trollop'\nrequire 'fileutils'\n#\n\nmodule GitMedia\n\n def self.get_media_buffer\n @@git_dir ||= `git rev-parse --git-dir`.chomp\n media_buffer = File.join(@@git_dir, 'media/objects')\n FileUtils.mkdir_p(media_buffer) if !File.exist?(media_buffer)\n return media_buffer\n end\n\n def self.media_path(sha)\n buf = self.get_media_buffer\n File.join(buf, sha)\n end\n\n # TODO: select the proper transports based on settings\n def self.get_push_transport\n self.get_transport\n end\n\n def self.get_credentials_from_netrc(url)\n require 'uri'\n require 'netrc'\n\n uri = URI(url)\n hostname = uri.host\n unless hostname\n raise \"Cannot identify hostname within git-media.webdavurl value\"\n end\n netrc = Netrc.read\n netrc[hostname]\n end\n\n def self.get_transport\n transport = `git config git-media.transport`.chomp\n case transport\n when \"\"\n raise \"git-media.transport not set\"\n\n when \"scp\"\n require 'git-media/transport/scp'\n\n user = `git config git-media.scpuser`.chomp\n host = `git config git-media.scphost`.chomp\n path = `git config git-media.scppath`.chomp\n port = `git config git-media.scpport`.chomp\n if user === \"\"\n raise \"git-media.scpuser not set for scp transport\"\n end\n if host === \"\"\n raise \"git-media.scphost not set for scp transport\"\n end\n if path === \"\"\n raise \"git-media.scppath not set for scp transport\"\n end\n GitMedia::Transport::Scp.new(user, host, path, port)\n\n when \"local\"\n require 'git-media/transport/local'\n\n path = `git config git-media.localpath`.chomp\n if path === \"\"\n raise \"git-media.localpath not set for local transport\"\n end\n GitMedia::Transport::Local.new(path)\n\n when \"s3\"\n require 'git-media/transport/s3'\n\n bucket = `git config git-media.s3bucket`.chomp\n key = `git config git-media.s3key`.chomp\n secret = `git config git-media.s3secret`.chomp\n if bucket === \"\"\n raise \"git-media.s3bucket not set for s3 transport\"\n end\n if key === \"\"\n raise \"git-media.s3key not set for s3 transport\"\n end\n if secret === \"\"\n raise \"git-media.s3secret not set for s3 transport\"\n end\n GitMedia::Transport::S3.new(bucket, key, secret)\n\n when \"atmos\"\n require 'git-media/transport/atmos_client'\n\n endpoint = `git config git-media.endpoint`.chomp\n uid = `git config git-media.uid`.chomp\n secret = `git config git-media.secret`.chomp\n tag = `git config git-media.tag`.chomp\n\n if endpoint == \"\"\n raise \"git-media.endpoint not set for atmos transport\"\n end\n\n if uid == \"\"\n raise \"git-media.uid not set for atmos transport\"\n end\n\n if secret == \"\"\n raise \"git-media.secret not set for atmos transport\"\n end\n GitMedia::Transport::AtmosClient.new(endpoint, uid, secret, tag)\n when \"webdav\"\n require 'git-media/transport/webdav'\n\n url = `git config git-media.webdavurl`.chomp\n user = `git config git-media.webdavuser`.chomp\n password = `git config git-media.webdavpassword`.chomp\n verify_server = `git config git-media.webdavverifyserver`.chomp == 'true'\n binary_transfer = `git config git-media.webdavbinarytransfer`.chomp == 'true'\n if url == \"\"\n raise \"git-media.webdavurl not set for webdav transport\"\n end\n if user == \"\"\n user, password = self.get_credentials_from_netrc(url)\n end\n if !user\n raise \"git-media.webdavuser not set for webdav transport\"\n end\n if !password\n raise \"git-media.webdavpassword not set for webdav transport\"\n end\n GitMedia::Transport::WebDav.new(url, user, password, verify_server, binary_transfer)\n when \"box\"\n require 'git-media/transport/box'\n\n client_id = `git config git-media.boxclientid`.chomp\n client_secret = `git config git-media.boxclientsecret`.chomp\n redirect_uri = `git config git-media.boxredirecturi`.chomp\n folder_id = `git config git-media.boxfolderid`.chomp\n\n access_token = `git config git-media.boxaccesstoken`.chomp\n refresh_token = `git config git-media.boxrefreshtoken`.chomp\n if client_id == \"\"\n raise \"git-media.boxclientid not set for box transport\"\n end\n if client_secret == \"\"\n raise \"git-media.boxclientsecret not set for box transport\"\n end\n if redirect_uri == \"\"\n raise \"git-media.boxredirecturi not set for box transport\"\n end\n if folder_id == \"\"\n raise \"git-media.boxfolderid not set for box transport\"\n end\n GitMedia::Transport::Box.new(client_id, client_secret, redirect_uri, folder_id, access_token, refresh_token)\n else\n raise \"Invalid transport #{transport}\"\n end\n end\n\n def self.get_pull_transport\n self.get_transport\n end\n\n module Application\n def self.run!\n\n if !system('git rev-parse')\n return\n end\n\n cmd = ARGV.shift # get the subcommand\n cmd_opts = case cmd\n when \"filter-clean\" # parse delete options\n require 'git-media/filter-clean'\n GitMedia::FilterClean.run!\n when \"filter-smudge\"\n require 'git-media/filter-smudge'\n GitMedia::FilterSmudge.run!\n when \"clear\" # parse delete options\n require 'git-media/clear'\n GitMedia::Clear.run!\n when \"sync\"\n require 'git-media/sync'\n GitMedia::Sync.run!\n when 'status'\n require 'git-media/status'\n opts = Trollop::options do\n opt :force, \"Force status\"\n opt :short, \"Short status\"\n end\n GitMedia::Status.run!(opts)\n when 'retroactively-apply'\n require 'git-media/filter-branch'\n GitMedia::FilterBranch.clean!\n arg2 = \"--index-filter 'git media index-filter #{ARGV.shift}'\"\n system(\"git filter-branch #{arg2} --tag-name-filter cat -- --all\")\n GitMedia::FilterBranch.clean!\n when 'index-filter'\n require 'git-media/filter-branch'\n GitMedia::FilterBranch.run!\n else\n print < to_rewrite'\n\nEOF\n end\n\n end\n end\nend\n" }, { "path": "spec/media_spec.rb", "content": "require File.expand_path(File.dirname(__FILE__) + '/spec_helper')\n\n# I realize this is horrible, horrible rspec but I want to run the actual\n# git commands and it takes forever to setup the test env each time, so\n# i'm squeezing a bunch of tests into each 'it' - don't judge me\n\ndescribe \"Media\" do\n \n it \"should clean and smudge and save data in buffer area\" do\n in_temp_git_w_media do\n git('add .')\n git(\"commit -m 'testing'\")\n \n # check that we saved the sha and not the data\n size = git(\"cat-file -s master:testing1.mov\")\n size.should eql('41')\n \n # check that the data is in our buffer area\n Dir.chdir('.git/media/objects') do\n objects = Dir.glob('*')\n objects.should include('20eabe5d64b0e216796e834f52d61fd0b70332fc')\n end\n \n # check that removing the file and checking out returns the data\n File.unlink('testing1.mov')\n git('checkout testing1.mov')\n File.size('testing1.mov').should eql(7)\n \n # check that removing the file and checking out sans data returns the sha\n File.unlink('testing1.mov')\n File.unlink('.git/media/objects/20eabe5d64b0e216796e834f52d61fd0b70332fc')\n git('checkout testing1.mov')\n File.size('testing1.mov').should eql(41)\n end\n end\n \n it \"should show me the status of my directory\"\n \n it \"should sync with a local transport\"\n \nend\n" }, { "path": "spec/spec_helper.rb", "content": "require 'rubygems'\nrequire 'spec'\nrequire 'tempfile'\nrequire 'pp'\n\n$LOAD_PATH.unshift(File.dirname(__FILE__))\n$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))\nrequire 'git-media'\n\nSpec::Runner.configure do |config|\nend\n\ndef in_temp_git\n tf = Tempfile.new('gitdir')\n temppath = tf.path\n tf.unlink\n FileUtils.mkdir(temppath)\n Dir.chdir(temppath) do\n `git init`\n yield\n end\nend\n\ndef in_temp_git_w_media\n bin = File.join(File.dirname(__FILE__), '..', 'bin', 'git-media') \n in_temp_git do\n append_file('testing1.mov', '1234567')\n append_file('testing2.mov', '123456789')\n append_file('normal.txt', 'hello world')\n append_file('.gitattributes', '*.mov filter=media')\n `git config filter.media.clean \"#{bin} clean\"`\n `git config filter.media.smudge \"#{bin} smudge\"`\n yield\n end\nend\n\ndef append_file(filename, content)\n File.open(filename, 'w+') do |f|\n f.print content\n end\nend\n\ndef git(command)\n output = `git #{command} 2>/dev/null`.strip\nend" } ]