diff --git a/storage/Gemfile b/storage/Gemfile index 1c6b5c34e..cf1e1f29a 100644 --- a/storage/Gemfile +++ b/storage/Gemfile @@ -1,4 +1,4 @@ -# Copyright 2015 Google, Inc +# Copyright 2016 Google, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ source "https://rubygems.org" -gem "google-api-client", "~>0.9.pre3" +gem "google-cloud-storage" group :test do gem "rspec" diff --git a/storage/Gemfile.lock b/storage/Gemfile.lock index 9f2769ef7..f256049a9 100644 --- a/storage/Gemfile.lock +++ b/storage/Gemfile.lock @@ -3,6 +3,7 @@ GEM specs: addressable (2.4.0) diff-lcs (1.2.5) + digest-crc (0.4.1) faraday (0.9.2) multipart-post (>= 1.2, < 3) google-api-client (0.9.15) @@ -14,6 +15,11 @@ GEM mime-types (>= 1.6) representable (~> 2.3.0) retriable (~> 2.0) + google-cloud-core (0.20.1) + google-cloud-storage (0.20.1) + digest-crc (~> 0.4) + google-api-client (~> 0.9.11) + google-cloud-core (~> 0.20.0) googleauth (0.5.1) faraday (~> 0.9) jwt (~> 1.4) @@ -63,7 +69,7 @@ PLATFORMS ruby DEPENDENCIES - google-api-client (~> 0.9.pre3) + google-cloud-storage rspec BUNDLED WITH diff --git a/storage/README.md b/storage/README.md new file mode 100644 index 000000000..b7c90f880 --- /dev/null +++ b/storage/README.md @@ -0,0 +1,58 @@ +Google Cloud Platform logo + +# Google Cloud Storage Ruby Samples + +[Cloud Storage][storage_docs] allows world-wide storage and retrieval of any +amount of data at any time. + +[storage_docs]: https://cloud.google.com/storage/docs/ + +## Run sample + +To run the sample, first install dependencies: + + bundle install + +Run the sample: + + bundle exec ruby buckets.rb + bundle exec ruby files.rb + +## Samples + +### Buckets + +**Usage:** `bundle exec ruby buckets.rb [command] [arguments]` + +``` +sage: bundle exec ruby buckets.rb [command] [arguments] + +Commands: + list List all buckets in the authenticated project + create Create a new bucket with the provided name + delete Delete bucket with the provided name + +Environment variables: + GCLOUD_PROJECT must be set to your Google Cloud project ID +``` + +### Files + +**Usage:** `bundle exec ruby files.rb [command] [arguments]` + +``` +Usage: bundle exec ruby files.rb [command] [arguments] + +Commands: + list List all files in the bucket + upload Upload local file to a bucket + download Download a file from a bucket + delete Delete a file from a bucket + metadata Display metadata for a file in a bucket + make_public Make a file in a bucket public + rename Rename a file in a bucket + copy Copy file to other bucket + +Environment variables: + GCLOUD_PROJECT must be set to your Google Cloud project ID +``` diff --git a/storage/buckets.rb b/storage/buckets.rb new file mode 100644 index 000000000..d6a38bb77 --- /dev/null +++ b/storage/buckets.rb @@ -0,0 +1,85 @@ +# Copyright 2016 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in write, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def list_buckets project_id: + # [START list_buckets] + # project_id = "Your Google Cloud project ID" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + + storage.buckets.each do |bucket| + puts bucket.name + end + # [END list_buckets] +end + +def create_bucket project_id:, bucket_name: + # [START create_bucket] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Name of Google Cloud Storage bucket to create" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.create_bucket bucket_name + + puts "Created bucket: #{bucket.name}" + # [END create_bucket] +end + +def delete_bucket project_id:, bucket_name: + # [START delete_bucket] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Name of your Google Cloud Storage bucket to delete" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + + bucket.delete + + puts "Deleted bucket: #{bucket.name}" + # [END delete_bucket] +end + +if __FILE__ == $0 + case ARGV.shift + when "list" + list_buckets project_id: ENV["GCLOUD_PROJECT"] + when "create" + create_bucket project_id: ENV["GCLOUD_PROJECT"], + bucket_name: ARGV.shift + when "delete" + delete_bucket project_id: ENV["GCLOUD_PROJECT"], + bucket_name: ARGV.shift + else + puts <<-usage +Usage: bundle exec ruby buckets.rb [command] [arguments] + +Commands: + list List all buckets in the authenticated project + create Create a new bucket with the provided name + delete Delete bucket with the provided name + +Environment variables: + GCLOUD_PROJECT must be set to your Google Cloud project ID + usage + end +end diff --git a/storage/files.rb b/storage/files.rb new file mode 100644 index 000000000..96364ed4c --- /dev/null +++ b/storage/files.rb @@ -0,0 +1,274 @@ +# Copyright 2016 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in write, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def list_bucket_contents project_id:, bucket_name: + # [START list_bucket_contents] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + + bucket.files.each do |file| + puts file.name + end + # [END list_bucket_contents] +end + +def list_bucket_contents_with_prefix project_id:, bucket_name:, prefix: + # [START list_bucket_contents] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + # prefix = "Filter results to files whose names begin with this prefix" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + files = bucket.files prefix: prefix + + files.each do |file| + puts file.name + end + # [END list_bucket_contents] +end + +def upload_file project_id:, bucket_name:, local_file_path:, + storage_file_path: + # [START upload_file] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + # local_file_path = "Path to local file to upload" + # storage_file_path = "Path to store the file in Google Cloud Storage" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + + file = bucket.create_file local_file_path, storage_file_path + + puts "Uploaded #{file.name}" + # [END upload_file] +end + +def download_file project_id:, bucket_name:, file_name:, local_path: + # [START download_file] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + # file_name = "Name of file in Google Cloud Storage to download locally" + # local_path = "Path to local file to save" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + file = bucket.file file_name + + file.download local_path + + puts "Downloaded #{file.name}" + # [END download_file] +end + +def delete_file project_id:, bucket_name:, file_name: + # [START delete_file] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + # file_name = "Name of file in Google Cloud Storage to delete" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + file = bucket.file file_name + + file.delete + + puts "Deleted #{file.name}" + # [END delete_file] +end + +def list_file_details project_id:, bucket_name:, file_name: + # [START list_file_details] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + # file_name = "Name of file in Google Cloud Storage" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + file = bucket.file file_name + + puts "Name: #{file.name}" + puts "Bucket: #{bucket.name}" + puts "Storage class: #{bucket.storage_class}" + puts "ID: #{file.id}" + puts "Size: #{file.size} bytes" + puts "Created: #{file.created_at}" + puts "Updated: #{file.updated_at}" + puts "Generation: #{file.generation}" + puts "Metageneration: #{file.metageneration}" + puts "Etag: #{file.etag}" + puts "Owners: #{file.acl.owners.join ","}" + puts "Crc32c: #{file.crc32c}" + puts "md5_hash: #{file.md5}" + puts "Cache-control: #{file.cache_control}" + puts "Content-type: #{file.content_type}" + puts "Content-disposition: #{file.content_disposition}" + puts "Content-encoding: #{file.content_encoding}" + puts "Content-language: #{file.content_language}" + puts "Metadata:" + file.metadata.each do |key, value| + puts " - #{key} = #{value}" + end + # [END list_file_details] +end + +def make_file_public project_id:, bucket_name:, file_name: + # [START make_file_public] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + # file_name = "Name of file in Google Cloud Storage to make public" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + file = bucket.file file_name + + file.acl.public! + + puts "#{file.name} is publicly accessible at #{file.public_url}" + # [END make_file_public] +end + +def rename_file project_id:, bucket_name:, file_name:, new_name: + # [START rename_file] + # project_id = "Your Google Cloud project ID" + # bucket_name = "Your Google Cloud Storage bucket name" + # file_name = "Name of file in Google Cloud Storage to rename" + # new_name = "File will be renamed to this new name" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket bucket_name + file = bucket.file file_name + + renamed_file = file.copy new_name + + file.delete + + puts "my-file.txt has been renamed to #{renamed_file.name}" + # [END rename_file] +end + +def copy_file project_id:, source_bucket_name:, source_file_name:, + dest_bucket_name:, dest_file_name: + # [START copy_file] + # project_id = "Your Google Cloud project ID" + # source_bucket_name = "Source bucket to copy file from" + # source_file_name = "Source file name" + # dest_bucket_name = "Destination bucket to copy file to" + # dest_file_name = "Destination file name" + + require "google/cloud" + + gcloud = Google::Cloud.new project_id + storage = gcloud.storage + bucket = storage.bucket source_bucket_name + file = bucket.file source_file_name + + destination_bucket = storage.bucket dest_bucket_name + destination_file = file.copy destination_bucket.name, file.name + + puts "#{file.name} in #{bucket.name} copied to " + + "#{copied_file.name} in #{destination_bucket.name}" + # [END copy_file] +end + +def run_sample arguments + command = arguments.shift + + case command + when "list" + list_bucket_contents project_id: ENV["GCLOUD_PROJECT"], + bucket_name: arguments.shift + when "upload" + upload_file project_id: ENV["GCLOUD_PROJECT"], + bucket_name: arguments.shift, + local_file_path: arguments.shift + when "download" + download_file project_id: ENV["GCLOUD_PROJECT"], + bucket_name: arguments.shift, + file_name: arguments.shift, + local_path: arguments.shift + when "delete" + delete_file project_id: ENV["GCLOUD_PROJECT"], + bucket_name: arguments.shift, + file_name: arguments.shift + when "metadata" + list_file_details project_id: ENV["GCLOUD_PROJECT"], + bucket_name: arguments.shift, + file_name: arguments.shift + when "make_public" + make_file_public project_id: ENV["GCLOUD_PROJECT"], + bucket_name: arguments.shift, + file_name: arguments.shift + when "rename" + rename_file project_id: ENV["GCLOUD_PROJECT"], + bucket_name: arguments.shift, + file_name: arguments.shift, + new_name: arguments.shift + when "copy" + copy_file project_id: ENV["GCLOUD_PROJECT"], + source_bucket_name: arguments.shift, + source_file_name: arguments.shift, + dest_bucket_name: arguments.shift, + dest_file_name: arguments.shift + else + puts <<-usage +Usage: bundle exec ruby files.rb [command] [arguments] + +Commands: + list List all files in the bucket + upload Upload local file to a bucket + download Download a file from a bucket + delete Delete a file from a bucket + metadata Display metadata for a file in a bucket + make_public Make a file in a bucket public + rename Rename a file in a bucket + copy Copy file to other bucket + +Environment variables: + GCLOUD_PROJECT must be set to your Google Cloud project ID + usage + end +end + +if __FILE__ == $0 + run_sample ARGV +end diff --git a/storage/spec/buckets_spec.rb b/storage/spec/buckets_spec.rb new file mode 100644 index 000000000..4e1fac8c6 --- /dev/null +++ b/storage/spec/buckets_spec.rb @@ -0,0 +1,83 @@ +# Copyright 2016 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative "../buckets" +require "rspec" +require "google/cloud" + +RSpec.describe "Google Cloud Storage buckets sample" do + + before :all do + @project_id = ENV["GOOGLE_PROJECT_ID"] + @bucket_name = ENV["STORAGE_BUCKET"] + @gcloud = Google::Cloud.new @project_id + @storage = @gcloud.storage + end + + before do + delete_bucket! + @storage.create_bucket @bucket_name + end + + after :all do + # Other tests assume that this bucket exists, + # so create it before exiting this spec suite + @storage.create_bucket @bucket_name unless @storage.bucket(@bucket_name) + end + + def delete_bucket! + bucket = @storage.bucket @bucket_name + + if bucket + bucket.files.each &:delete until bucket.files.empty? + bucket.delete + end + end + + example "list buckets" do + expect { + list_buckets project_id: @project_id + }.to output( + /#{@bucket_name}/ + ).to_stdout + end + + example "create bucket" do + delete_bucket! + + expect(@storage.bucket @bucket_name).to be nil + + expect { + create_bucket project_id: @project_id, + bucket_name: @bucket_name + }.to output( + "Created bucket: #{@bucket_name}\n" + ).to_stdout + + expect(@storage.bucket @bucket_name).not_to be nil + end + + example "delete bucket" do + expect(@storage.bucket @bucket_name).not_to be nil + + expect { + delete_bucket project_id: @project_id, bucket_name: @bucket_name + }.to output( + "Deleted bucket: #{@bucket_name}\n" + ).to_stdout + + expect(@storage.bucket @bucket_name).to be nil + end + +end diff --git a/storage/spec/files_spec.rb b/storage/spec/files_spec.rb new file mode 100644 index 000000000..0262af563 --- /dev/null +++ b/storage/spec/files_spec.rb @@ -0,0 +1,137 @@ +# Copyright 2016 Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative "../files" +require "rspec" +require "google/cloud" +require "tempfile" + +describe "Google Cloud Storage files sample" do + + before do + @project_id = ENV["GOOGLE_PROJECT_ID"] + @bucket_name = ENV["STORAGE_BUCKET"] + @gcloud = Google::Cloud.new @project_id + @storage = @gcloud.storage + @bucket = @storage.bucket @bucket_name + @local_file_path = File.expand_path "resources/file.txt", __dir__ + end + + # Delete given file in Cloud Storage test bucket if it exists + def delete_file storage_file_path + @bucket.file(storage_file_path).delete if @bucket.file storage_file_path + end + + # Upload a local file to the Cloud Storage test bucket + def upload local_file_path, storage_file_path + unless @bucket.file storage_file_path + @bucket.create_file local_file_path, storage_file_path + end + end + + # Returns the content of an uploaded file in Cloud Storage test bucket + def storage_file_content storage_file_path + local_tempfile = Tempfile.new "cloud-storage-tests" + storage_file = @bucket.file storage_file_path + storage_file.download local_tempfile.path + File.read local_tempfile.path + ensure + local_tempfile.close + local_tempfile.unlink + end + + # Capture and return STDOUT output by block + def capture &block + real_stdout = $stdout + $stdout = StringIO.new + block.call + @captured_output = $stdout.string + ensure + $stdout = real_stdout + end + attr_reader :captured_output + + it "can list files in a bucket" do + upload @local_file_path, "file.txt" + expect(@bucket.file "file.txt").not_to be nil + + expect { + list_bucket_contents project_id: @project_id, + bucket_name: @bucket_name + }.to output( + /file\.txt/ + ).to_stdout + end + + it "can list files with a prefix in a bucket" do + upload @local_file_path, "foo/hello" + upload @local_file_path, "foo/hi/there" + upload @local_file_path, "bar/hello" + upload @local_file_path, "bar/hi/there" + + capture do + list_bucket_contents_with_prefix project_id: @project_id, + bucket_name: @bucket_name, + prefix: "foo/" + end + + expect(captured_output).to include "foo/hello" + expect(captured_output).to include "foo/hi/there" + expect(captured_output).not_to include "bar/hello" + expect(captured_output).not_to include "bar/hi/there" + end + + it "can upload a local file to a bucket" do + delete_file "file.txt" + expect(@bucket.file "file.txt").to be nil + + expect { + upload_file project_id: @project_id, + bucket_name: @bucket_name, + local_file_path: @local_file_path, + storage_file_path: "file.txt" + }.to output( + /Uploaded .*file.txt/ + ).to_stdout + + expect(@bucket.file "file.txt").not_to be nil + expect(storage_file_content "file.txt").to eq "Content of test file.txt\n" + end + + it "can download a file from a bucket" do + begin + upload @local_file_path, "file.txt" + + local_file = Tempfile.new "cloud-storage-tests" + expect(File.size local_file.path).to eq 0 + + expect { + download_file project_id: @project_id, + bucket_name: @bucket_name, + local_path: local_file.path, + file_name: "file.txt" + }.to output( + "Downloaded file.txt\n" + ).to_stdout + + expect(File.size local_file.path).to be > 0 + expect(File.read local_file.path).to eq( + "Content of test file.txt\n" + ) + ensure + local_file.close + local_file.unlink + end + end +end diff --git a/storage/spec/list_buckets_spec.rb b/storage/spec/list_buckets_spec.rb deleted file mode 100644 index 6ca82abf3..000000000 --- a/storage/spec/list_buckets_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2015 Google, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require_relative "spec_helper" -require_relative "../list_buckets" - -RSpec.describe "List buckets sample" do - before do - @sample = Samples::Storage::ListBuckets.new - end - - it "lists buckets in provided project" do - expect { @sample.list_buckets PROJECT_ID }.to( - output(/#{BUCKET_NAME}/).to_stdout) - end -end diff --git a/storage/spec/resources/file.txt b/storage/spec/resources/file.txt new file mode 100644 index 000000000..2409b20a4 --- /dev/null +++ b/storage/spec/resources/file.txt @@ -0,0 +1 @@ +Content of test file.txt diff --git a/storage/spec/spec_helper.rb b/storage/spec/spec_helper.rb deleted file mode 100644 index 246364839..000000000 --- a/storage/spec/spec_helper.rb +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2015 Google, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "rspec" - -PROJECT_ID=ENV["GOOGLE_PROJECT_ID"] -BUCKET_NAME=ENV["GOOGLE_PROJECT_ID"]