Google云存储组成对象并上传复合材料

Question

我必须撰写更多的对象并上传文件，但我不清楚如何实现这一目标，我正在使用Python。 该代码应在Google Cloud功能上使用。 这些是我要做的步骤： - 从目录中调查文件列表， - 上覆盖的对象， - 设置公众阅读对文件的访问。

import csv
import json
from google.cloud import storage
from variables import *
import time
import datetime

MERCHANT_FILE_NAME = "/tmp/file.csv"
BUCKET_FILE_NAME = "feeds/file.csv"
BUCKET_NAME = "xxx-file-partial-feeds-bucket"

def set_bucket_public_iam(
    bucket,
    bucket_name: str,
    members: List[str] = ["allUsers"],
):
    """Set a public IAM Policy to bucket"""
    # bucket_name = "your-bucket-name"

    #storage_client = storage.Client()
    #bucket = storage_client.bucket(bucket_name)

    policy = bucket.get_iam_policy(requested_policy_version=3)
    policy.bindings.append(
        {"role": "roles/storage.objectViewer", "members": members}
    )

    bucket.set_iam_policy(policy)

    print(f"Bucket {bucket.name} is now publicly readable")

def compose_file(storage_client, bucket_name, blob_name_list, destination_blob_name):
    """Concatenate source blobs into destination blob."""
    # bucket_name = "your-bucket-name"
    # first_blob_name = "first-object-name"
    # second_blob_name = "second-blob-name"
    # destination_blob_name = "destination-object-name"

    storage_client = storage.Client()
    bucket = storage_client.bucket(bucket_name)
    destination = bucket.blob(destination_blob_name)
    destination.content_type = "text/plain"

    # sources is a list of Blob instances, up to the max of 32 instances per request
    sources = [bucket.get_blob(blob_name) for blob_name in blob_name_list]
    destination.compose(sources)

    return destination

def list_blobs(storage_client, bucket_name):
    blobs = storage_client.list_blobs(bucket_name)
    return [ blob.name for blob in blobs ]
    #blob_list = []
    #for blob in blobs:
    #    blob_li
    #    print(blob.name)

def create_file_storage(storage_client, bucket_name):
  try:
    bucket = storage_client.get_bucket(bucket_name)
    blob = bucket.blob(BUCKET_FILE_NAME)
    blob.upload_from_filename(filename=MERCHANT_FILE_NAME)
    #set_bucket_public_iam(bucket, bucket_name)
  except Exception:
    new_bucket = client.create_bucket(bucket_name, location="europe-west3")
    new_blob = new_bucket.blob(BUCKET_FILE_NAME)
    new_blob.upload_from_filename(filename=MERCHANT_FILE_NAME)
    #set_bucket_public_iam(new_bucket, bucket_name)

def merge_files():
    try:
        storage_client = storage.Client()
        blob_names = list_blobs(storage_client, BUCKET_NAME)
        desination = compose_file(storage_client, BUCKET_NAME, blob_names, BUCKET_FILE_NAME)
        set_bucket_public_iam(destination, BUCKET_NAME)
        return {"result": "success"}
    
    except Exception as e:
        print(e)
        return {"error": str(e)}

我尝试了此代码来删除当前目标以创建一个新的目的地以组成，但我会遇到错误，这是compose_file函数：

def compose_file(storage_client, bucket_name, blob_name_list, destination_blob_name):
    """Concatenate source blobs into destination blob."""
    # bucket_name = "your-bucket-name"
    # first_blob_name = "first-object-name"
    # second_blob_name = "second-blob-name"
    # destination_blob_name = "destination-object-name"

    storage_client = storage.Client()
    bucket = storage_client.bucket(bucket_name)
    destination_to_check = bucket.blob(destination_blob_name)
    destination = None

    if destination_to_check.exists():
        print("exists")
        destination_to_check.delete()
        bucket = storage_client.bucket(bucket_name)
        destination = bucket.blob(destination_blob_name)
    else:
        destination = bucket.blob(destination_blob_name)

    destination.content_type = "text/csv"

    # sources is a list of Blob instances, up to the max of 32 instances per request
    sources = [bucket.get_blob(blob_name) for blob_name in blob_name_list]
    destination.compose(sources)

    return destination

erre是：

File "/workspace/main.py", line 88, in merge_files
    destination = compose_file(storage_client, BUCKET_NAME, blob_names, BUCKET_FILE_NAME)
  File "/workspace/main.py", line 60, in compose_file
    destination.compose(sources)
  File "/layers/google.python.pip/pip/lib/python3.9/site-packages/google/cloud/storage/blob.py", line 3439, in compose
    source_object = {"name": source.name, "generation": source.generation}
AttributeError: 'NoneType' object has no attribute 'name'

Answer 1

如果组成的对象已经存在于您的存储桶中，我会成功复制您的错误。

我设法修复了它，因为我注意到 list_blobs 函数将返回包括组合对象或的对象列表，“ feeds/file.csv”文件，如果它已经存在于您的bucket中。您可以做的是在 list变量中创建 list_blobs 函数，然后删除feeds/file.csv项目，如果它已经存在于列表。

以下是 list_blobs函数上的摘要代码：

def list_blobs(storage_client, bucket_name):
    blobs = storage_client.list_blobs(bucket_name)
    #create a blob_list variable
    blob_list = [ blob.name for blob in blobs ]

    #Check if BUCKET_FILE_NAME or "feeds/file.csv" is showing in the list, else do nothing
    if BUCKET_FILE_NAME in blob_list:
        # If BUCKET_FILE_NAME exists, remove from the list
        blob_list.remove(BUCKET_FILE_NAME)

    # your previous code: return [ blob.name for blob in blobs ]
    return blob_list

示例输出：

Bucket <Bucket_NAME> is now publicly readable