#306

backend/app/services/google_drive/api_client.rb

@@ -26,13 +26,19 @@ module GoogleDrive
 
     def list_material_files_under_folder folder_id
       files = []
+      each_material_file_under_folder(folder_id) { |entry| files << entry }
+      files
+    end
+
+    def each_material_file_under_folder folder_id
+      return enum_for(__method__, folder_id) unless block_given?
+
       walk_folder(folder_id, nil) do |entry, relative_path|
         next if entry['mimeType'] == FOLDER_MIME_TYPE
         next if native_file?(entry['mimeType'])
 
-        files << build_file_entry(entry, relative_path)
+        yield build_file_entry(entry, relative_path)
       end
-      files
     end
 
     def fetch_material_file file_id
@@ -44,9 +50,10 @@ module GoogleDrive
 
     def download_to_tempfile file_id, filename:
       tempfile = Tempfile.new(['material-sync', File.extname(filename.to_s)])
+      tempfile.binmode
       request_binary("/files/#{ file_id }",
                      { alt: 'media', supportsAllDrives: true }) do |chunk|
-        tempfile.write(chunk)
+        tempfile.write(chunk.b)
       end
       tempfile.rewind
       tempfile
@@ -139,7 +146,7 @@ module GoogleDrive
         end
 
         response.read_body do |chunk|
-          yield chunk
+          yield chunk.b
         end
       end
     end

backend/app/services/material_sync_runner.rb

@@ -21,10 +21,14 @@ class MaterialSyncRunner
     result = Result.new(imported: 0, updated: 0, unchanged: 0,
                         suppressed: 0, failed: 0, errors: [])
 
-    candidates.each do |candidate|
-      next if candidate.blank?
+    if @source.source_kind == 'google_drive_path'
+      sync_google_drive_path!(result)
+    else
+      candidates.each do |candidate|
+        next if candidate.blank?
 
-      sync_candidate!(candidate, result)
+        sync_candidate!(candidate, result)
+      end
     end
 
     @source.update!(last_synced_at: Time.current)
@@ -43,8 +47,6 @@ class MaterialSyncRunner
     case @source.source_kind
     when 'uri'
       [uri_candidate]
-    when 'google_drive_path'
-      google_drive_path_candidates
     when 'google_drive_file'
       [google_drive_file_candidate]
     when 'legacy_drive_path'
@@ -104,12 +106,25 @@ class MaterialSyncRunner
   def google_drive_path_candidates
     folder_id = google_drive_folder_id
     Enumerator.new do |entries|
-      drive_client.list_material_files_under_folder(folder_id).each do |entry|
+      drive_client.each_material_file_under_folder(folder_id).each do |entry|
         entries << build_google_drive_candidate(entry)
       end
     end
   end
 
+  def sync_google_drive_path! result
+    folder_id = google_drive_folder_id
+    scanned_count = 0
+
+    drive_client.each_material_file_under_folder(folder_id) do |entry|
+      scanned_count += 1
+      sync_candidate!(build_google_drive_candidate(entry), result)
+      log_google_drive_progress(folder_id, scanned_count, result) if progress_log_scan_count?(scanned_count)
+    end
+
+    log_google_drive_progress(folder_id, scanned_count, result, summary: true)
+  end
+
   def google_drive_file_candidate
     entry = drive_client.fetch_material_file(google_drive_file_id)
     return nil unless entry
@@ -213,6 +228,22 @@ class MaterialSyncRunner
                                         failed: result.failed))
   end
 
+  def progress_log_scan_count? scanned_count
+    scanned_count == 1 || (scanned_count % 50).zero?
+  end
+
+  def log_google_drive_progress folder_id, scanned_count, result, summary: false
+    Rails.logger.info(
+        material_sync_log(folder_id:,
+                          scanned_count:,
+                          imported: result.imported,
+                          updated: result.updated,
+                          unchanged: result.unchanged,
+                          suppressed: result.suppressed,
+                          failed: result.failed,
+                          progress: summary ? 'summary' : 'scan'))
+  end
+
   def material_sync_log fields
     { material_sync_source_id: @source.id,
       material_sync_source_name: @source.name }.merge(fields).to_json

backend/spec/services/material_sync_runner_spec.rb

@@ -0,0 +1,89 @@
+require 'rails_helper'
+
+RSpec.describe MaterialSyncRunner do
+  let(:user) { create(:user, :member) }
+  let(:source) do
+    MaterialSyncSource.create!(
+        name: 'Drive source',
+        source_kind: 'google_drive_path',
+        source_file_id: 'folder-123',
+        profile: 'legacy_drive',
+        created_by_user: user,
+        updated_by_user: user)
+  end
+  let(:drive_client) { instance_double(GoogleDrive::ApiClient) }
+
+  describe '#sync!' do
+    it 'imports google drive path candidates as they are yielded' do
+      first_entry = { id: 'file-1',
+                      name: 'a.png',
+                      mime_type: 'image/png',
+                      relative_path: '素材/a.png',
+                      sha256_checksum: 'sha-a',
+                      web_view_link: 'https://drive.google.com/file/d/file-1/view',
+                      web_content_link: nil }
+      second_entry = { id: 'file-2',
+                       name: 'b.png',
+                       mime_type: 'image/png',
+                       relative_path: '素材/b.png',
+                       sha256_checksum: 'sha-b',
+                       web_view_link: 'https://drive.google.com/file/d/file-2/view',
+                       web_content_link: nil }
+      yielded = []
+
+      allow(GoogleDrive::ApiClient).to receive(:new).and_return(drive_client)
+      allow(drive_client).to receive(:each_material_file_under_folder) do |folder_id, &block|
+        expect(folder_id).to eq('folder-123')
+        block.call(first_entry)
+        expect(yielded).to eq(['素材/a.png'])
+        block.call(second_entry)
+      end
+      allow(MaterialSyncImporter).to receive(:import!) do |candidate|
+        yielded << candidate.fetch(:source_path)
+        instance_double(MaterialSyncImporter::Result,
+                        action: yielded.last == '素材/a.png' ? :imported : :updated)
+      end
+
+      result = described_class.new(source).sync!
+
+      expect(yielded).to eq(['素材/a.png', '素材/b.png'])
+      expect(result.imported).to eq(1)
+      expect(result.updated).to eq(1)
+      expect(result.unchanged).to eq(0)
+      expect(source.reload.last_synced_at).to be_present
+    end
+
+    it 'logs google drive path progress at first item and summary' do
+      entry = { id: 'file-1',
+                name: 'a.png',
+                mime_type: 'image/png',
+                relative_path: '素材/a.png',
+                sha256_checksum: 'sha-a',
+                web_view_link: 'https://drive.google.com/file/d/file-1/view',
+                web_content_link: nil }
+      logged = []
+
+      allow(GoogleDrive::ApiClient).to receive(:new).and_return(drive_client)
+      allow(drive_client).to receive(:each_material_file_under_folder) do |_folder_id, &block|
+        block.call(entry)
+      end
+      allow(MaterialSyncImporter).to receive(:import!)
+        .and_return(instance_double(MaterialSyncImporter::Result, action: :imported))
+      allow(Rails.logger).to receive(:info) { |message| logged << JSON.parse(message) }
+
+      described_class.new(source).sync!
+
+      progress_logs = logged.select { |row| row['folder_id'] == 'folder-123' }
+      expect(progress_logs.map { |row| row['progress'] }).to eq(['scan', 'summary'])
+      expect(progress_logs.last).to include(
+          'material_sync_source_id' => source.id,
+          'material_sync_source_name' => 'Drive source',
+          'scanned_count' => 1,
+          'imported' => 1,
+          'updated' => 0,
+          'unchanged' => 0,
+          'suppressed' => 0,
+          'failed' => 0)
+    end
+  end
+end