diff --git a/backend/app/services/google_drive/api_client.rb b/backend/app/services/google_drive/api_client.rb index 9467333..b3a9300 100644 --- a/backend/app/services/google_drive/api_client.rb +++ b/backend/app/services/google_drive/api_client.rb @@ -26,13 +26,19 @@ module GoogleDrive def list_material_files_under_folder folder_id files = [] + each_material_file_under_folder(folder_id) { |entry| files << entry } + files + end + + def each_material_file_under_folder folder_id + return enum_for(__method__, folder_id) unless block_given? + walk_folder(folder_id, nil) do |entry, relative_path| next if entry['mimeType'] == FOLDER_MIME_TYPE next if native_file?(entry['mimeType']) - files << build_file_entry(entry, relative_path) + yield build_file_entry(entry, relative_path) end - files end def fetch_material_file file_id @@ -44,9 +50,10 @@ module GoogleDrive def download_to_tempfile file_id, filename: tempfile = Tempfile.new(['material-sync', File.extname(filename.to_s)]) + tempfile.binmode request_binary("/files/#{ file_id }", { alt: 'media', supportsAllDrives: true }) do |chunk| - tempfile.write(chunk) + tempfile.write(chunk.b) end tempfile.rewind tempfile @@ -139,7 +146,7 @@ module GoogleDrive end response.read_body do |chunk| - yield chunk + yield chunk.b end end end diff --git a/backend/app/services/material_sync_runner.rb b/backend/app/services/material_sync_runner.rb index 00baac0..1e96c85 100644 --- a/backend/app/services/material_sync_runner.rb +++ b/backend/app/services/material_sync_runner.rb @@ -21,10 +21,14 @@ class MaterialSyncRunner result = Result.new(imported: 0, updated: 0, unchanged: 0, suppressed: 0, failed: 0, errors: []) - candidates.each do |candidate| - next if candidate.blank? + if @source.source_kind == 'google_drive_path' + sync_google_drive_path!(result) + else + candidates.each do |candidate| + next if candidate.blank? - sync_candidate!(candidate, result) + sync_candidate!(candidate, result) + end end @source.update!(last_synced_at: Time.current) @@ -43,8 +47,6 @@ class MaterialSyncRunner case @source.source_kind when 'uri' [uri_candidate] - when 'google_drive_path' - google_drive_path_candidates when 'google_drive_file' [google_drive_file_candidate] when 'legacy_drive_path' @@ -104,12 +106,25 @@ class MaterialSyncRunner def google_drive_path_candidates folder_id = google_drive_folder_id Enumerator.new do |entries| - drive_client.list_material_files_under_folder(folder_id).each do |entry| + drive_client.each_material_file_under_folder(folder_id).each do |entry| entries << build_google_drive_candidate(entry) end end end + def sync_google_drive_path! result + folder_id = google_drive_folder_id + scanned_count = 0 + + drive_client.each_material_file_under_folder(folder_id) do |entry| + scanned_count += 1 + sync_candidate!(build_google_drive_candidate(entry), result) + log_google_drive_progress(folder_id, scanned_count, result) if progress_log_scan_count?(scanned_count) + end + + log_google_drive_progress(folder_id, scanned_count, result, summary: true) + end + def google_drive_file_candidate entry = drive_client.fetch_material_file(google_drive_file_id) return nil unless entry @@ -213,6 +228,22 @@ class MaterialSyncRunner failed: result.failed)) end + def progress_log_scan_count? scanned_count + scanned_count == 1 || (scanned_count % 50).zero? + end + + def log_google_drive_progress folder_id, scanned_count, result, summary: false + Rails.logger.info( + material_sync_log(folder_id:, + scanned_count:, + imported: result.imported, + updated: result.updated, + unchanged: result.unchanged, + suppressed: result.suppressed, + failed: result.failed, + progress: summary ? 'summary' : 'scan')) + end + def material_sync_log fields { material_sync_source_id: @source.id, material_sync_source_name: @source.name }.merge(fields).to_json diff --git a/backend/spec/services/material_sync_runner_spec.rb b/backend/spec/services/material_sync_runner_spec.rb new file mode 100644 index 0000000..ace4f45 --- /dev/null +++ b/backend/spec/services/material_sync_runner_spec.rb @@ -0,0 +1,89 @@ +require 'rails_helper' + +RSpec.describe MaterialSyncRunner do + let(:user) { create(:user, :member) } + let(:source) do + MaterialSyncSource.create!( + name: 'Drive source', + source_kind: 'google_drive_path', + source_file_id: 'folder-123', + profile: 'legacy_drive', + created_by_user: user, + updated_by_user: user) + end + let(:drive_client) { instance_double(GoogleDrive::ApiClient) } + + describe '#sync!' do + it 'imports google drive path candidates as they are yielded' do + first_entry = { id: 'file-1', + name: 'a.png', + mime_type: 'image/png', + relative_path: '素材/a.png', + sha256_checksum: 'sha-a', + web_view_link: 'https://drive.google.com/file/d/file-1/view', + web_content_link: nil } + second_entry = { id: 'file-2', + name: 'b.png', + mime_type: 'image/png', + relative_path: '素材/b.png', + sha256_checksum: 'sha-b', + web_view_link: 'https://drive.google.com/file/d/file-2/view', + web_content_link: nil } + yielded = [] + + allow(GoogleDrive::ApiClient).to receive(:new).and_return(drive_client) + allow(drive_client).to receive(:each_material_file_under_folder) do |folder_id, &block| + expect(folder_id).to eq('folder-123') + block.call(first_entry) + expect(yielded).to eq(['素材/a.png']) + block.call(second_entry) + end + allow(MaterialSyncImporter).to receive(:import!) do |candidate| + yielded << candidate.fetch(:source_path) + instance_double(MaterialSyncImporter::Result, + action: yielded.last == '素材/a.png' ? :imported : :updated) + end + + result = described_class.new(source).sync! + + expect(yielded).to eq(['素材/a.png', '素材/b.png']) + expect(result.imported).to eq(1) + expect(result.updated).to eq(1) + expect(result.unchanged).to eq(0) + expect(source.reload.last_synced_at).to be_present + end + + it 'logs google drive path progress at first item and summary' do + entry = { id: 'file-1', + name: 'a.png', + mime_type: 'image/png', + relative_path: '素材/a.png', + sha256_checksum: 'sha-a', + web_view_link: 'https://drive.google.com/file/d/file-1/view', + web_content_link: nil } + logged = [] + + allow(GoogleDrive::ApiClient).to receive(:new).and_return(drive_client) + allow(drive_client).to receive(:each_material_file_under_folder) do |_folder_id, &block| + block.call(entry) + end + allow(MaterialSyncImporter).to receive(:import!) + .and_return(instance_double(MaterialSyncImporter::Result, action: :imported)) + allow(Rails.logger).to receive(:info) { |message| logged << JSON.parse(message) } + + described_class.new(source).sync! + + progress_logs = logged.select { |row| row['folder_id'] == 'folder-123' } + expect(progress_logs.map { |row| row['progress'] }).to eq(['scan', 'summary']) + expect(progress_logs.last).to include( + 'material_sync_source_id' => source.id, + 'material_sync_source_name' => 'Drive source', + 'scanned_count' => 1, + 'imported' => 1, + 'updated' => 0, + 'unchanged' => 0, + 'suppressed' => 0, + 'failed' => 0) + end + end +end diff --git a/frontend/src/components/TopNav.tsx b/frontend/src/components/TopNav.tsx index dccd9ee..872ddfb 100644 --- a/frontend/src/components/TopNav.tsx +++ b/frontend/src/components/TopNav.tsx @@ -55,7 +55,7 @@ export const menuOutline = ( visible: tagFlg }, { name: '履歴', to: `/tags/changes?id=${ tag?.id }`, visible: tagFlg && tag?.category !== 'nico' }] }, - { name: '素材', to: '/materials', visible: false, subMenu: [ + { name: '素材', to: '/materials', visible: true, subMenu: [ { name: '一覧', to: '/materials' }, { name: '追加', to: '/materials/new' }, { name: '全体履歴', to: '/materials/changes' }, @@ -63,7 +63,7 @@ export const menuOutline = ( { component: , visible: materialFlg }, { name: `広場 (${ postCount || 0 })`, to: `/posts?tags=${ encodeURIComponent (material?.tag?.name ?? '') }`, - visible: materialFlg }, + visible: materialFlg && Boolean (material?.tag) }, { name: '履歴', to: `/materials/changes?material_id=${ material?.id }`, visible: materialFlg }] }, { name: 'Wiki', to: '/wiki/ヘルプ:ホーム', base: '/wiki', subMenu: [ diff --git a/frontend/src/pages/materials/MaterialListPage.tsx b/frontend/src/pages/materials/MaterialListPage.tsx index f585df0..b76640d 100644 --- a/frontend/src/pages/materials/MaterialListPage.tsx +++ b/frontend/src/pages/materials/MaterialListPage.tsx @@ -118,7 +118,7 @@ const MaterialThumb: FC<{ material: Material }> = ({ material }) => ( text-stone-900 shadow-sm dark:border-stone-700 dark:bg-stone-900 dark:text-stone-100`}> {material.thumbnail - ? + ? : (