From 4ebff6ee4d9d0b49e92d2b2b92ea85d9737abf8e Mon Sep 17 00:00:00 2001 From: kj16609 Date: Mon, 10 Nov 2025 12:22:44 -0500 Subject: [PATCH] Implements importing duplicate & alternative file states from hydrus --- HydrusImporter/src/HydrusImporter.cpp | 90 +++++++- HydrusImporter/src/HydrusImporter.hpp | 2 + .../src/gui/hydrus/HydrusImporterWidget.cpp | 130 ++++++++--- .../src/gui/hydrus/TagServiceWidget.ui | 208 ----------------- .../FileRelationshipsWidget.cpp | 97 ++++++++ .../FileRelationshipsWidget.hpp | 47 ++++ .../FileRelationshipsWidget.ui | 179 +++++++++++++++ .../FileRelationshipsWorker.cpp | 200 ++++++++++++++++ .../FileRelationshipsWorker.hpp | 32 +++ .../{ => tag_service}/TagServiceWidget.cpp | 12 +- .../{ => tag_service}/TagServiceWidget.hpp | 3 +- .../hydrus/tag_service/TagServiceWidget.ui | 213 ++++++++++++++++++ .../{ => tag_service}/TagServiceWorker.cpp | 3 +- .../{ => tag_service}/TagServiceWorker.hpp | 0 HydrusImporter/src/gui/main/MainWindow.cpp | 15 +- .../src/gui/main/SettingsDialog.cpp | 11 +- IDHANClient/include/idhan/IDHANClient.hpp | 14 +- .../records/relationships/setAlternatives.cpp | 46 ++++ .../records/relationships/setDuplicates.cpp | 57 +++++ IDHANMigration/src/130-alternative_files.sql | 49 +++- IDHANServer/src/ServerContext.cpp | 10 +- IDHANServer/src/api/FileRelationshipsAPI.hpp | 26 +++ .../src/api/relationships/addAlternative.cpp | 124 ++++++++++ .../api/relationships/setBetterDuplicate.cpp | 61 +++++ 24 files changed, 1359 insertions(+), 270 deletions(-) delete mode 100644 HydrusImporter/src/gui/hydrus/TagServiceWidget.ui create mode 100644 HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.cpp create mode 100644 HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.hpp create mode 100644 HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.ui create mode 100644 HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.cpp create mode 100644 HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.hpp rename HydrusImporter/src/gui/hydrus/{ => tag_service}/TagServiceWidget.cpp (96%) rename HydrusImporter/src/gui/hydrus/{ => tag_service}/TagServiceWidget.hpp (98%) create mode 100644 HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.ui rename HydrusImporter/src/gui/hydrus/{ => tag_service}/TagServiceWorker.cpp (99%) rename HydrusImporter/src/gui/hydrus/{ => tag_service}/TagServiceWorker.hpp (100%) create mode 100644 IDHANClient/src/records/relationships/setAlternatives.cpp create mode 100644 IDHANClient/src/records/relationships/setDuplicates.cpp create mode 100644 IDHANServer/src/api/FileRelationshipsAPI.hpp create mode 100644 IDHANServer/src/api/relationships/addAlternative.cpp create mode 100644 IDHANServer/src/api/relationships/setBetterDuplicate.cpp diff --git a/HydrusImporter/src/HydrusImporter.cpp b/HydrusImporter/src/HydrusImporter.cpp index 23086db..239c6cf 100644 --- a/HydrusImporter/src/HydrusImporter.cpp +++ b/HydrusImporter/src/HydrusImporter.cpp @@ -50,15 +50,99 @@ HydrusImporter::~HydrusImporter() sqlite3_close_v2( mappings_db ); } +std::unordered_map< HashID, RecordID > HydrusImporter::mapHydrusRecords( std::vector< HashID > hash_ids ) const +{ + idhan::hydrus::TransactionBase master_tr { master_db }; + if ( hash_ids.empty() ) return {}; + + std::ranges::sort( hash_ids ); + std::ranges::unique( hash_ids ); + + std::unordered_map< std::uint32_t, std::string > hashes_map {}; + hashes_map.reserve( hash_ids.size() ); + + for ( const auto& hash_id : hash_ids ) + { + master_tr << "SELECT hex(hash) FROM hashes WHERE hash_id = $1" << hash_id >> + [ & ]( const std::string_view hash_i ) + { + if ( hash_i.size() != ( 256 / 8 * 2 ) ) + { + return; + } + hashes_map.emplace( hash_id, hash_i ); + }; + } + + std::vector< std::string > hashes {}; + hashes.reserve( hash_ids.size() ); + for ( const auto& hash : hashes_map | std::views::values ) hashes.emplace_back( hash ); + + auto& client { idhan::IDHANClient::instance() }; + auto created_records { client.createRecords( hashes ) }; + + created_records.waitForFinished(); + + const auto created_records_result { created_records.result<>() }; + if ( created_records_result.size() != hashes.size() ) throw std::runtime_error( "Failed to create records" ); + + std::unordered_map< std::string, RecordID > record_map {}; + + for ( const auto& [ record_id, hash ] : std::ranges::views::zip( created_records_result, hashes ) ) + { + record_map.emplace( hash, record_id ); + } + + std::unordered_map< HashID, RecordID > record_id_map {}; + + record_id_map.reserve( hash_ids.size() ); + + for ( const auto& [ hy_id, hash ] : hashes_map ) + { + const auto record_id { record_map.at( hash ) }; + record_id_map.emplace( hy_id, record_id ); + } + + return record_id_map; +} + +RecordID HydrusImporter::getRecordIDFromHyID( const HashID hash_id ) +{ + idhan::hydrus::TransactionBaseCoro client_tr { client_db }; + idhan::hydrus::Query< std::string_view > query { + client_tr, "SELECT hex(hash) FROM hashes WHERE hash_id = $1", hash_id + }; + + auto& client { IDHANClient::instance() }; + + std::vector< std::string > hashes {}; + + for ( const auto& [ hash ] : query ) + { + hashes.emplace_back( hash ); + } + + auto future { client.getRecordID( hashes.front() ) }; + + future.waitForFinished(); + + const auto result { future.result<>() }; + if ( !result ) throw std::runtime_error( "Failed to get record from client" ); + + return result.value(); +} + bool HydrusImporter::hasPTR() const { bool exists { false }; TransactionBaseCoro client_tr { client_db }; - Query< std::size_t, std::string_view > query { client_tr, - "SELECT service_id, name FROM services WHERE service_type = $1", - static_cast< int >( hy_constants::ServiceTypes::PTR_SERVICE ) }; + Query< std::size_t, std::string_view > query { + client_tr, + "SELECT service_id, name FROM services WHERE service_type = $1", + static_cast< int >( hy_constants::ServiceTypes::PTR_SERVICE ) + }; for ( [[maybe_unused]] const auto& [ serviae_id, name ] : query ) { diff --git a/HydrusImporter/src/HydrusImporter.hpp b/HydrusImporter/src/HydrusImporter.hpp index 16f62fe..f81246a 100644 --- a/HydrusImporter/src/HydrusImporter.hpp +++ b/HydrusImporter/src/HydrusImporter.hpp @@ -27,6 +27,8 @@ struct ServiceInfo ServiceInfo() = default; }; +using HashID = std::uint32_t; + class HydrusImporter { public: diff --git a/HydrusImporter/src/gui/hydrus/HydrusImporterWidget.cpp b/HydrusImporter/src/gui/hydrus/HydrusImporterWidget.cpp index d2cdac9..7fb6189 100644 --- a/HydrusImporter/src/gui/hydrus/HydrusImporterWidget.cpp +++ b/HydrusImporter/src/gui/hydrus/HydrusImporterWidget.cpp @@ -5,15 +5,16 @@ #include "HydrusImporterWidget.hpp" +#include + #include #include -#include - #include "HydrusImporter.hpp" -#include "TagServiceWidget.hpp" -#include "TagServiceWorker.hpp" +#include "file_relationships/FileRelationshipsWidget.hpp" +#include "tag_service/TagServiceWidget.hpp" #include "ui_HydrusImporterWidget.h" +#include "urls/UrlServiceWidget.hpp" class TagServiceWorker; @@ -41,6 +42,93 @@ HydrusImporterWidget::~HydrusImporterWidget() delete ui; } +void HydrusImporterWidget::parseTagServices() +{ + auto service_infos { m_importer->getTagServices() }; + + for ( const auto& service : service_infos ) + { + if ( service.name == "public tag repository" && !ui->cbProcessPTR->isChecked() ) + { + // idhan::logging::info( "Skipping PTR because cbProcessPTR is not checked" ); + continue; + } + + auto widget { new TagServiceWidget( m_importer.get(), this ) }; + + widget->setName( service.name ); + widget->setInfo( service ); + + // ui->tagServicesLayout->addWidget( widget ); + addServiceWidget( widget ); + + connect( + this, + &HydrusImporterWidget::triggerImport, + widget, + &TagServiceWidget::startImport, + Qt::SingleShotConnection ); + connect( + this, + &HydrusImporterWidget::triggerPreImport, + widget, + &TagServiceWidget::startPreImport, + Qt::SingleShotConnection ); + } +} + +void HydrusImporterWidget::addServiceWidget( QWidget* widget ) +{ + auto* groupFrame = new QFrame( this ); + groupFrame->setFrameShape( QFrame::Box ); + groupFrame->setFrameShadow( QFrame::Plain ); + groupFrame->setLineWidth( 1 ); + groupFrame->setStyleSheet( "QFrame { border: 1px solid #444; border-radius: 6px; }" ); + + auto* groupLayout = new QVBoxLayout( groupFrame ); + groupLayout->setContentsMargins( 6, 6, 6, 6 ); + groupLayout->addWidget( widget ); + widget->setStyleSheet( "QFrame { border: none; }" ); + + ui->tagServicesLayout->addWidget( groupFrame ); +} + +void HydrusImporterWidget::parseFileRelationships() +{ + auto* widget { new FileRelationshipsWidget( m_importer.get() ) }; + + connect( + this, + &HydrusImporterWidget::triggerImport, + widget, + &FileRelationshipsWidget::startImport, + Qt::SingleShotConnection ); + connect( + this, + &HydrusImporterWidget::triggerPreImport, + widget, + &FileRelationshipsWidget::startPreImport, + Qt::SingleShotConnection ); + + addServiceWidget( widget ); +} + +void HydrusImporterWidget::parseUrls() +{ + auto* widget { new UrlServiceWidget( m_importer.get() ) }; + + connect( + this, &HydrusImporterWidget::triggerImport, widget, &UrlServiceWidget::startImport, Qt::SingleShotConnection ); + connect( + this, + &HydrusImporterWidget::triggerPreImport, + widget, + &UrlServiceWidget::startPreImport, + Qt::SingleShotConnection ); + + addServiceWidget( widget ); +} + void HydrusImporterWidget::on_hydrusFolderPath_textChanged( [[maybe_unused]] const QString& path ) { testHydrusPath(); @@ -101,39 +189,11 @@ void HydrusImporterWidget::on_parseHydrusDB_pressed() m_importer = std::make_unique< idhan::hydrus::HydrusImporter >( ui->hydrusFolderPath->text().toStdString() ); const bool has_ptr { m_importer->hasPTR() }; - - auto service_infos { m_importer->getTagServices() }; - ui->parseStatusLabel->setText( QString( "Has PTR: %1" ).arg( has_ptr ? "Yes" : "No" ) ); - for ( const auto& service : service_infos ) - { - if ( service.name == "public tag repository" && !ui->cbProcessPTR->isChecked() ) - { - // idhan::logging::info( "Skipping PTR because cbProcessPTR is not checked" ); - continue; - } - - auto widget { new TagServiceWidget( m_importer.get(), this ) }; - - widget->setName( service.name ); - widget->setInfo( service ); - - ui->tagServicesLayout->addWidget( widget ); - - connect( - this, - &HydrusImporterWidget::triggerImport, - widget, - &TagServiceWidget::startImport, - Qt::SingleShotConnection ); - connect( - this, - &HydrusImporterWidget::triggerPreImport, - widget, - &TagServiceWidget::startPreImport, - Qt::SingleShotConnection ); - } + parseTagServices(); + parseFileRelationships(); + parseUrls(); emit triggerPreImport(); diff --git a/HydrusImporter/src/gui/hydrus/TagServiceWidget.ui b/HydrusImporter/src/gui/hydrus/TagServiceWidget.ui deleted file mode 100644 index 6ca7756..0000000 --- a/HydrusImporter/src/gui/hydrus/TagServiceWidget.ui +++ /dev/null @@ -1,208 +0,0 @@ - - - TagServiceWidget - - - - 0 - 0 - 691 - 96 - - - - - 0 - 0 - - - - Form - - - - - - QFrame::Shape::StyledPanel - - - QFrame::Shadow::Raised - - - - - - - 0 - 0 - - - - - 0 - 0 - - - - Name: - - - - - - - false - - - - 0 - 0 - - - - color: rgb(0, 170, 0); - - - 0 - - - 0 - - - -1 - - - true - - - - - - - - - - - - - - - - - - 0 - 0 - - - - - 0 - 0 - - - - QFrame::Shape::StyledPanel - - - QFrame::Shadow::Raised - - - - 6 - - - - - Mappings: 0 - - - - - - - - 0 - 0 - - - - Parents/Children: 0 - - - - - - - Aliases: 0 - - - - - - - Qt::Orientation::Vertical - - - - 20 - 40 - - - - - - - - - - - QFrame::Shape::StyledPanel - - - QFrame::Shadow::Raised - - - - - - Import - - - true - - - - - - - false - - - <html><head/><body><p>Only processes mappings for files that you've obtained</p></body></html> - - - Only acquired files - - - - - - - Qt::Orientation::Vertical - - - - 20 - 40 - - - - - - - - - - - - diff --git a/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.cpp b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.cpp new file mode 100644 index 0000000..f937af6 --- /dev/null +++ b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.cpp @@ -0,0 +1,97 @@ +// +// Created by kj16609 on 11/5/25. +// +#include "FileRelationshipsWidget.hpp" + +#include "FileRelationshipsWorker.hpp" +#include "ui_FileRelationshipsWidget.h" + +FileRelationshipsWidget::FileRelationshipsWidget( idhan::hydrus::HydrusImporter* importer, QWidget* parent ) : + QWidget( parent ), + m_importer( importer ), + m_worker( new FileRelationshipsWorker( this, importer ) ), + ui( new Ui::FileRelationshipsWidget() ) +{ + ui->setupUi( this ); + + connect( + m_worker, + &FileRelationshipsWorker::processedMaxDuplicates, + this, + &FileRelationshipsWidget::processedMaxDuplicates ); + connect( + m_worker, + &FileRelationshipsWorker::processedMaxAlternatives, + this, + &FileRelationshipsWidget::processedMaxAlternatives ); + + connect( + m_worker, &FileRelationshipsWorker::processedDuplicates, this, &FileRelationshipsWidget::processedDuplicates ); + connect( + m_worker, + &FileRelationshipsWorker::processedAlternatives, + this, + &FileRelationshipsWidget::processedAlternatives ); + + connect( m_worker, &FileRelationshipsWorker::statusMessage, this, &FileRelationshipsWidget::statusMessage ); +} + +void FileRelationshipsWidget::updateText() +{ + if ( alternatives_processed > 0 || duplicates_processed > 0 ) + { + ui->alternativesCount->setText( + QString( "Alternative groups: %L1 (%L2 processed)" ) + .arg( alternatives_total ) + .arg( alternatives_processed ) ); + ui->duplicatesCount->setText( + QString( "Duplicate pairs: %L1 (%L2 processed)" ).arg( duplicates_total ).arg( duplicates_processed ) ); + } + else + { + ui->alternativesCount->setText( QString( "Alternative groups: %L1" ).arg( alternatives_total ) ); + ui->duplicatesCount->setText( QString( "Duplicate pairs: %L1" ).arg( duplicates_total ) ); + } + + ui->progressBar->setMaximum( alternatives_total + duplicates_total ); + ui->progressBar->setValue( alternatives_processed + duplicates_processed ); +} + +void FileRelationshipsWidget::startImport() +{ + if ( ui->cbShouldImport->isChecked() ) QThreadPool::globalInstance()->start( m_worker ); +} + +void FileRelationshipsWidget::startPreImport() +{ + QThreadPool::globalInstance()->start( m_worker ); +} + +void FileRelationshipsWidget::statusMessage( const QString& msg ) +{ + ui->statusLabel->setText( msg ); +} + +void FileRelationshipsWidget::processedDuplicates( const std::size_t count ) +{ + duplicates_processed = count; + updateText(); +} + +void FileRelationshipsWidget::processedMaxDuplicates( const std::size_t count ) +{ + duplicates_total = count; + updateText(); +} + +void FileRelationshipsWidget::processedAlternatives( const std::size_t count ) +{ + alternatives_processed = count; + updateText(); +} + +void FileRelationshipsWidget::processedMaxAlternatives( const std::size_t count ) +{ + alternatives_total = count; + updateText(); +} \ No newline at end of file diff --git a/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.hpp b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.hpp new file mode 100644 index 0000000..60a16e5 --- /dev/null +++ b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.hpp @@ -0,0 +1,47 @@ +// +// Created by kj16609 on 11/5/25. +// +#pragma once +#include "FileRelationshipsWorker.hpp" +#include "gui/hydrus/HydrusImporterWidget.hpp" + +namespace Ui +{ +class FileRelationshipsWidget; +} + +class FileRelationshipsWidget : public QWidget +{ + idhan::hydrus::HydrusImporter* m_importer; + + std::size_t alternatives_processed { 0 }; + std::size_t alternatives_total { 0 }; + + std::size_t duplicates_processed { 0 }; + std::size_t duplicates_total { 0 }; + FileRelationshipsWorker* m_worker; + + public: + + Q_DISABLE_COPY_MOVE( FileRelationshipsWidget ); + + explicit FileRelationshipsWidget( idhan::hydrus::HydrusImporter* importer, QWidget* parent = nullptr ); + + private: + + void updateText(); + + public slots: + void startImport(); + void startPreImport(); + void statusMessage( const QString& msg ); + + void processedDuplicates( std::size_t count ); + void processedMaxDuplicates( std::size_t count ); + void processedAlternatives( std::size_t count ); + void processedMaxAlternatives( std::size_t count ); + + private: + + Ui::FileRelationshipsWidget* ui; +}; diff --git a/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.ui b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.ui new file mode 100644 index 0000000..4b04a40 --- /dev/null +++ b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.ui @@ -0,0 +1,179 @@ + + + FileRelationshipsWidget + + + + 0 + 0 + 907 + 98 + + + + + 0 + 0 + + + + Form + + + + + + 0 + + + + + 0 + + + + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + 0 + 0 + + + + + 0 + 0 + + + + Name: Duplicate & Alternative relationships +Type:File Relationships + + + + + + + + 0 + 0 + + + + + + + + + + + + + 0 + + + + + Alternative Groups: 0 + + + + + + + Duplicate Pairs: 0 + + + + + + + Qt::Orientation::Vertical + + + + 0 + 0 + + + + + + + + + + + + 0 + + + + + 0 + + + + + false + + + + 0 + 0 + + + + color: rgb(0, 170, 0); + + + 0 + + + 0 + + + -1 + + + true + + + + + + + Import + + + true + + + + + + + + + + + + + + diff --git a/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.cpp b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.cpp new file mode 100644 index 0000000..f68ece4 --- /dev/null +++ b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.cpp @@ -0,0 +1,200 @@ +// +// Created by kj16609 on 11/5/25. +// +#include "FileRelationshipsWorker.hpp" + +#include + +#include "sqlitehelper/Query.hpp" +#include "sqlitehelper/TransactionBaseCoro.hpp" + +FileRelationshipsWorker::FileRelationshipsWorker( QObject* parent, idhan::hydrus::HydrusImporter* importer ) : + QObject( parent ), + QRunnable(), + m_importer( importer ) +{ + this->setAutoDelete( false ); +} + +void FileRelationshipsWorker::preprocess() +{ + idhan::hydrus::TransactionBaseCoro client_tr { m_importer->client_db }; + + idhan::hydrus::Query< int, int > duplicate_file_members { + client_tr, + "SELECT hash_id, king_hash_id FROM duplicate_file_members dfm JOIN duplicate_files df ON dfm.media_id = df.media_id WHERE hash_id != king_hash_id" + }; + + std::size_t duplicate_file_members_counter { 0 }; + + for ( [[maybe_unused]] const auto& [ media_id, hash_id ] : duplicate_file_members ) + { + duplicate_file_members_counter++; + + if ( duplicate_file_members_counter % 1000 == 0 ) emit processedMaxDuplicates( duplicate_file_members_counter ); + } + + emit processedMaxDuplicates( duplicate_file_members_counter ); + + idhan::hydrus::Query< int, int > alternative_file_members { + client_tr, "SELECT * FROM alternate_file_group_members JOIN duplicate_file_members USING (media_id)" + }; + std::size_t alternative_file_members_counter { 0 }; + + for ( [[maybe_unused]] const auto& [ media_id, hash_id ] : alternative_file_members ) + { + alternative_file_members_counter++; + + if ( alternative_file_members_counter % 1000 == 0 ) + emit processedMaxAlternatives( alternative_file_members_counter ); + } + + emit processedMaxAlternatives( alternative_file_members_counter ); +} + +void FileRelationshipsWorker::process() +{ + idhan::hydrus::TransactionBaseCoro client_tr { m_importer->client_db }; + + using MediaID = std::uint32_t; + using HashID = std::uint32_t; + using KingID = std::uint32_t; + + std::vector< HashID > hash_ids {}; + + std::unordered_map< HashID, idhan::RecordID > record_map {}; + + emit statusMessage( "Started" ); + + auto flushHashIDs = [ &, this ]() + { + emit statusMessage( "Mapping Hydrus IDs to IDHAN IDs" ); + std::ranges::sort( hash_ids ); + std::ranges::unique( hash_ids ); + + std::ranges::remove_if( + hash_ids, [ &record_map ]( const HashID hash_id ) -> bool { return record_map.contains( hash_id ); } ); + + if ( hash_ids.empty() ) return; + + const auto batch_map { m_importer->mapHydrusRecords( hash_ids ) }; + // merge the new map into the existing one + + for ( const auto& [ hy_hash_id, idhan_hash_id ] : batch_map ) + record_map.insert_or_assign( hy_hash_id, idhan_hash_id ); + + hash_ids.clear(); + }; + + idhan::hydrus::Query< HashID, KingID > duplicate_files { + client_tr, + "SELECT hash_id, king_hash_id FROM duplicate_file_members dfm JOIN duplicate_files df ON dfm.media_id = df.media_id WHERE hash_id != king_hash_id" + }; + + std::vector< std::pair< HashID, KingID > > pairs {}; + + auto& client { idhan::IDHANClient::instance() }; + + std::size_t processed_count { 0 }; + + auto flushPairs = [ &, this ]() + { + flushHashIDs(); + + emit statusMessage( "Setting duplicates for batch" ); + + std::vector< std::pair< idhan::RecordID, idhan::RecordID > > idhan_pairs {}; + + for ( const auto& [ hy_hash_id, hy_king_id ] : pairs ) + { + const auto idhan_hash_id { record_map.at( hy_hash_id ) }; + const auto idhan_king_id { record_map.at( hy_king_id ) }; + + idhan_pairs.emplace_back( idhan_hash_id, idhan_king_id ); + + // auto future = client.setDuplicates( pairs ); + // future.waitForFinished(); + } + + auto future { client.setDuplicates( idhan_pairs ) }; + future.waitForFinished(); + + processed_count += pairs.size(); + emit processedDuplicates( processed_count ); + + pairs.clear(); + }; + + for ( const auto& [ hash_id, king_id ] : duplicate_files ) + { + if ( hash_id == king_id ) continue; + + if ( !record_map.contains( hash_id ) ) hash_ids.push_back( hash_id ); + if ( !record_map.contains( king_id ) ) hash_ids.push_back( king_id ); + + pairs.emplace_back( std::make_pair( hash_id, king_id ) ); + + if ( pairs.size() >= 100 ) + { + flushPairs(); + emit statusMessage( "Getting additional rows to process" ); + } + } + + flushPairs(); + + idhan::hydrus::Query< HashID, MediaID > alternative_files { + client_tr, + "SELECT hash_id, alternates_group_id FROM alternate_file_group_members JOIN duplicate_file_members USING (media_id);" + }; + + using GroupID = std::uint32_t; + + std::unordered_map< GroupID, std::vector< idhan::hydrus::HashID > > alternative_map {}; + + emit statusMessage( "Mapping alternative hashes to IDHAN" ); + + for ( const auto& [ hash_id, alternative_group_id ] : alternative_files ) + { + hash_ids.push_back( hash_id ); + + if ( hash_ids.size() >= 64 ) flushHashIDs(); + + if ( auto itter = alternative_map.find( alternative_group_id ); itter != alternative_map.end() ) + itter->second.push_back( hash_id ); + else + alternative_map.emplace( alternative_group_id, std::vector< idhan::hydrus::HashID > { hash_id } ); + } + + flushHashIDs(); + + emit statusMessage( "Setting alternatives for groups" ); + + std::size_t alternative_count { 0 }; + + for ( const auto hy_hashes : alternative_map | std::views::values ) + { + std::vector< idhan::RecordID > record_ids {}; + for ( const auto& hy_hash : hy_hashes ) record_ids.emplace_back( record_map.at( hy_hash ) ); + + alternative_count += record_ids.size(); + + auto future = client.setAlternativeGroups( record_ids ); + future.waitForFinished(); + emit processedAlternatives( alternative_count ); + } + + emit statusMessage( "Finished" ); +} + +void FileRelationshipsWorker::run() +{ + if ( !m_preprocessed ) + { + m_preprocessed = true; + preprocess(); + return; + } + + process(); +} diff --git a/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.hpp b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.hpp new file mode 100644 index 0000000..8460cf8 --- /dev/null +++ b/HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.hpp @@ -0,0 +1,32 @@ +// +// Created by kj16609 on 11/5/25. +// +#pragma once + +#include + +#include "HydrusImporter.hpp" + +class FileRelationshipsWorker : public QObject, public QRunnable +{ + Q_OBJECT + + idhan::hydrus::HydrusImporter* m_importer; + bool m_preprocessed { false }; + + signals: + + void processedMaxDuplicates( std::size_t counter ); + void processedMaxAlternatives( std::size_t counter ); + void statusMessage( const QString& message ); + void processedDuplicates( std::size_t ); + void processedAlternatives( std::size_t ); + + public: + + FileRelationshipsWorker( QObject* parent, idhan::hydrus::HydrusImporter* importer ); + void preprocess(); + void process(); + + void run() override; +}; diff --git a/HydrusImporter/src/gui/hydrus/TagServiceWidget.cpp b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.cpp similarity index 96% rename from HydrusImporter/src/gui/hydrus/TagServiceWidget.cpp rename to HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.cpp index c0ec3c9..9be551b 100644 --- a/HydrusImporter/src/gui/hydrus/TagServiceWidget.cpp +++ b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.cpp @@ -53,7 +53,7 @@ TagServiceWidget::~TagServiceWidget() void TagServiceWidget::setName( const QString& name ) { m_name = name; - ui->name->setText( QString( "Name: %L1" ).arg( name ) ); + ui->name->setText( QString( "Name: %L1\nType: Tag Service" ).arg( name ) ); } void TagServiceWidget::recordMappingProcessed( std::size_t count ) @@ -113,6 +113,12 @@ void TagServiceWidget::updateTime() const std::size_t to_process { m_info.num_mappings + m_info.num_parents + m_info.num_aliases }; const std::size_t total_processed { mappings_processed + parents_processed + aliases_processed }; + if ( total_processed == 0 ) + { + ui->statusLabel->setText( "Ready!" ); + return; + } + // const bool over_limit { to_process > std::numeric_limits< int >::max() }; // const std::size_t multip { over_limit ? 16 : 1 }; @@ -182,7 +188,7 @@ void TagServiceWidget::processedMappings( std::size_t count, std::size_t record_ QLocale locale { QLocale::English, QLocale::UnitedStates }; locale.setNumberOptions( QLocale::DefaultNumberOptions ); ui->mappingsCount->setText( - QString( "Mappings: %L1 (%L2 processed)\nRecords: (%L3 Records)" ) + QString( "Mappings: %L1 (%L2 processed)\nRecords: (%L3 processed)" ) .arg( m_info.num_mappings ) .arg( mappings_processed ) .arg( records_processed ) ); @@ -222,7 +228,7 @@ void TagServiceWidget::setMaxMappings( std::size_t count ) m_info.num_mappings = count; if ( mappings_processed > 0 ) ui->mappingsCount->setText( - QString( "Mappings: %L1 (%L2 processed)\nRecords: (%L3 Records)" ) + QString( "Mappings: %L1 (%L2 processed)\nRecords: (%L3 processed)" ) .arg( m_info.num_mappings ) .arg( mappings_processed ) .arg( records_processed ) ); diff --git a/HydrusImporter/src/gui/hydrus/TagServiceWidget.hpp b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.hpp similarity index 98% rename from HydrusImporter/src/gui/hydrus/TagServiceWidget.hpp rename to HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.hpp index 7a13834..0fcc295 100644 --- a/HydrusImporter/src/gui/hydrus/TagServiceWidget.hpp +++ b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.hpp @@ -9,7 +9,8 @@ #include #include "HydrusImporter.hpp" -#include "HydrusImporterWidget.hpp" + +class TagServiceWorker; namespace Ui { diff --git a/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.ui b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.ui new file mode 100644 index 0000000..a14a5ae --- /dev/null +++ b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.ui @@ -0,0 +1,213 @@ + + + TagServiceWidget + + + + 0 + 0 + 605 + 104 + + + + + 0 + 0 + + + + Form + + + + + + 0 + + + + + 0 + + + + + 0 + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + 0 + 0 + + + + + 0 + 0 + + + + Name: + + + + + + + + 0 + 0 + + + + Preprocessing + + + + + + + + + 0 + + + + + 0 + + + QLayout::SizeConstraint::SetDefaultConstraint + + + + + Mappings: 0 + + + + + + + + 0 + 0 + + + + Aliases: 0 + + + + + + + + + 0 + + + + + + 0 + 0 + + + + Parents/Children: 0 + + + + + + + false + + + + 0 + 0 + + + + <html><head/><body><p>Only processes mappings for files that you've obtained</p></body></html> + + + Only acquired files + + + + + + + + + + + + + 0 + + + + + false + + + + 0 + 0 + + + + color: rgb(0, 170, 0); + + + 0 + + + 0 + + + -1 + + + true + + + + + + + Import + + + true + + + + + + + + + + + + diff --git a/HydrusImporter/src/gui/hydrus/TagServiceWorker.cpp b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWorker.cpp similarity index 99% rename from HydrusImporter/src/gui/hydrus/TagServiceWorker.cpp rename to HydrusImporter/src/gui/hydrus/tag_service/TagServiceWorker.cpp index 333986c..adc4689 100644 --- a/HydrusImporter/src/gui/hydrus/TagServiceWorker.cpp +++ b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWorker.cpp @@ -4,12 +4,13 @@ #include "TagServiceWorker.hpp" +#include + #include #include #include -#include "moc_TagServiceWorker.cpp" #include "sqlitehelper/Query.hpp" #include "sqlitehelper/Transaction.hpp" #include "sqlitehelper/TransactionBaseCoro.hpp" diff --git a/HydrusImporter/src/gui/hydrus/TagServiceWorker.hpp b/HydrusImporter/src/gui/hydrus/tag_service/TagServiceWorker.hpp similarity index 100% rename from HydrusImporter/src/gui/hydrus/TagServiceWorker.hpp rename to HydrusImporter/src/gui/hydrus/tag_service/TagServiceWorker.hpp diff --git a/HydrusImporter/src/gui/main/MainWindow.cpp b/HydrusImporter/src/gui/main/MainWindow.cpp index 61eec37..8b18735 100644 --- a/HydrusImporter/src/gui/main/MainWindow.cpp +++ b/HydrusImporter/src/gui/main/MainWindow.cpp @@ -5,14 +5,16 @@ #include "MainWindow.hpp" -#include +#include + #include #include -#include "../hydrus/HydrusImporterWidget.hpp" #include "NET_CONSTANTS.hpp" #include "SettingsDialog.hpp" +#include "gui/hydrus/HydrusImporterWidget.hpp" +#include "gui/hydrus/tag_service/TagServiceWidget.hpp" #include "ui_MainWindow.h" MainWindow::MainWindow( QWidget* parent ) : @@ -96,10 +98,11 @@ void MainWindow::checkHeartbeat() { const auto result { watcher->result() }; - ui->statusbar->showMessage( QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" ) - .arg( result.server.str ) - .arg( result.build_type ) - .arg( result.commit ) ); + ui->statusbar->showMessage( + QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" ) + .arg( result.server.str ) + .arg( result.build_type ) + .arg( result.commit ) ); } catch ( std::exception& e ) { diff --git a/HydrusImporter/src/gui/main/SettingsDialog.cpp b/HydrusImporter/src/gui/main/SettingsDialog.cpp index e02a846..de8f979 100644 --- a/HydrusImporter/src/gui/main/SettingsDialog.cpp +++ b/HydrusImporter/src/gui/main/SettingsDialog.cpp @@ -4,6 +4,8 @@ #include "SettingsDialog.hpp" +#include + #include #include "idhan/IDHANClient.hpp" @@ -41,10 +43,11 @@ void SettingsDialog::on_testConnection_pressed() { const auto result { watcher->result() }; - ui->networkSettingsLabel->setText( QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" ) - .arg( result.server.str ) - .arg( result.build_type ) - .arg( result.commit ) ); + ui->networkSettingsLabel->setText( + QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" ) + .arg( result.server.str ) + .arg( result.build_type ) + .arg( result.commit ) ); } catch ( std::exception& e ) { diff --git a/IDHANClient/include/idhan/IDHANClient.hpp b/IDHANClient/include/idhan/IDHANClient.hpp index 5bfcade..324b8f2 100644 --- a/IDHANClient/include/idhan/IDHANClient.hpp +++ b/IDHANClient/include/idhan/IDHANClient.hpp @@ -143,6 +143,18 @@ class IDHANClient TagDomainID tag_domain_id, std::vector< std::vector< std::pair< std::string, std::string > > >&& tag_sets ); + // File relationships + QFuture< void > setAlternativeGroups( std::vector< RecordID >& record_ids ); + + QFuture< void > setDuplicates( RecordID worse_duplicate, RecordID better_duplicate ); + + /** + * + * @param pairs Pairs of ids in a (worse_id, better_id) format + * @return + */ + QFuture< void > setDuplicates( const std::vector< std::pair< RecordID, RecordID > >& pairs ); + /** * @brief Creates a parent/child relationship between two tags * @param parent_id @@ -238,7 +250,7 @@ class IDHANClient std::uint16_t ratio, bool readonly ); - QFuture< void > addUrls( RecordID record_id, std::vector< std::string >& urls ); + QFuture< void > addUrls( RecordID record_id, const std::vector< std::string >& urls ); inline QFuture< void > addUrl( const RecordID record_id, std::string url ) { diff --git a/IDHANClient/src/records/relationships/setAlternatives.cpp b/IDHANClient/src/records/relationships/setAlternatives.cpp new file mode 100644 index 0000000..a7ba158 --- /dev/null +++ b/IDHANClient/src/records/relationships/setAlternatives.cpp @@ -0,0 +1,46 @@ +// +// Created by kj16609 on 11/10/25. +// +#include +#include +#include + +#include "IDHANTypes.hpp" +#include "idhan/IDHANClient.hpp" + +namespace idhan +{ + +QFuture< void > IDHANClient::setAlternativeGroups( std::vector< RecordID >& record_ids ) +{ + if ( record_ids.empty() ) return QtFuture::makeReadyVoidFuture(); + + QJsonArray record_array {}; + + for ( const auto& record : record_ids ) record_array.append( record ); + + auto promise { std::make_shared< QPromise< void > >() }; + + auto handleResponse = [ promise ]( QNetworkReply* reply ) -> void + { + const auto data = reply->readAll(); + if ( !reply->isFinished() ) + { + logging::info( "Failed to read response" ); + throw std::runtime_error( "Failed to read response" ); + } + + promise->finish(); + reply->deleteLater(); + }; + + QJsonDocument doc {}; + doc.setArray( record_array ); + + sendClientPost( + std::move( doc ), "/relationships/alternatives/add", handleResponse, defaultErrorHandler( promise ) ); + + return promise->future(); +} + +} // namespace idhan \ No newline at end of file diff --git a/IDHANClient/src/records/relationships/setDuplicates.cpp b/IDHANClient/src/records/relationships/setDuplicates.cpp new file mode 100644 index 0000000..3662219 --- /dev/null +++ b/IDHANClient/src/records/relationships/setDuplicates.cpp @@ -0,0 +1,57 @@ +// +// Created by kj16609 on 11/5/25. +// + +#include +#include +#include + +#include "IDHANTypes.hpp" +#include "idhan/IDHANClient.hpp" + +namespace idhan +{ + +QFuture< void > IDHANClient::setDuplicates( const RecordID worse_duplicate, const RecordID better_duplicate ) +{ + const std::vector< std::pair< RecordID, RecordID > > duplicates { { worse_duplicate, better_duplicate } }; + return setDuplicates( duplicates ); +} + +QFuture< void > IDHANClient::setDuplicates( const std::vector< std::pair< RecordID, RecordID > >& pairs ) +{ + QJsonArray json_pairs {}; + + for ( const auto& [ worse_id, better_id ] : pairs ) + { + QJsonObject json_pair {}; + json_pair[ "worse_id" ] = worse_id; + json_pair[ "better_id" ] = better_id; + + json_pairs.append( json_pair ); + } + + auto promise { std::make_shared< QPromise< void > >() }; + + auto handleResponse = [ promise ]( QNetworkReply* response ) -> void + { + const auto data = response->readAll(); + if ( !response->isFinished() ) + { + logging::info( "Failed to read response" ); + throw std::runtime_error( "Failed to read response" ); + } + + promise->finish(); + response->deleteLater(); + }; + + QJsonDocument doc {}; + doc.setArray( std::move( json_pairs ) ); + + sendClientPost( std::move( doc ), "/relationships/duplicates/add", handleResponse, defaultErrorHandler( promise ) ); + + return promise->future(); +} + +} // namespace idhan \ No newline at end of file diff --git a/IDHANMigration/src/130-alternative_files.sql b/IDHANMigration/src/130-alternative_files.sql index 9739c63..1362bf4 100644 --- a/IDHANMigration/src/130-alternative_files.sql +++ b/IDHANMigration/src/130-alternative_files.sql @@ -5,12 +5,49 @@ CREATE TABLE alternative_groups CREATE TABLE alternative_group_members ( - group_id INTEGER REFERENCES alternative_groups (group_id), - record_id INTEGER REFERENCES records (record_id) + group_id INTEGER REFERENCES alternative_groups (group_id) NOT NULL, + record_id INTEGER REFERENCES records (record_id) UNIQUE NOT NULL, + UNIQUE (group_id, record_id) ); -CREATE TABLE alternative_pairs +CREATE INDEX ON alternative_group_members (group_id); + +CREATE TABLE duplicate_pairs ( - worse_record_id INTEGER REFERENCES records (record_id), - better_record_id INTEGER REFERENCES records (record_id) -); \ No newline at end of file + worse_record_id INTEGER REFERENCES records (record_id) UNIQUE NOT NULL, + better_record_id INTEGER REFERENCES records (record_id) NOT NULL, + CHECK (worse_record_id != better_record_id) +); + +CREATE INDEX ON duplicate_pairs (better_record_id); + +CREATE OR REPLACE FUNCTION insert_duplicate_pair( + worse INTEGER, + better INTEGER +) RETURNS VOID AS +$$ +BEGIN + + IF (EXISTS(SELECT 1 FROM duplicate_pairs WHERE worse_record_id = worse AND better_record_id = better)) THEN + RETURN; + END IF; + + -- check that the worse id isn't being added again + IF (EXISTS(SELECT 1 FROM duplicate_pairs WHERE worse_record_id = worse)) THEN + RAISE EXCEPTION 'Can''t insert an already inserted worse record'; + END IF; + + -- Check that the worse record wouldn't make a cyclic chain + IF (EXISTS(SELECT 1 FROM duplicate_pairs WHERE better_record_id = worse AND worse_record_id = better)) THEN + RAISE EXCEPTION 'Inserting this duplicate pair would result in a cyclic chain'; + end if; + + INSERT INTO duplicate_pairs (worse_record_id, better_record_id) + VALUES (worse, better); + + UPDATE duplicate_pairs + SET better_record_id = better + WHERE better_record_id = worse; +END; + +$$ LANGUAGE plpgsql; diff --git a/IDHANServer/src/ServerContext.cpp b/IDHANServer/src/ServerContext.cpp index 860980c..f7e11e8 100644 --- a/IDHANServer/src/ServerContext.cpp +++ b/IDHANServer/src/ServerContext.cpp @@ -58,8 +58,14 @@ void ServerContext::setupCORSSupport() const } ); drogon::app().registerPostHandlingAdvice( - []( [[maybe_unused]] const drogon::HttpRequestPtr& request, const drogon::HttpResponsePtr& response ) - { addCORSHeaders( response ); } ); + [ this ]( [[maybe_unused]] const drogon::HttpRequestPtr& request, const drogon::HttpResponsePtr& response ) + { + if ( args.testmode ) + log::info( "Finished Handling query: {}:{}", request->getMethodString(), request->getPath() ); + else + log::debug( "Finished Handling query: {}:{}", request->getMethodString(), request->getPath() ); + addCORSHeaders( response ); + } ); } void exceptionHandler( const std::exception& e, const drogon::HttpRequestPtr& request, ResponseFunction&& callback ) diff --git a/IDHANServer/src/api/FileRelationshipsAPI.hpp b/IDHANServer/src/api/FileRelationshipsAPI.hpp new file mode 100644 index 0000000..35e2f58 --- /dev/null +++ b/IDHANServer/src/api/FileRelationshipsAPI.hpp @@ -0,0 +1,26 @@ +// +// Created by kj16609 on 11/5/25. +// +#pragma once +#include "drogon/HttpController.h" + +namespace idhan::api +{ + +class FileRelationshipsAPI : public drogon::HttpController< FileRelationshipsAPI > +{ + drogon::Task< drogon::HttpResponsePtr > setBetterDuplicate( drogon::HttpRequestPtr request ); + drogon::Task< drogon::HttpResponsePtr > addAlternative( drogon::HttpRequestPtr request ); + + public: + + METHOD_LIST_BEGIN + + ADD_METHOD_TO( FileRelationshipsAPI::setBetterDuplicate, "/relationships/duplicates/add" ); + + ADD_METHOD_TO( FileRelationshipsAPI::addAlternative, "/relationships/alternatives/add" ); + + METHOD_LIST_END +}; + +} // namespace idhan::api \ No newline at end of file diff --git a/IDHANServer/src/api/relationships/addAlternative.cpp b/IDHANServer/src/api/relationships/addAlternative.cpp new file mode 100644 index 0000000..6d19958 --- /dev/null +++ b/IDHANServer/src/api/relationships/addAlternative.cpp @@ -0,0 +1,124 @@ +// +// Created by kj16609 on 11/5/25. +// + +#include "IDHANTypes.hpp" +#include "api/FileRelationshipsAPI.hpp" +#include "api/helpers/createBadRequest.hpp" +#include "db/drogonArrayBind.hpp" + +namespace idhan::api +{ + +using GroupID = idhan::Integer; + +drogon::Task< GroupID > createNewGroup( DbClientPtr db ) +{ + const auto group_result { + co_await db->execSqlCoro( "INSERT INTO alternative_groups DEFAULT VALUES RETURNING group_id" ) + }; + + const auto group_id { group_result[ 0 ][ 0 ].as< GroupID >() }; + co_return group_id; +} + +drogon::Task<> addItemsToNewGroup( std::vector< RecordID > record_ids, DbClientPtr db ) +{ + const auto group_id { co_await createNewGroup( db ) }; + co_await db->execSqlCoro( + "INSERT INTO alternative_group_members (group_id, record_id) VALUES ($1, UNNEST($2::" RECORD_PG_TYPE_NAME + "[]))", + group_id, + std::move( record_ids ) ); +} + +drogon::Task<> addItemsToExistingGroup( + std::vector< RecordID > record_ids, + const std::vector< GroupID > group_ids, + drogon::orm::DbClientPtr db ) +{ + const auto group_id { group_ids[ 0 ] }; + + co_await db->execSqlCoro( + "INSERT INTO alternative_group_members (group_id, record_id) VALUES ($1, UNNEST($2::" RECORD_PG_TYPE_NAME + "[])) ON CONFLICT (group_id, record_id) DO NOTHING", + group_id, + std::move( record_ids ) ); +} + +drogon::Task<> addItemsToExistingGroupsMerge( + std::vector< RecordID > record_ids, + std::vector< GroupID > group_ids, + drogon::orm::DbClientPtr db ) +{ + const auto group_id { co_await createNewGroup( db ) }; + + co_await db->execSqlCoro( + "UPDATE alternative_group_members SET group_id = $1 WHERE group_id = ANY($2)", + group_id, + std::forward< std::vector< GroupID > >( group_ids ) ); + + co_await db->execSqlCoro( + "INSERT INTO alternative_group_members (group_id, record_id) VALUES ($1, UNNEST($2::" RECORD_PG_TYPE_NAME + "[])) ON CONFLICT (group_id, record_id) DO NOTHING", + group_id, + std::forward< std::vector< RecordID > >( record_ids ) ); + + co_await db->execSqlCoro( + "DELETE FROM alternative_groups WHERE group_id = ANY($1)", + std::forward< std::vector< GroupID > >( group_ids ) ); +} + +drogon::Task< drogon::HttpResponsePtr > FileRelationshipsAPI::addAlternative( drogon::HttpRequestPtr request ) +{ + auto db { drogon::app().getDbClient() }; + + const auto json_ptr { request->getJsonObject() }; + if ( !json_ptr ) co_return createBadRequest( "Expected json body" ); + + const auto& json { *json_ptr }; + + if ( !json.isArray() ) co_return createBadRequest( "Expected json array of integers" ); + + std::vector< RecordID > record_ids {}; + + for ( const auto& id : json ) + { + record_ids.emplace_back( id.as< RecordID >() ); + } + + const auto group_search { co_await db->execSqlCoro( + "SELECT DISTINCT group_id FROM alternative_group_members WHERE record_id = ANY($1)", + std::forward< std::vector< RecordID > >( record_ids ) ) }; + + std::vector< GroupID > group_ids {}; + + for ( const auto& id : group_search ) + { + group_ids.emplace_back( id[ 0 ].as< GroupID >() ); + } + + std::ranges::sort( group_ids ); + std::ranges::unique( group_ids ); + + const bool no_existing_groups { group_ids.empty() }; + const bool existing_groups { not no_existing_groups }; + const bool multiple_groups { not no_existing_groups && group_ids.size() > 1 }; + + if ( no_existing_groups ) + { + co_await addItemsToNewGroup( record_ids, db ); + } + else if ( existing_groups && !multiple_groups ) + { + co_await addItemsToExistingGroup( record_ids, group_ids, db ); + } + else if ( multiple_groups ) + { + co_await addItemsToExistingGroupsMerge( record_ids, group_ids, db ); + } + + co_return drogon::HttpResponse::newHttpJsonResponse( {} ); +} + +} // namespace idhan::api diff --git a/IDHANServer/src/api/relationships/setBetterDuplicate.cpp b/IDHANServer/src/api/relationships/setBetterDuplicate.cpp new file mode 100644 index 0000000..f621df9 --- /dev/null +++ b/IDHANServer/src/api/relationships/setBetterDuplicate.cpp @@ -0,0 +1,61 @@ +// +// Created by kj16609 on 11/5/25. +// + +#include "IDHANTypes.hpp" +#include "api/FileRelationshipsAPI.hpp" +#include "api/helpers/createBadRequest.hpp" + +namespace idhan::api +{ + +drogon::Task< drogon::HttpResponsePtr > setBetterDuplicateMultiple( const Json::Value json ) +{ + auto db { drogon::app().getDbClient() }; + + for ( const auto& object : json ) + { + if ( !object.isMember( "worse_id" ) || !object.isMember( "better_id" ) ) + co_return createBadRequest( "Expected object to have `worse_id` and `better_id`" ); + + if ( !object[ "worse_id" ].isUInt() || !object[ "better_id" ].isUInt() ) + co_return createBadRequest( "Expected `worse_id` and `better_id` to be unsigned integers" ); + + const RecordID worse_id { object[ "worse_id" ].as< RecordID >() }; + const RecordID better_id { object[ "better_id" ].as< RecordID >() }; + + co_await db->execSqlCoro( "SELECT insert_duplicate_pair($1, $2)", worse_id, better_id ); + } + + co_return drogon::HttpResponse::newHttpJsonResponse( {} ); +} + +drogon::Task< drogon::HttpResponsePtr > setBetterDuplicateSingle( const Json::Value json ) +{ + auto db { drogon::app().getDbClient() }; + + if ( !json[ "worse_id" ].isUInt() || !json[ "better_id" ].isUInt() ) + co_return createBadRequest( "Expected `worse_id` and `better_id` to be unsigned integers" ); + + const RecordID worse_id { json[ "worse_id" ].as< RecordID >() }; + const RecordID better_id { json[ "better_id" ].as< RecordID >() }; + + co_await db->execSqlCoro( "SELECT insert_duplicate_pair($1, $2)", worse_id, better_id ); + + co_return drogon::HttpResponse::newHttpJsonResponse( {} ); +} + +drogon::Task< drogon::HttpResponsePtr > FileRelationshipsAPI::setBetterDuplicate( const drogon::HttpRequestPtr request ) +{ + const auto json_ptr { request->getJsonObject() }; + if ( !json_ptr ) co_return createBadRequest( "Expected json body" ); + const auto& json { *json_ptr }; + + if ( json.isArray() ) co_return co_await setBetterDuplicateMultiple( json ); + if ( json.isObject() ) co_return co_await setBetterDuplicateSingle( json ); + + co_return createBadRequest( + "Expected json body of either array of objects, or a single object. Objects must have `better_id` and `worse_id`" ); +} + +} // namespace idhan::api \ No newline at end of file