Fixes no EOF being sent for ffmpeg read function

- Implements video metadata & thumbnailer
- Implements PSD metadata & thumbnailer

.github/workflows/docker-build-dev.yml

@@ -0,0 +1,54 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - dev
+
+env:
+  REGISTRY: git.futuregadgetlabs.net
+  IMAGE_NAME: kj16609/idhan
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    if: github.server_url == 'https://git.futuregadgetlabs.net'
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: git.futuregadgetlabs.net
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=tag
+            type=raw,value=dev,enable={{is_default_branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max

Dockerfile

@@ -2,7 +2,8 @@
 # Stage 1: Build environment
 FROM ubuntu:24.04 AS builder
 
-RUN apt-get update && DEBIAN_FRONTNED=noninteractive apt-get install -y \
+RUN apt-get update && \
+    DEBIAN_FRONTNED=noninteractive apt-get install -y \
     build-essential \
     cmake \
     git \
@@ -17,7 +18,11 @@ RUN apt-get update && DEBIAN_FRONTNED=noninteractive apt-get install -y \
     libqt6core6 \
     libqt6multimedia6 \
     libjsoncpp-dev \
-    libvips-dev
+    libvips-dev \
+    libavcodec-dev \
+    libavcodec-extra \
+    libavfilter-dev \
+    libavutil-dev
 
 # Set C++23 capable compiler as default
 RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100 && \
@@ -50,10 +55,9 @@ RUN --mount=type=cache,target=/root/.ccache \
 # Stage 2: Runtime environment
 FROM ubuntu:24.04
 
-RUN apt-get update
-
 # Install runtime dependencies only
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
     # Qt6 runtime libraries
     libqt6core6 \
     libqt6multimedia6 \
@@ -68,7 +72,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \
     uuid-runtime \
     zlib1g \
     libssl3 \
-    libc-ares2
+    libc-ares2 \
+    ffmpeg
 
 # Cleanup
 RUN apt-get clean

HydrusImporter/CMakeLists.txt

@@ -5,6 +5,10 @@ find_package(Qt6 REQUIRED COMPONENTS Core Network Concurrent Widgets)
 AddFGLExecutable(HydrusImporter ${CMAKE_CURRENT_SOURCE_DIR}/src)
 target_link_libraries(HydrusImporter PUBLIC Qt6::Core Qt6::Concurrent IDHANClient sqlite3 spdlog Qt6::Widgets)
 
+file(GLOB_RECURSE UI_FILES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/**.ui)
+
+target_sources(HydrusImporter PRIVATE ${UI_FILES})
+
 if (DEFINED IMPORTER_TESTS AND IMPORTER_TESTS EQUAL 1)
 	target_compile_definitions(HydrusImporter PUBLIC IMPORTER_TESTS=1)
 endif ()

HydrusImporter/src/HydrusImporter.cpp

@@ -50,15 +50,99 @@ HydrusImporter::~HydrusImporter()
 	sqlite3_close_v2( mappings_db );
 }
 
+std::unordered_map< HashID, RecordID > HydrusImporter::mapHydrusRecords( std::vector< HashID > hash_ids ) const
+{
+	idhan::hydrus::TransactionBase master_tr { master_db };
+	if ( hash_ids.empty() ) return {};
+
+	std::ranges::sort( hash_ids );
+	std::ranges::unique( hash_ids );
+
+	std::unordered_map< std::uint32_t, std::string > hashes_map {};
+	hashes_map.reserve( hash_ids.size() );
+
+	for ( const auto& hash_id : hash_ids )
+	{
+		master_tr << "SELECT hex(hash) FROM hashes WHERE hash_id = $1" << hash_id >>
+			[ & ]( const std::string_view hash_i )
+		{
+			if ( hash_i.size() != ( 256 / 8 * 2 ) )
+			{
+				return;
+			}
+			hashes_map.emplace( hash_id, hash_i );
+		};
+	}
+
+	std::vector< std::string > hashes {};
+	hashes.reserve( hash_ids.size() );
+	for ( const auto& hash : hashes_map | std::views::values ) hashes.emplace_back( hash );
+
+	auto& client { idhan::IDHANClient::instance() };
+	auto created_records { client.createRecords( hashes ) };
+
+	created_records.waitForFinished();
+
+	const auto created_records_result { created_records.result<>() };
+	if ( created_records_result.size() != hashes.size() ) throw std::runtime_error( "Failed to create records" );
+
+	std::unordered_map< std::string, RecordID > record_map {};
+
+	for ( const auto& [ record_id, hash ] : std::ranges::views::zip( created_records_result, hashes ) )
+	{
+		record_map.emplace( hash, record_id );
+	}
+
+	std::unordered_map< HashID, RecordID > record_id_map {};
+
+	record_id_map.reserve( hash_ids.size() );
+
+	for ( const auto& [ hy_id, hash ] : hashes_map )
+	{
+		const auto record_id { record_map.at( hash ) };
+		record_id_map.emplace( hy_id, record_id );
+	}
+
+	return record_id_map;
+}
+
+RecordID HydrusImporter::getRecordIDFromHyID( const HashID hash_id )
+{
+	idhan::hydrus::TransactionBaseCoro client_tr { client_db };
+	idhan::hydrus::Query< std::string_view > query {
+		client_tr, "SELECT hex(hash) FROM hashes WHERE hash_id = $1", hash_id
+	};
+
+	auto& client { IDHANClient::instance() };
+
+	std::vector< std::string > hashes {};
+
+	for ( const auto& [ hash ] : query )
+	{
+		hashes.emplace_back( hash );
+	}
+
+	auto future { client.getRecordID( hashes.front() ) };
+
+	future.waitForFinished();
+
+	const auto result { future.result<>() };
+	if ( !result ) throw std::runtime_error( "Failed to get record from client" );
+
+	return result.value();
+}
+
 bool HydrusImporter::hasPTR() const
 {
 	bool exists { false };
 
 	TransactionBaseCoro client_tr { client_db };
 
-	Query< std::size_t, std::string_view > query { client_tr,
-		                                           "SELECT service_id, name FROM services WHERE service_type = $1",
-		                                           static_cast< int >( hy_constants::ServiceTypes::PTR_SERVICE ) };
+	Query< std::size_t, std::string_view > query {
+		client_tr,
+		"SELECT service_id, name FROM services WHERE service_type = $1",
+		static_cast< int >( hy_constants::ServiceTypes::PTR_SERVICE )
+	};
 
 	for ( [[maybe_unused]] const auto& [ serviae_id, name ] : query )
 	{

HydrusImporter/src/HydrusImporter.hpp

@@ -27,6 +27,8 @@ struct ServiceInfo
 	ServiceInfo() = default;
 };
 
+using HashID = std::uint32_t;
+
 class HydrusImporter
 {
   public:
@@ -51,6 +53,9 @@ class HydrusImporter
 	HydrusImporter( const std::filesystem::path& path );
 	~HydrusImporter();
 
+	std::unordered_map< HashID, RecordID > mapHydrusRecords( std::vector< HashID > hash_ids ) const;
+	RecordID getRecordIDFromHyID( HashID hash_id );
+
 	bool hasPTR() const;
 
 	std::vector< ServiceInfo > getTagServices();

HydrusImporter/src/gui/hydrus/HydrusImporterWidget.cpp

@@ -5,15 +5,16 @@
 
 #include "HydrusImporterWidget.hpp"
 
+#include <moc_HydrusImporterWidget.cpp>
+
 #include <QFileDialog>
 #include <QFileInfo>
 
-#include <ui_TagServiceWidget.h>
-
 #include "HydrusImporter.hpp"
-#include "TagServiceWidget.hpp"
-#include "TagServiceWorker.hpp"
+#include "file_relationships/FileRelationshipsWidget.hpp"
+#include "tag_service/TagServiceWidget.hpp"
 #include "ui_HydrusImporterWidget.h"
+#include "urls/UrlServiceWidget.hpp"
 
 class TagServiceWorker;
 
@@ -41,6 +42,93 @@ HydrusImporterWidget::~HydrusImporterWidget()
 	delete ui;
 }
 
+void HydrusImporterWidget::parseTagServices()
+{
+	auto service_infos { m_importer->getTagServices() };
+
+	for ( const auto& service : service_infos )
+	{
+		if ( service.name == "public tag repository" && !ui->cbProcessPTR->isChecked() )
+		{
+			// idhan::logging::info( "Skipping PTR because cbProcessPTR is not checked" );
+			continue;
+		}
+
+		auto widget { new TagServiceWidget( m_importer.get(), this ) };
+
+		widget->setName( service.name );
+		widget->setInfo( service );
+
+		// ui->tagServicesLayout->addWidget( widget );
+		addServiceWidget( widget );
+
+		connect(
+			this,
+			&HydrusImporterWidget::triggerImport,
+			widget,
+			&TagServiceWidget::startImport,
+			Qt::SingleShotConnection );
+		connect(
+			this,
+			&HydrusImporterWidget::triggerPreImport,
+			widget,
+			&TagServiceWidget::startPreImport,
+			Qt::SingleShotConnection );
+	}
+}
+
+void HydrusImporterWidget::addServiceWidget( QWidget* widget )
+{
+	auto* groupFrame = new QFrame( this );
+	groupFrame->setFrameShape( QFrame::Box );
+	groupFrame->setFrameShadow( QFrame::Plain );
+	groupFrame->setLineWidth( 1 );
+	groupFrame->setStyleSheet( "QFrame { border: 1px solid #444; border-radius: 6px; }" );
+
+	auto* groupLayout = new QVBoxLayout( groupFrame );
+	groupLayout->setContentsMargins( 6, 6, 6, 6 );
+	groupLayout->addWidget( widget );
+	widget->setStyleSheet( "QFrame { border: none; }" );
+
+	ui->tagServicesLayout->addWidget( groupFrame );
+}
+
+void HydrusImporterWidget::parseFileRelationships()
+{
+	auto* widget { new FileRelationshipsWidget( m_importer.get() ) };
+
+	connect(
+		this,
+		&HydrusImporterWidget::triggerImport,
+		widget,
+		&FileRelationshipsWidget::startImport,
+		Qt::SingleShotConnection );
+	connect(
+		this,
+		&HydrusImporterWidget::triggerPreImport,
+		widget,
+		&FileRelationshipsWidget::startPreImport,
+		Qt::SingleShotConnection );
+
+	addServiceWidget( widget );
+}
+
+void HydrusImporterWidget::parseUrls()
+{
+	auto* widget { new UrlServiceWidget( m_importer.get() ) };
+
+	connect(
+		this, &HydrusImporterWidget::triggerImport, widget, &UrlServiceWidget::startImport, Qt::SingleShotConnection );
+	connect(
+		this,
+		&HydrusImporterWidget::triggerPreImport,
+		widget,
+		&UrlServiceWidget::startPreImport,
+		Qt::SingleShotConnection );
+
+	addServiceWidget( widget );
+}
+
 void HydrusImporterWidget::on_hydrusFolderPath_textChanged( [[maybe_unused]] const QString& path )
 {
 	testHydrusPath();
@@ -101,39 +189,11 @@ void HydrusImporterWidget::on_parseHydrusDB_pressed()
 	m_importer = std::make_unique< idhan::hydrus::HydrusImporter >( ui->hydrusFolderPath->text().toStdString() );
 
 	const bool has_ptr { m_importer->hasPTR() };
-
-	auto service_infos { m_importer->getTagServices() };
-
 	ui->parseStatusLabel->setText( QString( "Has PTR: %1" ).arg( has_ptr ? "Yes" : "No" ) );
 
-	for ( const auto& service : service_infos )
-	{
-		if ( service.name == "public tag repository" && !ui->cbProcessPTR->isChecked() )
-		{
-			// idhan::logging::info( "Skipping PTR because cbProcessPTR is not checked" );
-			continue;
-		}
-
-		auto widget { new TagServiceWidget( m_importer.get(), this ) };
-
-		widget->setName( service.name );
-		widget->setInfo( service );
-
-		ui->tagServicesLayout->addWidget( widget );
-
-		connect(
-			this,
-			&HydrusImporterWidget::triggerImport,
-			widget,
-			&TagServiceWidget::startImport,
-			Qt::SingleShotConnection );
-		connect(
-			this,
-			&HydrusImporterWidget::triggerPreImport,
-			widget,
-			&TagServiceWidget::startPreImport,
-			Qt::SingleShotConnection );
-	}
+	parseTagServices();
+	parseFileRelationships();
+	parseUrls();
 
 	emit triggerPreImport();
 

HydrusImporter/src/gui/hydrus/HydrusImporterWidget.hpp

@@ -36,6 +36,11 @@ class HydrusImporterWidget final : public QWidget
 	explicit HydrusImporterWidget( QWidget* parent = nullptr );
 	~HydrusImporterWidget() override;
 
+	void parseTagServices();
+	void addServiceWidget( QWidget* widget );
+	void parseFileRelationships();
+	void parseUrls();
+
   public slots:
 	void on_hydrusFolderPath_textChanged( const QString& path );
 	void testHydrusPath();

HydrusImporter/src/gui/hydrus/TagServiceWidget.ui

@@ -1,208 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>TagServiceWidget</class>
- <widget class="QWidget" name="TagServiceWidget">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>691</width>
-    <height>96</height>
-   </rect>
-  </property>
-  <property name="sizePolicy">
-   <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
-    <horstretch>0</horstretch>
-    <verstretch>0</verstretch>
-   </sizepolicy>
-  </property>
-  <property name="windowTitle">
-   <string>Form</string>
-  </property>
-  <layout class="QHBoxLayout" name="horizontalLayout">
-   <item>
-    <widget class="QFrame" name="frame_3">
-     <property name="frameShape">
-      <enum>QFrame::Shape::StyledPanel</enum>
-     </property>
-     <property name="frameShadow">
-      <enum>QFrame::Shadow::Raised</enum>
-     </property>
-     <layout class="QVBoxLayout" name="verticalLayout_3">
-      <item>
-       <widget class="QLabel" name="name">
-        <property name="sizePolicy">
-         <sizepolicy hsizetype="MinimumExpanding" vsizetype="Preferred">
-          <horstretch>0</horstretch>
-          <verstretch>0</verstretch>
-         </sizepolicy>
-        </property>
-        <property name="minimumSize">
-         <size>
-          <width>0</width>
-          <height>0</height>
-         </size>
-        </property>
-        <property name="text">
-         <string>Name: </string>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <widget class="QProgressBar" name="progressBar">
-        <property name="enabled">
-         <bool>false</bool>
-        </property>
-        <property name="sizePolicy">
-         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
-          <horstretch>0</horstretch>
-          <verstretch>0</verstretch>
-         </sizepolicy>
-        </property>
-        <property name="styleSheet">
-         <string notr="true">color: rgb(0, 170, 0);</string>
-        </property>
-        <property name="minimum">
-         <number>0</number>
-        </property>
-        <property name="maximum">
-         <number>0</number>
-        </property>
-        <property name="value">
-         <number>-1</number>
-        </property>
-        <property name="textVisible">
-         <bool>true</bool>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <widget class="QLabel" name="statusLabel">
-        <property name="text">
-         <string/>
-        </property>
-       </widget>
-      </item>
-     </layout>
-    </widget>
-   </item>
-   <item>
-    <widget class="QFrame" name="frame">
-     <property name="sizePolicy">
-      <sizepolicy hsizetype="MinimumExpanding" vsizetype="Preferred">
-       <horstretch>0</horstretch>
-       <verstretch>0</verstretch>
-      </sizepolicy>
-     </property>
-     <property name="minimumSize">
-      <size>
-       <width>0</width>
-       <height>0</height>
-      </size>
-     </property>
-     <property name="frameShape">
-      <enum>QFrame::Shape::StyledPanel</enum>
-     </property>
-     <property name="frameShadow">
-      <enum>QFrame::Shadow::Raised</enum>
-     </property>
-     <layout class="QVBoxLayout" name="verticalLayout">
-      <property name="spacing">
-       <number>6</number>
-      </property>
-      <item alignment="Qt::AlignmentFlag::AlignLeft">
-       <widget class="QLabel" name="mappingsCount">
-        <property name="text">
-         <string>Mappings: 0</string>
-        </property>
-       </widget>
-      </item>
-      <item alignment="Qt::AlignmentFlag::AlignLeft">
-       <widget class="QLabel" name="parentsCount">
-        <property name="sizePolicy">
-         <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
-          <horstretch>0</horstretch>
-          <verstretch>0</verstretch>
-         </sizepolicy>
-        </property>
-        <property name="text">
-         <string>Parents/Children: 0</string>
-        </property>
-       </widget>
-      </item>
-      <item alignment="Qt::AlignmentFlag::AlignLeft">
-       <widget class="QLabel" name="aliasesCount">
-        <property name="text">
-         <string>Aliases: 0</string>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <spacer name="verticalSpacer_3">
-        <property name="orientation">
-         <enum>Qt::Orientation::Vertical</enum>
-        </property>
-        <property name="sizeHint" stdset="0">
-         <size>
-          <width>20</width>
-          <height>40</height>
-         </size>
-        </property>
-       </spacer>
-      </item>
-     </layout>
-    </widget>
-   </item>
-   <item>
-    <widget class="QFrame" name="frame_2">
-     <property name="frameShape">
-      <enum>QFrame::Shape::StyledPanel</enum>
-     </property>
-     <property name="frameShadow">
-      <enum>QFrame::Shadow::Raised</enum>
-     </property>
-     <layout class="QVBoxLayout" name="verticalLayout_2">
-      <item>
-       <widget class="QCheckBox" name="cbShouldImport">
-        <property name="text">
-         <string>Import</string>
-        </property>
-        <property name="checked">
-         <bool>true</bool>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <widget class="QCheckBox" name="checkBox">
-        <property name="enabled">
-         <bool>false</bool>
-        </property>
-        <property name="toolTip">
-         <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Only processes mappings for files that you've obtained&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
-        </property>
-        <property name="text">
-         <string>Only acquired files</string>
-        </property>
-       </widget>
-      </item>
-      <item>
-       <spacer name="verticalSpacer">
-        <property name="orientation">
-         <enum>Qt::Orientation::Vertical</enum>
-        </property>
-        <property name="sizeHint" stdset="0">
-         <size>
-          <width>20</width>
-          <height>40</height>
-         </size>
-        </property>
-       </spacer>
-      </item>
-     </layout>
-    </widget>
-   </item>
-  </layout>
- </widget>
- <resources/>
- <connections/>
-</ui>

HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.cpp

@@ -0,0 +1,101 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+#include "FileRelationshipsWidget.hpp"
+
+#include "FileRelationshipsWorker.hpp"
+#include "ui_FileRelationshipsWidget.h"
+
+FileRelationshipsWidget::FileRelationshipsWidget( idhan::hydrus::HydrusImporter* importer, QWidget* parent ) :
+  QWidget( parent ),
+  m_importer( importer ),
+  m_worker( new FileRelationshipsWorker( this, importer ) ),
+  ui( new Ui::FileRelationshipsWidget() )
+{
+	ui->setupUi( this );
+
+	connect(
+		m_worker,
+		&FileRelationshipsWorker::processedMaxDuplicates,
+		this,
+		&FileRelationshipsWidget::processedMaxDuplicates );
+	connect(
+		m_worker,
+		&FileRelationshipsWorker::processedMaxAlternatives,
+		this,
+		&FileRelationshipsWidget::processedMaxAlternatives );
+
+	connect(
+		m_worker, &FileRelationshipsWorker::processedDuplicates, this, &FileRelationshipsWidget::processedDuplicates );
+	connect(
+		m_worker,
+		&FileRelationshipsWorker::processedAlternatives,
+		this,
+		&FileRelationshipsWidget::processedAlternatives );
+
+	connect( m_worker, &FileRelationshipsWorker::statusMessage, this, &FileRelationshipsWidget::statusMessage );
+}
+
+void FileRelationshipsWidget::updateText()
+{
+	if ( alternatives_processed > 0 || duplicates_processed > 0 )
+	{
+		ui->alternativesCount->setText(
+			QString( "Alternative files: %L1 (%L2 processed)" )
+				.arg( alternatives_total )
+				.arg( alternatives_processed ) );
+		ui->duplicatesCount->setText(
+			QString( "Duplicate pairs: %L1 (%L2 processed)" ).arg( duplicates_total ).arg( duplicates_processed ) );
+	}
+	else
+	{
+		ui->alternativesCount->setText( QString( "Alternative files: %L1" ).arg( alternatives_total ) );
+		ui->duplicatesCount->setText( QString( "Duplicate pairs: %L1" ).arg( duplicates_total ) );
+	}
+
+	ui->progressBar->setMaximum( alternatives_total + duplicates_total );
+
+	if ( alternatives_processed + duplicates_processed == 0 )
+		ui->progressBar->setValue( -1 );
+	else
+		ui->progressBar->setValue( alternatives_processed + duplicates_processed );
+}
+
+void FileRelationshipsWidget::startImport()
+{
+	if ( ui->cbShouldImport->isChecked() ) QThreadPool::globalInstance()->start( m_worker );
+}
+
+void FileRelationshipsWidget::startPreImport()
+{
+	QThreadPool::globalInstance()->start( m_worker );
+}
+
+void FileRelationshipsWidget::statusMessage( const QString& msg )
+{
+	ui->statusLabel->setText( msg );
+}
+
+void FileRelationshipsWidget::processedDuplicates( const std::size_t count )
+{
+	duplicates_processed = count;
+	updateText();
+}
+
+void FileRelationshipsWidget::processedMaxDuplicates( const std::size_t count )
+{
+	duplicates_total = count;
+	updateText();
+}
+
+void FileRelationshipsWidget::processedAlternatives( const std::size_t count )
+{
+	alternatives_processed = count;
+	updateText();
+}
+
+void FileRelationshipsWidget::processedMaxAlternatives( const std::size_t count )
+{
+	alternatives_total = count;
+	updateText();
+}

HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.hpp

@@ -0,0 +1,47 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+#pragma once
+#include "FileRelationshipsWorker.hpp"
+#include "gui/hydrus/HydrusImporterWidget.hpp"
+
+namespace Ui
+{
+class FileRelationshipsWidget;
+}
+
+class FileRelationshipsWidget : public QWidget
+{
+	idhan::hydrus::HydrusImporter* m_importer;
+
+	std::size_t alternatives_processed { 0 };
+	std::size_t alternatives_total { 0 };
+
+	std::size_t duplicates_processed { 0 };
+	std::size_t duplicates_total { 0 };
+	FileRelationshipsWorker* m_worker;
+
+  public:
+
+	Q_DISABLE_COPY_MOVE( FileRelationshipsWidget );
+
+	explicit FileRelationshipsWidget( idhan::hydrus::HydrusImporter* importer, QWidget* parent = nullptr );
+
+  private:
+
+	void updateText();
+
+  public slots:
+	void startImport();
+	void startPreImport();
+	void statusMessage( const QString& msg );
+
+	void processedDuplicates( std::size_t count );
+	void processedMaxDuplicates( std::size_t count );
+	void processedAlternatives( std::size_t count );
+	void processedMaxAlternatives( std::size_t count );
+
+  private:
+
+	Ui::FileRelationshipsWidget* ui;
+};

HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWidget.ui

@@ -0,0 +1,179 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>FileRelationshipsWidget</class>
+ <widget class="QWidget" name="FileRelationshipsWidget">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>907</width>
+    <height>98</height>
+   </rect>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout_5">
+   <item>
+    <layout class="QVBoxLayout" name="verticalLayout_4">
+     <property name="spacing">
+      <number>0</number>
+     </property>
+     <item>
+      <layout class="QHBoxLayout" name="horizontalLayout">
+       <property name="spacing">
+        <number>0</number>
+       </property>
+       <item>
+        <layout class="QVBoxLayout" name="verticalLayout">
+         <property name="spacing">
+          <number>0</number>
+         </property>
+         <property name="leftMargin">
+          <number>0</number>
+         </property>
+         <property name="topMargin">
+          <number>0</number>
+         </property>
+         <property name="rightMargin">
+          <number>0</number>
+         </property>
+         <property name="bottomMargin">
+          <number>0</number>
+         </property>
+         <item alignment="Qt::AlignmentFlag::AlignTop">
+          <widget class="QLabel" name="name">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="minimumSize">
+            <size>
+             <width>0</width>
+             <height>0</height>
+            </size>
+           </property>
+           <property name="text">
+            <string>Name: Duplicate &amp; Alternative relationships
+Type: File Relationships</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QLabel" name="statusLabel">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="text">
+            <string/>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <layout class="QVBoxLayout" name="verticalLayout_2">
+         <property name="spacing">
+          <number>0</number>
+         </property>
+         <item alignment="Qt::AlignmentFlag::AlignVCenter">
+          <widget class="QLabel" name="alternativesCount">
+           <property name="text">
+            <string>Alternative Groups: 0</string>
+           </property>
+          </widget>
+         </item>
+         <item alignment="Qt::AlignmentFlag::AlignVCenter">
+          <widget class="QLabel" name="duplicatesCount">
+           <property name="text">
+            <string>Duplicate Pairs: 0</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <spacer name="verticalSpacer">
+           <property name="orientation">
+            <enum>Qt::Orientation::Vertical</enum>
+           </property>
+           <property name="sizeHint" stdset="0">
+            <size>
+             <width>0</width>
+             <height>0</height>
+            </size>
+           </property>
+          </spacer>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <layout class="QVBoxLayout" name="verticalLayout_3">
+       <property name="spacing">
+        <number>0</number>
+       </property>
+       <item>
+        <layout class="QHBoxLayout" name="horizontalLayout_2">
+         <property name="spacing">
+          <number>0</number>
+         </property>
+         <item>
+          <widget class="QProgressBar" name="progressBar">
+           <property name="enabled">
+            <bool>false</bool>
+           </property>
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Expanding" vsizetype="Maximum">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="styleSheet">
+            <string notr="true">color: rgb(0, 170, 0);</string>
+           </property>
+           <property name="minimum">
+            <number>0</number>
+           </property>
+           <property name="maximum">
+            <number>0</number>
+           </property>
+           <property name="value">
+            <number>-1</number>
+           </property>
+           <property name="textVisible">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QCheckBox" name="cbShouldImport">
+           <property name="text">
+            <string>Import</string>
+           </property>
+           <property name="checked">
+            <bool>true</bool>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.cpp

@@ -0,0 +1,200 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+#include "FileRelationshipsWorker.hpp"
+
+#include <moc_FileRelationshipsWorker.cpp>
+
+#include "sqlitehelper/Query.hpp"
+#include "sqlitehelper/TransactionBaseCoro.hpp"
+
+FileRelationshipsWorker::FileRelationshipsWorker( QObject* parent, idhan::hydrus::HydrusImporter* importer ) :
+  QObject( parent ),
+  QRunnable(),
+  m_importer( importer )
+{
+	this->setAutoDelete( false );
+}
+
+void FileRelationshipsWorker::preprocess()
+{
+	idhan::hydrus::TransactionBaseCoro client_tr { m_importer->client_db };
+
+	idhan::hydrus::Query< int, int > duplicate_file_members {
+		client_tr,
+		"SELECT hash_id, king_hash_id FROM duplicate_file_members dfm JOIN duplicate_files df ON dfm.media_id = df.media_id WHERE hash_id != king_hash_id"
+	};
+
+	std::size_t duplicate_file_members_counter { 0 };
+
+	for ( [[maybe_unused]] const auto& [ media_id, hash_id ] : duplicate_file_members )
+	{
+		duplicate_file_members_counter++;
+
+		if ( duplicate_file_members_counter % 1000 == 0 ) emit processedMaxDuplicates( duplicate_file_members_counter );
+	}
+
+	emit processedMaxDuplicates( duplicate_file_members_counter );
+
+	idhan::hydrus::Query< int, int > alternative_file_members {
+		client_tr, "SELECT * FROM alternate_file_group_members JOIN duplicate_file_members USING (media_id)"
+	};
+	std::size_t alternative_file_members_counter { 0 };
+
+	for ( [[maybe_unused]] const auto& [ media_id, hash_id ] : alternative_file_members )
+	{
+		alternative_file_members_counter++;
+
+		if ( alternative_file_members_counter % 1000 == 0 )
+			emit processedMaxAlternatives( alternative_file_members_counter );
+	}
+
+	emit processedMaxAlternatives( alternative_file_members_counter );
+}
+
+void FileRelationshipsWorker::process()
+{
+	idhan::hydrus::TransactionBaseCoro client_tr { m_importer->client_db };
+
+	using MediaID = std::uint32_t;
+	using HashID = std::uint32_t;
+	using KingID = std::uint32_t;
+
+	std::vector< HashID > hash_ids {};
+
+	std::unordered_map< HashID, idhan::RecordID > record_map {};
+
+	emit statusMessage( "Started" );
+
+	auto flushHashIDs = [ &, this ]()
+	{
+		emit statusMessage( "Mapping Hydrus IDs to IDHAN IDs" );
+		std::ranges::sort( hash_ids );
+		std::ranges::unique( hash_ids );
+
+		std::ranges::remove_if(
+			hash_ids, [ &record_map ]( const HashID hash_id ) -> bool { return record_map.contains( hash_id ); } );
+
+		if ( hash_ids.empty() ) return;
+
+		const auto batch_map { m_importer->mapHydrusRecords( hash_ids ) };
+		// merge the new map into the existing one
+
+		for ( const auto& [ hy_hash_id, idhan_hash_id ] : batch_map )
+			record_map.insert_or_assign( hy_hash_id, idhan_hash_id );
+
+		hash_ids.clear();
+	};
+
+	idhan::hydrus::Query< HashID, KingID > duplicate_files {
+		client_tr,
+		"SELECT hash_id, king_hash_id FROM duplicate_file_members dfm JOIN duplicate_files df ON dfm.media_id = df.media_id WHERE hash_id != king_hash_id"
+	};
+
+	std::vector< std::pair< HashID, KingID > > pairs {};
+
+	auto& client { idhan::IDHANClient::instance() };
+
+	std::size_t processed_count { 0 };
+
+	auto flushPairs = [ &, this ]()
+	{
+		flushHashIDs();
+
+		emit statusMessage( "Setting duplicates for batch" );
+
+		std::vector< std::pair< idhan::RecordID, idhan::RecordID > > idhan_pairs {};
+
+		for ( const auto& [ hy_hash_id, hy_king_id ] : pairs )
+		{
+			const auto idhan_hash_id { record_map.at( hy_hash_id ) };
+			const auto idhan_king_id { record_map.at( hy_king_id ) };
+
+			idhan_pairs.emplace_back( idhan_hash_id, idhan_king_id );
+
+			// auto future = client.setDuplicates( pairs );
+			// future.waitForFinished();
+		}
+
+		auto future { client.setDuplicates( idhan_pairs ) };
+		future.waitForFinished();
+
+		processed_count += pairs.size();
+		emit processedDuplicates( processed_count );
+
+		pairs.clear();
+	};
+
+	for ( const auto& [ hash_id, king_id ] : duplicate_files )
+	{
+		if ( hash_id == king_id ) continue;
+
+		if ( !record_map.contains( hash_id ) ) hash_ids.push_back( hash_id );
+		if ( !record_map.contains( king_id ) ) hash_ids.push_back( king_id );
+
+		pairs.emplace_back( std::make_pair( hash_id, king_id ) );
+
+		if ( pairs.size() >= 100 )
+		{
+			flushPairs();
+			emit statusMessage( "Getting additional rows to process" );
+		}
+	}
+
+	flushPairs();
+
+	idhan::hydrus::Query< HashID, MediaID > alternative_files {
+		client_tr,
+		"SELECT hash_id, alternates_group_id FROM alternate_file_group_members JOIN duplicate_file_members USING (media_id);"
+	};
+
+	using GroupID = std::uint32_t;
+
+	std::unordered_map< GroupID, std::vector< idhan::hydrus::HashID > > alternative_map {};
+
+	emit statusMessage( "Mapping alternative hashes to IDHAN" );
+
+	for ( const auto& [ hash_id, alternative_group_id ] : alternative_files )
+	{
+		hash_ids.push_back( hash_id );
+
+		if ( hash_ids.size() >= 64 ) flushHashIDs();
+
+		if ( auto itter = alternative_map.find( alternative_group_id ); itter != alternative_map.end() )
+			itter->second.push_back( hash_id );
+		else
+			alternative_map.emplace( alternative_group_id, std::vector< idhan::hydrus::HashID > { hash_id } );
+	}
+
+	flushHashIDs();
+
+	emit statusMessage( "Setting alternatives for groups" );
+
+	std::size_t alternative_count { 0 };
+
+	for ( const auto hy_hashes : alternative_map | std::views::values )
+	{
+		std::vector< idhan::RecordID > record_ids {};
+		for ( const auto& hy_hash : hy_hashes ) record_ids.emplace_back( record_map.at( hy_hash ) );
+
+		alternative_count += record_ids.size();
+
+		auto future = client.setAlternativeGroups( record_ids );
+		future.waitForFinished();
+		emit processedAlternatives( alternative_count );
+	}
+
+	emit statusMessage( "Finished" );
+}
+
+void FileRelationshipsWorker::run()
+{
+	if ( !m_preprocessed )
+	{
+		m_preprocessed = true;
+		preprocess();
+		return;
+	}
+
+	process();
+}

HydrusImporter/src/gui/hydrus/file_relationships/FileRelationshipsWorker.hpp

@@ -0,0 +1,32 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+#pragma once
+
+#include <QObject>
+
+#include "HydrusImporter.hpp"
+
+class FileRelationshipsWorker : public QObject, public QRunnable
+{
+	Q_OBJECT
+
+	idhan::hydrus::HydrusImporter* m_importer;
+	bool m_preprocessed { false };
+
+  signals:
+
+	void processedMaxDuplicates( std::size_t counter );
+	void processedMaxAlternatives( std::size_t counter );
+	void statusMessage( const QString& message );
+	void processedDuplicates( std::size_t );
+	void processedAlternatives( std::size_t );
+
+  public:
+
+	FileRelationshipsWorker( QObject* parent, idhan::hydrus::HydrusImporter* importer );
+	void preprocess();
+	void process();
+
+	void run() override;
+};

HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.cpp

@@ -53,7 +53,7 @@ TagServiceWidget::~TagServiceWidget()
 void TagServiceWidget::setName( const QString& name )
 {
 	m_name = name;
-	ui->name->setText( QString( "Name: %L1" ).arg( name ) );
+	ui->name->setText( QString( "Name: %L1\nType: Tag Service" ).arg( name ) );
 }
 
 void TagServiceWidget::recordMappingProcessed( std::size_t count )
@@ -113,6 +113,12 @@ void TagServiceWidget::updateTime()
 	const std::size_t to_process { m_info.num_mappings + m_info.num_parents + m_info.num_aliases };
 	const std::size_t total_processed { mappings_processed + parents_processed + aliases_processed };
 
+	if ( total_processed == 0 )
+	{
+		ui->statusLabel->setText( "Ready!" );
+		return;
+	}
+
 	// const bool over_limit { to_process > std::numeric_limits< int >::max() };
 	// const std::size_t multip { over_limit ? 16 : 1 };
 
@@ -169,7 +175,11 @@ void TagServiceWidget::updateProcessed()
 	const auto over_limit { to_process > std::numeric_limits< int >::max() };
 	const auto multip { over_limit ? 1024 : 1 };
 
-	ui->progressBar->setValue( static_cast< int >( total_processed / multip ) );
+	if ( total_processed == 0 )
+		ui->progressBar->setValue( -1 );
+	else
+		ui->progressBar->setValue( static_cast< int >( total_processed / multip ) );
+
 	ui->progressBar->setMaximum( static_cast< int >( to_process / multip ) );
 }
 
@@ -214,6 +224,7 @@ void TagServiceWidget::processedAliases( std::size_t count )
 void TagServiceWidget::preprocessingFinished()
 {
 	m_preprocessed = true;
+	updateProcessed();
 	updateTime();
 }
 

HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.hpp

@@ -9,7 +9,8 @@
 #include <deque>
 
 #include "HydrusImporter.hpp"
-#include "HydrusImporterWidget.hpp"
+
+class TagServiceWorker;
 
 namespace Ui
 {

HydrusImporter/src/gui/hydrus/tag_service/TagServiceWidget.ui

@@ -0,0 +1,213 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>TagServiceWidget</class>
+ <widget class="QWidget" name="TagServiceWidget">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>605</width>
+    <height>104</height>
+   </rect>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="Preferred" vsizetype="Fixed">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout_5">
+   <item>
+    <layout class="QVBoxLayout" name="verticalLayout_4">
+     <property name="spacing">
+      <number>0</number>
+     </property>
+     <item>
+      <layout class="QHBoxLayout" name="horizontalLayout_3">
+       <property name="spacing">
+        <number>0</number>
+       </property>
+       <item>
+        <layout class="QVBoxLayout" name="verticalLayout_2">
+         <property name="spacing">
+          <number>0</number>
+         </property>
+         <property name="leftMargin">
+          <number>0</number>
+         </property>
+         <property name="topMargin">
+          <number>0</number>
+         </property>
+         <property name="rightMargin">
+          <number>0</number>
+         </property>
+         <property name="bottomMargin">
+          <number>0</number>
+         </property>
+         <item>
+          <widget class="QLabel" name="name">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Preferred" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="minimumSize">
+            <size>
+             <width>0</width>
+             <height>0</height>
+            </size>
+           </property>
+           <property name="text">
+            <string>Name: </string>
+           </property>
+          </widget>
+         </item>
+         <item alignment="Qt::AlignmentFlag::AlignBottom">
+          <widget class="QLabel" name="statusLabel">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Preferred" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="text">
+            <string>Preprocessing</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <layout class="QVBoxLayout" name="verticalLayout_3">
+         <property name="spacing">
+          <number>0</number>
+         </property>
+         <item>
+          <layout class="QVBoxLayout" name="verticalLayout">
+           <property name="spacing">
+            <number>0</number>
+           </property>
+           <property name="sizeConstraint">
+            <enum>QLayout::SizeConstraint::SetDefaultConstraint</enum>
+           </property>
+           <item>
+            <widget class="QLabel" name="mappingsCount">
+             <property name="text">
+              <string>Mappings: 0</string>
+             </property>
+            </widget>
+           </item>
+           <item alignment="Qt::AlignmentFlag::AlignTop">
+            <widget class="QLabel" name="aliasesCount">
+             <property name="sizePolicy">
+              <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+               <horstretch>0</horstretch>
+               <verstretch>0</verstretch>
+              </sizepolicy>
+             </property>
+             <property name="text">
+              <string>Aliases: 0</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
+         <item>
+          <layout class="QHBoxLayout" name="horizontalLayout_2">
+           <property name="spacing">
+            <number>0</number>
+           </property>
+           <item alignment="Qt::AlignmentFlag::AlignTop">
+            <widget class="QLabel" name="parentsCount">
+             <property name="sizePolicy">
+              <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+               <horstretch>0</horstretch>
+               <verstretch>0</verstretch>
+              </sizepolicy>
+             </property>
+             <property name="text">
+              <string>Parents/Children: 0</string>
+             </property>
+            </widget>
+           </item>
+           <item alignment="Qt::AlignmentFlag::AlignRight|Qt::AlignmentFlag::AlignBottom">
+            <widget class="QCheckBox" name="checkBox">
+             <property name="enabled">
+              <bool>false</bool>
+             </property>
+             <property name="sizePolicy">
+              <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
+               <horstretch>0</horstretch>
+               <verstretch>0</verstretch>
+              </sizepolicy>
+             </property>
+             <property name="toolTip">
+              <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Only processes mappings for files that you've obtained&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+             </property>
+             <property name="text">
+              <string>Only acquired files</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <layout class="QHBoxLayout" name="horizontalLayout">
+       <property name="spacing">
+        <number>0</number>
+       </property>
+       <item>
+        <widget class="QProgressBar" name="progressBar">
+         <property name="enabled">
+          <bool>false</bool>
+         </property>
+         <property name="sizePolicy">
+          <sizepolicy hsizetype="Expanding" vsizetype="Maximum">
+           <horstretch>0</horstretch>
+           <verstretch>0</verstretch>
+          </sizepolicy>
+         </property>
+         <property name="styleSheet">
+          <string notr="true">color: rgb(0, 170, 0);</string>
+         </property>
+         <property name="minimum">
+          <number>0</number>
+         </property>
+         <property name="maximum">
+          <number>0</number>
+         </property>
+         <property name="value">
+          <number>-1</number>
+         </property>
+         <property name="textVisible">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <widget class="QCheckBox" name="cbShouldImport">
+         <property name="text">
+          <string>Import</string>
+         </property>
+         <property name="checked">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

HydrusImporter/src/gui/hydrus/tag_service/TagServiceWorker.cpp

@@ -4,12 +4,13 @@
 
 #include "TagServiceWorker.hpp"
 
+#include <moc_TagServiceWorker.cpp>
+
 #include <QObject>
 #include <QtConcurrent>
 
 #include <fstream>
 
-#include "moc_TagServiceWorker.cpp"
 #include "sqlitehelper/Query.hpp"
 #include "sqlitehelper/Transaction.hpp"
 #include "sqlitehelper/TransactionBaseCoro.hpp"

HydrusImporter/src/gui/hydrus/urls/UrlServiceWidget.cpp

@@ -0,0 +1,58 @@
+//
+// Created by kj16609 on 11/7/25.
+//
+// You may need to build the project (run Qt uic code generator) to get "ui_UrlServiceWidget.h" resolved
+
+#include "UrlServiceWidget.hpp"
+
+#include <QThreadPool>
+
+#include "UrlServiceWorker.hpp"
+#include "ui_UrlServiceWidget.h"
+
+UrlServiceWidget::UrlServiceWidget( idhan::hydrus::HydrusImporter* get, QWidget* parent ) :
+  QWidget( parent ),
+  ui( new Ui::UrlServiceWidget() ),
+  m_importer( get )
+{
+	ui->setupUi( this );
+
+	m_worker = new UrlServiceWorker( this, m_importer );
+
+	connect( m_worker, &UrlServiceWorker::processedMaxUrls, this, &UrlServiceWidget::processedMaxUrls );
+	connect( m_worker, &UrlServiceWorker::processedUrls, this, &UrlServiceWidget::processedUrls );
+	connect( m_worker, &UrlServiceWorker::statusMessage, this, &UrlServiceWidget::statusMessage );
+}
+
+UrlServiceWidget::~UrlServiceWidget()
+{
+	delete ui;
+}
+
+void UrlServiceWidget::startPreImport()
+{
+	QThreadPool::globalInstance()->start( m_worker );
+}
+
+void UrlServiceWidget::startImport()
+{
+	if ( ui->cbShouldImport->isChecked() ) QThreadPool::globalInstance()->start( m_worker );
+}
+
+void UrlServiceWidget::statusMessage( const QString& msg )
+{
+	ui->statusLabel->setText( msg );
+}
+
+void UrlServiceWidget::processedMaxUrls( const std::size_t count )
+{
+	ui->urlCount->setText( QString( "URL mappings: %L1" ).arg( count ) );
+	ui->progressBar->setMaximum( static_cast< int >( count ) );
+	m_max_urls = count;
+}
+
+void UrlServiceWidget::processedUrls( const std::size_t count )
+{
+	ui->urlCount->setText( QString( "URL mappings: %L1 (%L2 processed)" ).arg( m_max_urls ).arg( count ) );
+	ui->progressBar->setValue( static_cast< int >( count ) );
+}

HydrusImporter/src/gui/hydrus/urls/UrlServiceWidget.hpp

@@ -0,0 +1,47 @@
+//
+// Created by kj16609 on 11/7/25.
+//
+#ifndef IDHAN_URLSERVICEWIDGET_HPP
+#define IDHAN_URLSERVICEWIDGET_HPP
+
+#include <QWidget>
+
+class UrlServiceWorker;
+
+namespace idhan::hydrus
+{
+class HydrusImporter;
+}
+
+QT_BEGIN_NAMESPACE
+
+namespace Ui
+{
+class UrlServiceWidget;
+}
+
+QT_END_NAMESPACE
+
+class UrlServiceWidget : public QWidget
+{
+	Q_OBJECT
+
+	Ui::UrlServiceWidget* ui;
+	idhan::hydrus::HydrusImporter* m_importer;
+	UrlServiceWorker* m_worker { nullptr };
+	std::size_t m_max_urls { 0 };
+
+  public:
+
+	explicit UrlServiceWidget( idhan::hydrus::HydrusImporter* get, QWidget* parent = nullptr );
+	~UrlServiceWidget() override;
+
+  public slots:
+	void startPreImport();
+	void startImport();
+	void statusMessage( const QString& msg );
+	void processedMaxUrls( std::size_t count );
+	void processedUrls( std::size_t count );
+};
+
+#endif //IDHAN_URLSERVICEWIDGET_HPP

HydrusImporter/src/gui/hydrus/urls/UrlServiceWidget.ui

@@ -0,0 +1,143 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>UrlServiceWidget</class>
+ <widget class="QWidget" name="UrlServiceWidget">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>543</width>
+    <height>106</height>
+   </rect>
+  </property>
+  <property name="sizePolicy">
+   <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+    <horstretch>0</horstretch>
+    <verstretch>0</verstretch>
+   </sizepolicy>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout_5">
+   <item>
+    <layout class="QVBoxLayout" name="verticalLayout_4">
+     <item>
+      <layout class="QHBoxLayout" name="horizontalLayout_3">
+       <item>
+        <layout class="QVBoxLayout" name="verticalLayout_2">
+         <property name="spacing">
+          <number>0</number>
+         </property>
+         <property name="leftMargin">
+          <number>0</number>
+         </property>
+         <property name="topMargin">
+          <number>0</number>
+         </property>
+         <property name="rightMargin">
+          <number>0</number>
+         </property>
+         <property name="bottomMargin">
+          <number>0</number>
+         </property>
+         <item alignment="Qt::AlignmentFlag::AlignTop">
+          <widget class="QLabel" name="name">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="minimumSize">
+            <size>
+             <width>0</width>
+             <height>0</height>
+            </size>
+           </property>
+           <property name="text">
+            <string>Name: Record URLs
+Type: URLs</string>
+           </property>
+          </widget>
+         </item>
+         <item alignment="Qt::AlignmentFlag::AlignBottom">
+          <widget class="QLabel" name="statusLabel">
+           <property name="text">
+            <string/>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item>
+        <layout class="QVBoxLayout" name="verticalLayout_3">
+         <property name="spacing">
+          <number>0</number>
+         </property>
+         <item alignment="Qt::AlignmentFlag::AlignTop">
+          <widget class="QLabel" name="urlCount">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="text">
+            <string>URLs: 0</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <layout class="QHBoxLayout" name="horizontalLayout">
+       <item>
+        <widget class="QProgressBar" name="progressBar">
+         <property name="enabled">
+          <bool>false</bool>
+         </property>
+         <property name="sizePolicy">
+          <sizepolicy hsizetype="Expanding" vsizetype="Maximum">
+           <horstretch>0</horstretch>
+           <verstretch>0</verstretch>
+          </sizepolicy>
+         </property>
+         <property name="styleSheet">
+          <string notr="true">color: rgb(0, 170, 0);</string>
+         </property>
+         <property name="minimum">
+          <number>0</number>
+         </property>
+         <property name="maximum">
+          <number>0</number>
+         </property>
+         <property name="value">
+          <number>-1</number>
+         </property>
+         <property name="textVisible">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+       <item>
+        <widget class="QCheckBox" name="cbShouldImport">
+         <property name="text">
+          <string>Import</string>
+         </property>
+         <property name="checked">
+          <bool>true</bool>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

HydrusImporter/src/gui/hydrus/urls/UrlServiceWorker.cpp

@@ -0,0 +1,116 @@
+//
+// Created by kj16609 on 11/7/25.
+//
+#include "UrlServiceWorker.hpp"
+
+#include <moc_UrlServiceWorker.cpp>
+
+#include "sqlitehelper/Query.hpp"
+#include "sqlitehelper/TransactionBaseCoro.hpp"
+
+UrlServiceWorker::UrlServiceWorker( QObject* parent, idhan::hydrus::HydrusImporter* importer ) :
+  QObject( parent ),
+  QRunnable(),
+  m_importer( importer )
+{
+	this->setAutoDelete( false );
+}
+
+void UrlServiceWorker::preprocess()
+{
+	idhan::hydrus::TransactionBaseCoro client_tr { m_importer->client_db };
+
+	std::size_t url_counter { 0 };
+
+	idhan::hydrus::Query< int, int > query { client_tr, "SELECT hash_id, url_id FROM url_map" };
+
+	for ( [[maybe_unused]] const auto& [ hash_id, url_id ] : query )
+	{
+		url_counter += 1;
+		if ( url_counter % 10'000 == 0 ) emit processedMaxUrls( url_counter );
+	}
+
+	emit processedMaxUrls( url_counter );
+}
+
+void UrlServiceWorker::process()
+{
+	auto& client { idhan::IDHANClient::instance() };
+
+	idhan::hydrus::TransactionBaseCoro client_tr { m_importer->client_db };
+	idhan::hydrus::TransactionBaseCoro master_tr { m_importer->master_db };
+
+	std::size_t url_counter { 0 };
+
+	idhan::hydrus::Query< int, int > query { client_tr, "SELECT hash_id, url_id FROM url_map ORDER BY hash_id ASC" };
+
+	std::unordered_map< idhan::hydrus::HashID, std::vector< std::string > > current_urls {};
+
+	auto flushUrls = [ &, this ]()
+	{
+		emit statusMessage( "Mapping hydrus IDs to IDHAN IDs" );
+		std::vector< idhan::hydrus::HashID > hashes {};
+		for ( const auto& hash_id : current_urls | std::views::keys )
+		{
+			hashes.emplace_back( hash_id );
+		}
+
+		const auto mapped_ids { m_importer->mapHydrusRecords( hashes ) };
+
+		emit statusMessage( "Adding URLs to records" );
+
+		std::vector< QFuture< void > > futures {};
+
+		for ( const auto& [ hash_id, idhan_id ] : mapped_ids )
+		{
+			auto urls { current_urls[ hash_id ] };
+			auto future { client.addUrls( idhan_id, urls ) };
+			// futures.emplace_back( client.addUrls( idhan_id, urls ) );
+			future.waitForFinished();
+		}
+
+		for ( auto& future : futures ) future.waitForFinished();
+
+		current_urls.clear();
+		emit processedUrls( url_counter );
+	};
+
+	for ( [[maybe_unused]] const auto& [ hash_id, url_id ] : query )
+	{
+		idhan::hydrus::Query< std::string_view > url_query {
+			master_tr, "SELECT url FROM urls WHERE url_id = $1", url_id
+		};
+
+		std::vector< std::string > urls {};
+
+		for ( const auto& [ url ] : url_query )
+		{
+			urls.emplace_back( url );
+		}
+
+		url_counter += urls.size();
+
+		if ( auto itter = current_urls.find( hash_id ); itter != current_urls.end() )
+			itter->second.insert( itter->second.end(), urls.begin(), urls.end() );
+		else
+			current_urls.emplace( hash_id, std::move( urls ) );
+
+		if ( url_counter % 500 == 0 ) flushUrls();
+	}
+
+	flushUrls();
+
+	emit statusMessage( "Finished!" );
+}
+
+void UrlServiceWorker::run()
+{
+	if ( !m_preprocessed )
+	{
+		m_preprocessed = true;
+		preprocess();
+		return;
+	}
+
+	process();
+}

HydrusImporter/src/gui/hydrus/urls/UrlServiceWorker.hpp

@@ -0,0 +1,29 @@
+//
+// Created by kj16609 on 11/7/25.
+//
+#pragma once
+
+#include <QObject>
+
+#include "HydrusImporter.hpp"
+
+class UrlServiceWorker : public QObject, public QRunnable
+{
+	Q_OBJECT
+
+	idhan::hydrus::HydrusImporter* m_importer;
+	bool m_preprocessed { false };
+
+  signals:
+	void processedMaxUrls( std::size_t counter );
+	void processedUrls( std::size_t counter );
+	void statusMessage( const QString& message );
+
+  public:
+
+	UrlServiceWorker( QObject* parent, idhan::hydrus::HydrusImporter* importer );
+	void preprocess();
+	void process();
+
+	void run() override;
+};

HydrusImporter/src/gui/main/MainWindow.cpp

@@ -5,14 +5,16 @@
 
 #include "MainWindow.hpp"
 
-#include <QFutureWatcher>
+#include <moc_MainWindow.cpp>
+
 #include <QTimer>
 
 #include <idhan/IDHANClient.hpp>
 
-#include "../hydrus/HydrusImporterWidget.hpp"
 #include "NET_CONSTANTS.hpp"
 #include "SettingsDialog.hpp"
+#include "gui/hydrus/HydrusImporterWidget.hpp"
+#include "gui/hydrus/tag_service/TagServiceWidget.hpp"
 #include "ui_MainWindow.h"
 
 MainWindow::MainWindow( QWidget* parent ) :
@@ -96,10 +98,11 @@ void MainWindow::checkHeartbeat()
 		{
 			const auto result { watcher->result() };
 
-			ui->statusbar->showMessage( QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" )
-			                                .arg( result.server.str )
-			                                .arg( result.build_type )
-			                                .arg( result.commit ) );
+			ui->statusbar->showMessage(
+				QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" )
+					.arg( result.server.str )
+					.arg( result.build_type )
+					.arg( result.commit ) );
 		}
 		catch ( std::exception& e )
 		{

HydrusImporter/src/gui/main/SettingsDialog.cpp

@@ -4,6 +4,8 @@
 
 #include "SettingsDialog.hpp"
 
+#include <moc_SettingsDialog.cpp>
+
 #include <QFutureWatcher>
 
 #include "idhan/IDHANClient.hpp"
@@ -41,10 +43,11 @@ void SettingsDialog::on_testConnection_pressed()
 		{
 			const auto result { watcher->result() };
 
-			ui->networkSettingsLabel->setText( QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" )
-			                                       .arg( result.server.str )
-			                                       .arg( result.build_type )
-			                                       .arg( result.commit ) );
+			ui->networkSettingsLabel->setText(
+				QString( "Connected to IDHAN v%1 (Build: %4, Commit: %5)" )
+					.arg( result.server.str )
+					.arg( result.build_type )
+					.arg( result.commit ) );
 		}
 		catch ( std::exception& e )
 		{

IDHAN/CMakeLists.txt

@@ -40,22 +40,23 @@ set(GENERATED_HEADERS "")
 
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/hydrus)
 
+set(COMMAND_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/GenerateHydrusConstants.cmake")
+
 foreach (HYDRUS_FILE ${HYDRUS_SCAN_FILES})
 	get_filename_component(FILE_NAME ${HYDRUS_FILE} NAME_WE)
 	set(OUTPUT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/include/hydrus/${FILE_NAME}_gen.hpp")
 	list(APPEND GENERATED_HEADERS ${OUTPUT_FILE})
 
-	add_custom_target(
-			generate_${FILE_NAME}_constants ALL
+	add_custom_command(
+			OUTPUT ${OUTPUT_FILE}
 			COMMAND ${CMAKE_COMMAND}
 			-DHYDRUS_DIR=${HYDRUS_REPO_PATH}
 			-DHYDRUS_CONSTANTS_FILE=${HYDRUS_FILE}
 			-DOUT_TARGET=${OUTPUT_FILE}
-			-P "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules/GenerateHydrusConstants.cmake"
+			-P ${COMMAND_FILE}
+			DEPENDS ${HYDRUS_FILE} ${COMMAND_FILE}
 			COMMENT "Generating ${FILE_NAME} constants file"
 	)
-
-	add_dependencies(IDHAN generate_${FILE_NAME}_constants)
 endforeach ()
 
 target_sources(IDHAN PUBLIC ${GENERATED_HEADERS})

IDHAN/cmake_modules/GenerateHydrusConstants.cmake

@@ -19,7 +19,8 @@ list(APPEND HYDRUS_CONSTANTS "SORT_FILES_BY_FILESIZE" "SORT_FILES_BY_DURATION" "
 list(APPEND HYDRUS_CONSTANTS "SORT_ASC" "SORT_DESC")
 list(APPEND HYDRUS_CONSTANTS "PAGE_TYPE_DUMPER" "PAGE_TYPE_IMPORT_MULTIPLE_GALLERY" "PAGE_TYPE_IMPORT_SIMPLE_DOWNLOADER" "PAGE_TYPE_IMPORT_HDD" "PAGE_TYPE_IMPORT_WATCHER" "PAGE_TYPE_PETITIONS" "PAGE_TYPE_QUERY" "PAGE_TYPE_IMPORT_URLS" "PAGE_TYPE_DUPLICATE_FILTER" "PAGE_TYPE_IMPORT_MULTIPLE_WATCHER" "PAGE_TYPE_PAGE_OF_PAGES")
 list(APPEND HYDRUS_CONSTANTS "PAGE_STATE_NORMAL")
-list(APPEND HYDRUS_CONSTANTS "GENERAL_APPLICATION" "GENERAL_AUDIO" "GENERAL_IMAGE" "GENERAL_VIDEO" "GENERAL_ANIMATION" "GENERAL_APPLICATION_ARCHIVE" "GENERAL_IMAGE_PROJECT")
+#list(APPEND HYDRUS_CONSTANTS "GENERAL_APPLICATION" "GENERAL_AUDIO" "GENERAL_IMAGE" "GENERAL_VIDEO" "GENERAL_ANIMATION" "GENERAL_APPLICATION_ARCHIVE" "GENERAL_IMAGE_PROJECT")
+list(APPEND HYDRUS_CONSTANTS "APPLICATION_HYDRUS_CLIENT_COLLECTION" "IMAGE_JPEG" "IMAGE_PNG" "ANIMATION_GIF" "IMAGE_BMP" "APPLICATION_FLASH" "APPLICATION_YAML" "IMAGE_ICON" "TEXT_HTML" "VIDEO_FLV" "APPLICATION_PDF" "APPLICATION_ZIP" "APPLICATION_HYDRUS_ENCRYPTED_ZIP" "AUDIO_MP3" "VIDEO_MP4" "AUDIO_OGG" "AUDIO_FLAC" "AUDIO_WMA" "VIDEO_WMV" "UNDETERMINED_WM" "VIDEO_MKV" "VIDEO_WEBM" "APPLICATION_JSON" "ANIMATION_APNG" "UNDETERMINED_PNG" "VIDEO_MPEG" "VIDEO_MOV" "VIDEO_AVI" "APPLICATION_HYDRUS_UPDATE_DEFINITIONS" "APPLICATION_HYDRUS_UPDATE_CONTENT" "TEXT_PLAIN" "APPLICATION_RAR" "APPLICATION_7Z" "IMAGE_WEBP" "IMAGE_TIFF" "APPLICATION_PSD" "AUDIO_M4A" "VIDEO_REALMEDIA" "AUDIO_REALMEDIA" "AUDIO_TRUEAUDIO" "GENERAL_AUDIO" "GENERAL_IMAGE" "GENERAL_VIDEO" "GENERAL_APPLICATION" "GENERAL_ANIMATION" "APPLICATION_CLIP" "AUDIO_WAVE" "VIDEO_OGV" "AUDIO_MKV" "AUDIO_MP4" "UNDETERMINED_MP4" "APPLICATION_CBOR" "APPLICATION_WINDOWS_EXE" "AUDIO_WAVPACK" "APPLICATION_SAI2" "APPLICATION_KRITA" "IMAGE_SVG" "APPLICATION_XCF" "APPLICATION_GZIP" "GENERAL_APPLICATION_ARCHIVE" "GENERAL_IMAGE_PROJECT" "IMAGE_HEIF" "IMAGE_HEIF_SEQUENCE" "IMAGE_HEIC" "IMAGE_HEIC_SEQUENCE" "IMAGE_AVIF" "IMAGE_AVIF_SEQUENCE" "UNDETERMINED_GIF" "IMAGE_GIF" "APPLICATION_PROCREATE" "IMAGE_QOI" "APPLICATION_EPUB" "APPLICATION_DJVU" "APPLICATION_CBZ" "ANIMATION_UGOIRA" "APPLICATION_RTF" "APPLICATION_DOCX" "APPLICATION_XLSX" "APPLICATION_PPTX" "UNDETERMINED_OLE" "APPLICATION_DOC" "APPLICATION_XLS" "APPLICATION_PPT" "ANIMATION_WEBP" "UNDETERMINED_WEBP" "IMAGE_JXL" "APPLICATION_OCTET_STREAM" "APPLICATION_UNKNOWN")
 
 # OUT_TARGET = file to write too
 

IDHAN/include/hydrus_constants.hpp

@@ -4,6 +4,7 @@
 #pragma once
 
 #include "hydrus/HydrusConstants_gen.hpp"
+#include "mime_type_map.hpp"
 
 namespace idhan::hydrus::hy_constants
 {
@@ -14,27 +15,11 @@ enum ServiceTypes
 	TAG_SERVICE = gen_constants::LOCAL_TAG
 };
 
-inline std::uint16_t simpleToHyType( SimpleMimeType type )
+inline std::uint16_t mimeToHyType( const std::string& mime_name )
 {
-	switch ( type )
-	{
-		default:
-			[[fallthrough]];
-		case SimpleMimeType::NONE:
-			[[fallthrough]];
-		case SimpleMimeType::IMAGE:
-			return gen_constants::GENERAL_IMAGE;
-		case SimpleMimeType::VIDEO:
-			return gen_constants::GENERAL_VIDEO;
-		case SimpleMimeType::ANIMATION:
-			return gen_constants::GENERAL_ANIMATION;
-		case SimpleMimeType::AUDIO:
-			return gen_constants::GENERAL_AUDIO;
-		case SimpleMimeType::ARCHIVE:
-			return gen_constants::GENERAL_APPLICATION_ARCHIVE;
-		case SimpleMimeType::IMAGE_PROJECT:
-			return gen_constants::GENERAL_IMAGE_PROJECT;
-	}
+	if ( auto itter = hy_type_mime.find( mime_name ); itter != hy_type_mime.end() ) return itter->second;
+
+	return gen_constants::GENERAL_IMAGE;
 }
 
 } // namespace idhan::hydrus::hy_constants

IDHAN/include/idhan/versions.hpp

@@ -0,0 +1,24 @@
+//
+// Created by kj16609 on 7/30/24.
+//
+
+#pragma once
+
+#define MAKE_IDHAN_VERSION( major, minor, patch ) int( ( major << 16 ) | ( minor < 8 ) | patch )
+
+#ifndef IDHAN_MAJOR_VERSION
+#error Major version must be specified for release builds
+#endif
+
+#ifndef IDHAN_MINOR_VERSION
+#error Minor version must be specified for release builds
+#endif
+
+#ifndef IDHAN_PATCH_VERSION
+#error Patch version must be specified for release builds
+#endif
+
+#define IDHAN_VERSION MAKE_IDHAN_VERSION( IDHAN_MAJOR_VERSION, IDHAN_MINOR_VERSION, IDHAN_PATCH_VERSION )
+
+#define IDHAN_BUILD_TIME __TIME__
+#define IDHAN_BUILD_DATE __DATE__

IDHAN/include/mime_type_map.hpp

@@ -0,0 +1,102 @@
+//
+// Created by kj16609 on 11/10/25.
+//
+#pragma once
+#include <string>
+#include <unordered_map>
+
+#include "hydrus/HydrusConstants_gen.hpp"
+
+static const std::unordered_map< std::string_view, int > hy_type_mime {
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_HYDRUS_CLIENT_COLLECTION },
+	{ "image/jpeg", idhan::hydrus::gen_constants::IMAGE_JPEG },
+	{ "image/png", idhan::hydrus::gen_constants::IMAGE_PNG },
+	{ "image/gif", idhan::hydrus::gen_constants::ANIMATION_GIF },
+	{ "image/bmp", idhan::hydrus::gen_constants::IMAGE_BMP },
+	{ "application/x-shockwave-flash", idhan::hydrus::gen_constants::APPLICATION_FLASH },
+	{ "application/yaml", idhan::hydrus::gen_constants::APPLICATION_YAML },
+	{ "image/x-icon", idhan::hydrus::gen_constants::IMAGE_ICON },
+	{ "text/html", idhan::hydrus::gen_constants::TEXT_HTML },
+	{ "video/x-flv", idhan::hydrus::gen_constants::VIDEO_FLV },
+	{ "application/pdf", idhan::hydrus::gen_constants::APPLICATION_PDF },
+	{ "application/zip", idhan::hydrus::gen_constants::APPLICATION_ZIP },
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_HYDRUS_ENCRYPTED_ZIP },
+	{ "audio/mpeg", idhan::hydrus::gen_constants::AUDIO_MP3 },
+	{ "video/mp4", idhan::hydrus::gen_constants::VIDEO_MP4 },
+	{ "audio/ogg", idhan::hydrus::gen_constants::AUDIO_OGG },
+	{ "audio/flac", idhan::hydrus::gen_constants::AUDIO_FLAC },
+	{ "audio/x-ms-wma", idhan::hydrus::gen_constants::AUDIO_WMA },
+	{ "video/x-ms-wmv", idhan::hydrus::gen_constants::VIDEO_WMV },
+	{ "unknown", idhan::hydrus::gen_constants::UNDETERMINED_WM },
+	{ "video/x-matroska", idhan::hydrus::gen_constants::VIDEO_MKV },
+	{ "video/webm", idhan::hydrus::gen_constants::VIDEO_WEBM },
+	{ "application/json", idhan::hydrus::gen_constants::APPLICATION_JSON },
+	{ "image/apng", idhan::hydrus::gen_constants::ANIMATION_APNG },
+	{ "unknown", idhan::hydrus::gen_constants::UNDETERMINED_PNG },
+	{ "video/mpeg", idhan::hydrus::gen_constants::VIDEO_MPEG },
+	{ "video/quicktime", idhan::hydrus::gen_constants::VIDEO_MOV },
+	{ "video/x-msvideo", idhan::hydrus::gen_constants::VIDEO_AVI },
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_HYDRUS_UPDATE_DEFINITIONS },
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_HYDRUS_UPDATE_CONTENT },
+	{ "text/plain", idhan::hydrus::gen_constants::TEXT_PLAIN },
+	{ "application/x-rar-compressed", idhan::hydrus::gen_constants::APPLICATION_RAR },
+	{ "application/x-7z-compressed", idhan::hydrus::gen_constants::APPLICATION_7Z },
+	{ "image/webp", idhan::hydrus::gen_constants::IMAGE_WEBP },
+	{ "image/tiff", idhan::hydrus::gen_constants::IMAGE_TIFF },
+	{ "application/psd", idhan::hydrus::gen_constants::APPLICATION_PSD },
+	{ "audio/mp4", idhan::hydrus::gen_constants::AUDIO_M4A },
+	{ "video/x-pn-realvideo", idhan::hydrus::gen_constants::VIDEO_REALMEDIA },
+	{ "audio/x-pn-realaudio", idhan::hydrus::gen_constants::AUDIO_REALMEDIA },
+	{ "audio/x-tta", idhan::hydrus::gen_constants::AUDIO_TRUEAUDIO },
+	{ "audio/*", idhan::hydrus::gen_constants::GENERAL_AUDIO },
+	{ "image/*", idhan::hydrus::gen_constants::GENERAL_IMAGE },
+	{ "video/*", idhan::hydrus::gen_constants::GENERAL_VIDEO },
+	{ "application/*", idhan::hydrus::gen_constants::GENERAL_APPLICATION },
+	{ "unknown", idhan::hydrus::gen_constants::GENERAL_ANIMATION },
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_CLIP },
+	{ "audio/x-wav", idhan::hydrus::gen_constants::AUDIO_WAVE },
+	{ "video/ogg", idhan::hydrus::gen_constants::VIDEO_OGV },
+	{ "audio/x-matroska", idhan::hydrus::gen_constants::AUDIO_MKV },
+	{ "audio/mp4", idhan::hydrus::gen_constants::AUDIO_MP4 },
+	{ "unknown", idhan::hydrus::gen_constants::UNDETERMINED_MP4 },
+	{ "application/cbor", idhan::hydrus::gen_constants::APPLICATION_CBOR },
+	{ "application/x-msdownload", idhan::hydrus::gen_constants::APPLICATION_WINDOWS_EXE },
+	{ "audio/x-wavpack", idhan::hydrus::gen_constants::AUDIO_WAVPACK },
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_SAI2 },
+	{ "application/x-krita", idhan::hydrus::gen_constants::APPLICATION_KRITA },
+	{ "image/svg+xml", idhan::hydrus::gen_constants::IMAGE_SVG },
+	{ "image/x-xcf", idhan::hydrus::gen_constants::APPLICATION_XCF },
+	{ "application/gzip", idhan::hydrus::gen_constants::APPLICATION_GZIP },
+	{ "application/x-archive", idhan::hydrus::gen_constants::GENERAL_APPLICATION_ARCHIVE },
+	{ "unknown", idhan::hydrus::gen_constants::GENERAL_IMAGE_PROJECT },
+	{ "image/heif", idhan::hydrus::gen_constants::IMAGE_HEIF },
+	{ "image/heif-sequence", idhan::hydrus::gen_constants::IMAGE_HEIF_SEQUENCE },
+	{ "image/heic", idhan::hydrus::gen_constants::IMAGE_HEIC },
+	{ "image/heic-sequence", idhan::hydrus::gen_constants::IMAGE_HEIC_SEQUENCE },
+	{ "image/avif", idhan::hydrus::gen_constants::IMAGE_AVIF },
+	{ "image/avif-sequence", idhan::hydrus::gen_constants::IMAGE_AVIF_SEQUENCE },
+	{ "unknown", idhan::hydrus::gen_constants::UNDETERMINED_GIF },
+	{ "image/gif", idhan::hydrus::gen_constants::IMAGE_GIF },
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_PROCREATE },
+	{ "image/qoi", idhan::hydrus::gen_constants::IMAGE_QOI },
+	{ "application/epub+zip", idhan::hydrus::gen_constants::APPLICATION_EPUB },
+	{ "image/vnd.djvu", idhan::hydrus::gen_constants::APPLICATION_DJVU },
+	{ "application/vnd.comicbook+zip", idhan::hydrus::gen_constants::APPLICATION_CBZ },
+	{ "unknown", idhan::hydrus::gen_constants::ANIMATION_UGOIRA },
+	{ "application/rtf", idhan::hydrus::gen_constants::APPLICATION_RTF },
+	{ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+	  idhan::hydrus::gen_constants::APPLICATION_DOCX },
+	{ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+	  idhan::hydrus::gen_constants::APPLICATION_XLSX },
+	{ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+	  idhan::hydrus::gen_constants::APPLICATION_PPTX },
+	{ "unknown", idhan::hydrus::gen_constants::UNDETERMINED_OLE },
+	{ "application/msword", idhan::hydrus::gen_constants::APPLICATION_DOC },
+	{ "application/vnd.ms-excel", idhan::hydrus::gen_constants::APPLICATION_XLS },
+	{ "application/vnd.ms-powerpoint", idhan::hydrus::gen_constants::APPLICATION_PPT },
+	{ "image/webp", idhan::hydrus::gen_constants::ANIMATION_WEBP },
+	{ "unknown", idhan::hydrus::gen_constants::UNDETERMINED_WEBP },
+	{ "image/jxl", idhan::hydrus::gen_constants::IMAGE_JXL },
+	{ "application/octet-stream", idhan::hydrus::gen_constants::APPLICATION_OCTET_STREAM },
+	{ "unknown", idhan::hydrus::gen_constants::APPLICATION_UNKNOWN }
+};

IDHANClient/include/idhan/IDHANClient.hpp

@@ -143,6 +143,18 @@ class IDHANClient
 		TagDomainID tag_domain_id,
 		std::vector< std::vector< std::pair< std::string, std::string > > >&& tag_sets );
 
+	// File relationships
+	QFuture< void > setAlternativeGroups( std::vector< RecordID >& record_ids );
+
+	QFuture< void > setDuplicates( RecordID worse_duplicate, RecordID better_duplicate );
+
+	/**
+	 *
+	 * @param pairs Pairs of ids in a (worse_id, better_id) format
+	 * @return
+	 */
+	QFuture< void > setDuplicates( const std::vector< std::pair< RecordID, RecordID > >& pairs );
+
 	/**
 	 * @brief Creates a parent/child relationship between two tags
 	 * @param parent_id
@@ -238,6 +250,14 @@ class IDHANClient
 		std::uint16_t ratio,
 		bool readonly );
 
+	QFuture< void > addUrls( RecordID record_id, const std::vector< std::string >& urls );
+
+	inline QFuture< void > addUrl( const RecordID record_id, std::string url )
+	{
+		std::vector< std::string > urls { { url } };
+		return addUrls( record_id, urls );
+	}
+
   private:
 
 	void sendClientGet( UrlVariant url, IDHANResponseHandler&& responseHandler, IDHANErrorHandler&& errorHandler );

IDHANClient/src/IDHANClient.cpp

@@ -200,68 +200,70 @@ void IDHANClient::sendClientJson(
 
 	const auto submit_time { std::chrono::high_resolution_clock::now() };
 
-	QObject::connect(
-		response,
-		&QNetworkReply::finished,
-		[ responseHandler, response, submit_time ]()
+	auto responseSlot = [ responseHandler, response, submit_time ]()
+	{
+		const auto response_in_time { std::chrono::high_resolution_clock::now() };
+
+		if ( const auto response_time = response_in_time - submit_time; response_time > std::chrono::seconds( 5 ) )
 		{
-			const auto response_in_time { std::chrono::high_resolution_clock::now() };
+			logging::warn(
+				"Server took {}ms to response to query {}. Might be doing a lot of work?",
+				std::chrono::duration_cast< std::chrono::milliseconds >( response_time ).count(),
+				response->url().path().toStdString() );
+		}
 
-			if ( const auto response_time = response_in_time - submit_time; response_time > std::chrono::seconds( 5 ) )
-			{
-				logging::warn(
-					"Server took {}ms to response to query {}. Might be doing a lot of work?",
-					std::chrono::duration_cast< std::chrono::milliseconds >( response_time ).count(),
-					response->url().path().toStdString() );
-			}
+		if ( response->error() != QNetworkReply::NoError ) return;
 
-			if ( response->error() != QNetworkReply::NoError ) return;
+		QThreadPool::globalInstance()->start( std::bind( responseHandler, response ) );
+	};
 
-			QThreadPool::globalInstance()->start( std::bind( responseHandler, response ) );
-			// responseHandler( response );
-			// response->deleteLater();
-		} );
-
-	QObject::connect(
-		response,
-		&QNetworkReply::errorOccurred,
-		[ errorHandler, response, url ]( const QNetworkReply::NetworkError error )
+	auto errorSlot = [ errorHandler, response, url ]( const QNetworkReply::NetworkError error )
+	{
+		if ( error == QNetworkReply::NetworkError::OperationCanceledError )
 		{
-			if ( error == QNetworkReply::NetworkError::OperationCanceledError )
-			{
-				logging::critical(
-					"Operation timed out with request to {}: {}",
-					url.toString(),
-					response->errorString().toStdString() );
-				std::abort();
-			}
+			logging::critical(
+				"Operation timed out with request to {}: {}", url.toString(), response->errorString().toStdString() );
+			std::abort();
+		}
 
-			// check if this is a special error or not.
-			// It should have json if so
-			auto header = response->header( QNetworkRequest::ContentTypeHeader );
-			if ( header.isValid() && header.toString().contains( "application/json" ) )
+		// check if this is a special error or not.
+		// It should have json if so
+		auto header = response->header( QNetworkRequest::ContentTypeHeader );
+		if ( header.isValid() && header.toString().contains( "application/json" ) )
+		{
+			const auto response_body { response->readAll() };
+			QJsonDocument response_doc { QJsonDocument::fromJson( response_body ) };
+			if ( response_doc.isObject() )
 			{
-				const auto response_body { response->readAll() };
-				QJsonDocument response_doc { QJsonDocument::fromJson( response_body ) };
-				if ( response_doc.isObject() )
+				QJsonObject response_object { response_doc.object() };
+				if ( response_object.contains( "error" ) )
 				{
-					QJsonObject response_object { response_doc.object() };
-					if ( response_object.contains( "error" ) )
-					{
-						const auto error_msg { response_object[ "error" ].toString().toStdString() };
-						// logging::error( object[ "error" ].toString().toStdString() );
+					const auto error_msg { response_object[ "error" ].toString().toStdString() };
+					// logging::error( object[ "error" ].toString().toStdString() );
 
-						QThreadPool::globalInstance()->start( std::bind( errorHandler, response, error, error_msg ) );
-						return;
-					}
+					QThreadPool::globalInstance()->start( std::bind( errorHandler, response, error, error_msg ) );
+					return;
 				}
 			}
+		}
 
-			QThreadPool::globalInstance()->start(
-				std::bind( errorHandler, response, error, response->errorString().toStdString() ) );
-			// errorHandler( response, error );
-			// response->deleteLater();
-		} );
+		QThreadPool::globalInstance()->start(
+			std::bind( errorHandler, response, error, response->errorString().toStdString() ) );
+	};
+
+	const auto response_connection { QObject::connect( response, &QNetworkReply::finished, responseSlot ) };
+	const auto error_connection { QObject::connect( response, &QNetworkReply::errorOccurred, errorSlot ) };
+
+	if ( response->isFinished() )
+	{
+		response->disconnect( response_connection );
+		response->disconnect( error_connection );
+
+		if ( response->error() == QNetworkReply::NoError )
+			responseSlot();
+		else
+			errorSlot( response->error() );
+	}
 }
 
 QFuture< void > IDHANClient::createFileCluster(

IDHANClient/src/records/addUrls.cpp

@@ -0,0 +1,54 @@
+//
+// Created by kj16609 on 11/8/25.
+//
+
+#include <QFuture>
+#include <QJsonArray>
+#include <QJsonObject>
+
+#include "idhan/IDHANClient.hpp"
+
+namespace idhan
+{
+
+QFuture< void > IDHANClient::addUrls( const RecordID record_id, const std::vector< std::string >& urls )
+{
+	if ( urls.empty() ) return QtFuture::makeReadyVoidFuture();
+
+	QJsonObject json {};
+
+	QJsonArray array {};
+
+	for ( const auto& url : urls )
+	{
+		array.append( QString::fromStdString( url ) );
+	}
+
+	json[ "urls" ] = array;
+
+	auto promise = std::make_shared< QPromise< void > >();
+
+	const auto url { format_ns::format( "/records/{}/urls/add", record_id ) };
+
+	auto handleResponse = [ promise ]( QNetworkReply* response )
+	{
+		const auto data = response->readAll();
+		if ( !response->isFinished() )
+		{
+			logging::info( "Failed to read response" );
+			throw std::runtime_error( "Failed to read response" );
+		}
+
+		promise->finish();
+		response->deleteLater();
+	};
+
+	QJsonDocument doc {};
+	doc.setObject( json );
+
+	sendClientPost( std::move( doc ), url.data(), handleResponse, defaultErrorHandler( promise ) );
+
+	return promise->future();
+}
+
+} // namespace idhan

IDHANClient/src/records/relationships/setAlternatives.cpp

@@ -0,0 +1,46 @@
+//
+// Created by kj16609 on 11/10/25.
+//
+#include <QFuture>
+#include <QJsonArray>
+#include <QJsonObject>
+
+#include "IDHANTypes.hpp"
+#include "idhan/IDHANClient.hpp"
+
+namespace idhan
+{
+
+QFuture< void > IDHANClient::setAlternativeGroups( std::vector< RecordID >& record_ids )
+{
+	if ( record_ids.empty() ) return QtFuture::makeReadyVoidFuture();
+
+	QJsonArray record_array {};
+
+	for ( const auto& record : record_ids ) record_array.append( record );
+
+	auto promise { std::make_shared< QPromise< void > >() };
+
+	auto handleResponse = [ promise ]( QNetworkReply* reply ) -> void
+	{
+		const auto data = reply->readAll();
+		if ( !reply->isFinished() )
+		{
+			logging::info( "Failed to read response" );
+			throw std::runtime_error( "Failed to read response" );
+		}
+
+		promise->finish();
+		reply->deleteLater();
+	};
+
+	QJsonDocument doc {};
+	doc.setArray( record_array );
+
+	sendClientPost(
+		std::move( doc ), "/relationships/alternatives/add", handleResponse, defaultErrorHandler( promise ) );
+
+	return promise->future();
+}
+
+} // namespace idhan

IDHANClient/src/records/relationships/setDuplicates.cpp

@@ -0,0 +1,57 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+
+#include <QFuture>
+#include <QJsonArray>
+#include <QJsonObject>
+
+#include "IDHANTypes.hpp"
+#include "idhan/IDHANClient.hpp"
+
+namespace idhan
+{
+
+QFuture< void > IDHANClient::setDuplicates( const RecordID worse_duplicate, const RecordID better_duplicate )
+{
+	const std::vector< std::pair< RecordID, RecordID > > duplicates { { worse_duplicate, better_duplicate } };
+	return setDuplicates( duplicates );
+}
+
+QFuture< void > IDHANClient::setDuplicates( const std::vector< std::pair< RecordID, RecordID > >& pairs )
+{
+	QJsonArray json_pairs {};
+
+	for ( const auto& [ worse_id, better_id ] : pairs )
+	{
+		QJsonObject json_pair {};
+		json_pair[ "worse_id" ] = worse_id;
+		json_pair[ "better_id" ] = better_id;
+
+		json_pairs.append( json_pair );
+	}
+
+	auto promise { std::make_shared< QPromise< void > >() };
+
+	auto handleResponse = [ promise ]( QNetworkReply* response ) -> void
+	{
+		const auto data = response->readAll();
+		if ( !response->isFinished() )
+		{
+			logging::info( "Failed to read response" );
+			throw std::runtime_error( "Failed to read response" );
+		}
+
+		promise->finish();
+		response->deleteLater();
+	};
+
+	QJsonDocument doc {};
+	doc.setArray( std::move( json_pairs ) );
+
+	sendClientPost( std::move( doc ), "/relationships/duplicates/add", handleResponse, defaultErrorHandler( promise ) );
+
+	return promise->future();
+}
+
+} // namespace idhan

IDHANMigration/src/130-alternative_files.sql

@@ -0,0 +1,53 @@
+CREATE TABLE alternative_groups
+(
+    group_id SERIAL PRIMARY KEY NOT NULL
+);
+
+CREATE TABLE alternative_group_members
+(
+    group_id  INTEGER REFERENCES alternative_groups (group_id) NOT NULL,
+    record_id INTEGER REFERENCES records (record_id) UNIQUE    NOT NULL,
+    UNIQUE (group_id, record_id)
+);
+
+CREATE INDEX ON alternative_group_members (group_id);
+
+CREATE TABLE duplicate_pairs
+(
+    worse_record_id  INTEGER REFERENCES records (record_id) UNIQUE NOT NULL,
+    better_record_id INTEGER REFERENCES records (record_id)        NOT NULL,
+    CHECK (worse_record_id != better_record_id)
+);
+
+CREATE INDEX ON duplicate_pairs (better_record_id);
+
+CREATE OR REPLACE FUNCTION insert_duplicate_pair(
+    worse INTEGER,
+    better INTEGER
+) RETURNS VOID AS
+$$
+BEGIN
+
+    IF (EXISTS(SELECT 1 FROM duplicate_pairs WHERE worse_record_id = worse AND better_record_id = better)) THEN
+        RETURN;
+    END IF;
+
+    -- check that the worse id isn't being added again
+    IF (EXISTS(SELECT 1 FROM duplicate_pairs WHERE worse_record_id = worse)) THEN
+        RAISE EXCEPTION 'Can''t insert an already inserted worse record';
+    END IF;
+
+    -- Check that the worse record wouldn't make a cyclic chain
+    IF (EXISTS(SELECT 1 FROM duplicate_pairs WHERE better_record_id = worse AND worse_record_id = better)) THEN
+        RAISE EXCEPTION 'Inserting this duplicate pair would result in a cyclic chain';
+    end if;
+
+    INSERT INTO duplicate_pairs (worse_record_id, better_record_id)
+    VALUES (worse, better);
+
+    UPDATE duplicate_pairs
+    SET better_record_id = better
+    WHERE better_record_id = worse;
+END;
+
+$$ LANGUAGE plpgsql;

IDHANMigration/src/135-video_metadata.sql

@@ -0,0 +1,11 @@
+CREATE TABLE video_metadata
+(
+    record_id INTEGER REFERENCES records (record_id),
+    duration  FLOAT   NOT NULL,
+    framerate FLOAT   NOT NULL,
+    width     INTEGER NOT NULL,
+    height    INTEGER NOT NULL,
+    bitrate   INTEGER NOT NULL,
+    has_audio BOOLEAN NOT NULL,
+    UNIQUE (record_id)
+);

IDHANMigration/src/140-image_project_metadata.sql

@@ -0,0 +1,9 @@
+CREATE TABLE image_project_metadata
+(
+    record_id INTEGER REFERENCES records (record_id),
+    width     INTEGER  NOT NULL,
+    height    INTEGER  NOT NULL,
+    channels  SMALLINT NOT NULL,
+    layers    SMALLINT NOT NULL,
+    UNIQUE (record_id)
+);

IDHANModules/CMakeLists.txt

@@ -4,11 +4,27 @@ target_link_libraries(IDHANModules PUBLIC IDHAN)
 include(ExternalProject)
 
 find_package(PkgConfig REQUIRED)
+
+# VideoMetadata & VideoThumbnailer
+pkg_check_modules(FFMPEG REQUIRED IMPORTED_TARGET
+		libavcodec
+		libavformat
+		libavutil
+		libswscale)
+
+# ImageVipsMetadata & ImageVipsThumbnailer
 pkg_check_modules(VIPS REQUIRED vips)
 
 AddFGLModule(IDHANPremadeModules ${CMAKE_CURRENT_SOURCE_DIR}/premade)
-target_link_libraries(IDHANPremadeModules PUBLIC IDHANModules IDHAN ${VIPS_LIBRARIES} spdlog)
+target_link_libraries(IDHANPremadeModules PUBLIC IDHANModules IDHAN spdlog)
+
+# Vips
+target_link_libraries(IDHANPremadeModules PUBLIC ${VIPS_LIBRARIES})
 target_include_directories(IDHANPremadeModules PRIVATE ${VIPS_INCLUDE_DIRS})
+#target_link_libraries(IDHANPremadeModules PRIVATE PkgConfig::VIPS)
+
+# Video (FFMPEG)
+target_link_libraries(IDHANPremadeModules PRIVATE PkgConfig::FFMPEG)
 
 set_target_properties(IDHANPremadeModules PROPERTIES
 		RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin/modules"

IDHANModules/include/MetadataModule.hpp

@@ -12,19 +12,40 @@
 
 namespace idhan
 {
+
 struct MetadataInfoImage
 {
-	int width;
-	int height;
-	std::uint8_t channels;
+	int width { 0 };
+	int height { 0 };
+	std::uint8_t channels { 0 };
 };
 
 struct MetadataInfoAnimation
 {};
 
+struct MetadataInfoImageProject
+{
+	MetadataInfoImage image_info {};
+	std::uint8_t layers { 0 };
+};
+
+struct MetadataInfoVideo
+{
+	bool m_has_audio { false };
+	int m_width { 0 };
+	int m_height { 0 };
+	int m_bitrate { 0 };
+	double m_duration { 0.0 };
+	double m_fps { 0.0 };
+};
+
+using MetadataVariant = std::
+	variant< std::monostate, MetadataInfoImage, MetadataInfoVideo, MetadataInfoImageProject, MetadataInfoAnimation >;
+
 struct MetadataInfo
 {
-	std::variant< std::monostate, MetadataInfoImage, MetadataInfoAnimation > m_metadata {};
+	MetadataVariant m_metadata {};
+
 	std::string m_extra {};
 	SimpleMimeType m_simple_type { SimpleMimeType::NONE };
 };
@@ -40,7 +61,7 @@ class FGL_EXPORT MetadataModuleI : public ModuleBase
 	virtual std::vector< std::string_view > handleableMimes() = 0;
 
 	virtual std::expected< MetadataInfo, ModuleError > parseFile(
-		void* data,
+		const void* data,
 		std::size_t length,
 		std::string mime_name ) = 0;
 

IDHANModules/include/ThumbnailerModule.hpp

@@ -28,7 +28,7 @@ class FGL_EXPORT ThumbnailerModuleI : public ModuleBase
 	virtual std::vector< std::string_view > handleableMimes() = 0;
 
 	virtual std::expected< ThumbnailInfo, ModuleError > createThumbnail(
-		void* data,
+		const void* data,
 		std::size_t length,
 		std::size_t width,
 		std::size_t height,

IDHANModules/premade/FFMPEGMetadata.cpp

@@ -0,0 +1,128 @@
+//
+// Created by kj16609 on 11/12/25.
+//
+#include "FFMPEGMetadata.hpp"
+
+#include <cstring>
+#include <iostream>
+#include <memory>
+
+#include "ffmpeg.hpp"
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libswscale/swscale.h>
+}
+
+std::string_view FFMPEGMetadata::name()
+{
+	return "Video Metadata Module";
+}
+
+idhan::ModuleVersion FFMPEGMetadata::version()
+{
+	return { .m_major = 1, .m_minor = 0, .m_patch = 0 };
+}
+
+std::vector< std::string_view > FFMPEGMetadata::handleableMimes()
+{
+	return ffmpeg_handleable_mimes;
+}
+
+std::expected< idhan::MetadataInfo, idhan::ModuleError > FFMPEGMetadata::parseFile(
+	const void* data,
+	std::size_t length,
+	std::string mime_name )
+{
+	idhan::MetadataInfo base_info {};
+	base_info.m_simple_type = idhan::SimpleMimeType::VIDEO;
+	idhan::MetadataInfoVideo video_metadata {};
+
+	OpaqueInfo opaque_info { .m_data = std::string_view( static_cast< const char* >( data ), length ), .m_cursor = 0 };
+
+	constexpr auto BUFFER_SIZE { 4096 };
+	// std::array< std::byte, BUFFER_SIZE > buffer {};
+	std::byte* buffer_ptr { new std::byte[ BUFFER_SIZE ] };
+
+	const std::shared_ptr< AVIOContext > avio_context(
+		avio_alloc_context(
+			reinterpret_cast< unsigned char* >( buffer_ptr ),
+			BUFFER_SIZE,
+			0,
+			&opaque_info,
+			&readFunction,
+			nullptr,
+			seekFunction ),
+		&av_free );
+
+	const auto format_context_p =
+		std::shared_ptr< AVFormatContext >( avformat_alloc_context(), &avformat_free_context );
+	auto format_context { format_context_p.get() };
+
+	format_context->pb = avio_context.get();
+	format_context->flags |= AVFMT_FLAG_CUSTOM_IO;
+
+	if ( avformat_open_input( &format_context, "", nullptr, nullptr ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to open file" ) );
+	}
+
+	if ( avformat_find_stream_info( format_context, nullptr ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to find stream info" ) );
+	}
+
+	bool has_video { false };
+	bool has_audio { false };
+
+	for ( unsigned int i = 0; i < format_context->nb_streams; ++i )
+	{
+		AVStream* stream = format_context->streams[ i ];
+		AVCodecParameters* codecpar = stream->codecpar;
+
+		if ( codecpar->codec_type == AVMEDIA_TYPE_VIDEO && !has_video )
+		{
+			has_video = true;
+			video_metadata.m_width = codecpar->width;
+			video_metadata.m_height = codecpar->height;
+
+			// Calculate bitrate (prefer codec bitrate, fall back to container bitrate)
+			if ( codecpar->bit_rate > 0 )
+			{
+				video_metadata.m_bitrate = codecpar->bit_rate;
+			}
+			else if ( format_context->bit_rate > 0 )
+			{
+				// Fall back to container bitrate if no stream-specific bitrate
+				video_metadata.m_bitrate = format_context->bit_rate;
+			}
+
+			// Get duration (in seconds)
+			if ( stream->duration != AV_NOPTS_VALUE )
+			{
+				video_metadata.m_duration = static_cast< double >( stream->duration ) * av_q2d( stream->time_base );
+			}
+			else if ( format_context->duration != AV_NOPTS_VALUE )
+			{
+				video_metadata.m_duration = static_cast< double >( format_context->duration ) / AV_TIME_BASE;
+			}
+
+			// Get frame rate
+			if ( stream->avg_frame_rate.den > 0 )
+			{
+				video_metadata.m_fps = av_q2d( stream->avg_frame_rate );
+			}
+		}
+		else if ( codecpar->codec_type == AVMEDIA_TYPE_AUDIO )
+		{
+			has_audio = true;
+		}
+	}
+
+	video_metadata.m_has_audio = has_audio;
+
+	base_info.m_metadata = video_metadata;
+
+	return base_info;
+}

IDHANModules/premade/FFMPEGMetadata.hpp

@@ -0,0 +1,22 @@
+//
+// Created by kj16609 on 11/12/25.
+//
+#pragma once
+#include "MetadataModule.hpp"
+
+
+class FFMPEGMetadata final : public idhan::MetadataModuleI
+{
+  public:
+
+	std::string_view name() override;
+
+	idhan::ModuleVersion version() override;
+
+	std::vector< std::string_view > handleableMimes() override;
+
+	std::expected< idhan::MetadataInfo, idhan::ModuleError > parseFile(
+		const void* data,
+		std::size_t length,
+		std::string mime_name ) override;
+};

IDHANModules/premade/FFMPEGThumbnailer.cpp

@@ -0,0 +1,346 @@
+//
+// Created by kj16609 on 11/13/25.
+//
+#include "FFMPEGThumbnailer.hpp"
+
+#include <vips/vips8>
+
+#include <cstring>
+#include <memory>
+#include <vector>
+
+#include "ffmpeg.hpp"
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavformat/avio.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/mem.h>
+#include <libswscale/swscale.h>
+}
+
+// Custom deleters for FFmpeg types
+struct AVFormatContextDeleter
+{
+	void operator()( AVFormatContext* ctx ) const
+	{
+		if ( ctx )
+		{
+			avformat_close_input( &ctx );
+			avformat_free_context( ctx );
+		}
+	}
+};
+
+struct AVCodecContextDeleter
+{
+	void operator()( AVCodecContext* ctx ) const
+	{
+		if ( ctx )
+		{
+			avcodec_free_context( &ctx );
+		}
+	}
+};
+
+struct AVPacketDeleter
+{
+	void operator()( AVPacket* pkt ) const
+	{
+		if ( pkt )
+		{
+			av_packet_free( &pkt );
+		}
+	}
+};
+
+struct AVFrameDeleter
+{
+	void operator()( AVFrame* frame ) const
+	{
+		if ( frame )
+		{
+			av_frame_free( &frame );
+		}
+	}
+};
+
+struct SwsContextDeleter
+{
+	void operator()( SwsContext* ctx ) const
+	{
+		if ( ctx )
+		{
+			sws_freeContext( ctx );
+		}
+	}
+};
+
+struct AVIOContextDeleter
+{
+	void operator()( AVIOContext* ctx ) const
+	{
+		if ( ctx )
+		{
+			av_free( ctx );
+		}
+	}
+};
+
+struct VipsImageDeleter
+{
+	void operator()( VipsImage* img ) const
+	{
+		if ( img )
+		{
+			g_object_unref( img );
+		}
+	}
+};
+
+std::string_view FFMPEGThumbnailer::name()
+{
+	return "FFMPEGThumbnailer";
+}
+
+idhan::ModuleVersion FFMPEGThumbnailer::version()
+{
+	return idhan::ModuleVersion { .m_major = 0, .m_minor = 1, .m_patch = 0 };
+}
+
+std::vector< std::string_view > FFMPEGThumbnailer::handleableMimes()
+{
+	return ffmpeg_handleable_mimes;
+}
+
+std::expected< idhan::ThumbnailerModuleI::ThumbnailInfo, idhan::ModuleError > FFMPEGThumbnailer::createThumbnail(
+	const void* data,
+	std::size_t length,
+	std::size_t width,
+	std::size_t height,
+	std::string mime_name )
+{
+	OpaqueInfo opaque_info { .m_data = std::string_view( static_cast< const char* >( data ), length ), .m_cursor = 0 };
+
+	constexpr auto BUFFER_SIZE { 4096 };
+	std::byte* buffer_ptr { new std::byte[ BUFFER_SIZE ] };
+
+	std::unique_ptr< AVIOContext, AVIOContextDeleter > avio_context( avio_alloc_context(
+		reinterpret_cast< unsigned char* >( buffer_ptr ),
+		BUFFER_SIZE,
+		0,
+		&opaque_info,
+		&readFunction,
+		nullptr,
+		seekFunction ) );
+
+	if ( !avio_context )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to allocate AVIO context" ) );
+	}
+
+	std::unique_ptr< AVFormatContext, AVFormatContextDeleter > format_context( avformat_alloc_context() );
+	if ( !format_context )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to allocate format context" ) );
+	}
+
+	format_context->pb = avio_context.get();
+	format_context->flags |= AVFMT_FLAG_CUSTOM_IO;
+
+	AVFormatContext* format_context_raw = format_context.get();
+	if ( avformat_open_input( &format_context_raw, "", nullptr, nullptr ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to open file" ) );
+	}
+	// avformat_open_input may reallocate the context, update our unique_ptr
+	format_context.release();
+	format_context.reset( format_context_raw );
+
+	if ( avformat_find_stream_info( format_context.get(), nullptr ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to find stream info" ) );
+	}
+
+	// Find video stream
+	int video_stream_index = -1;
+	AVCodecParameters* codec_params = nullptr;
+	for ( unsigned int i = 0; i < format_context->nb_streams; i++ )
+	{
+		if ( format_context->streams[ i ]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO )
+		{
+			video_stream_index = i;
+			codec_params = format_context->streams[ i ]->codecpar;
+			break;
+		}
+	}
+
+	if ( video_stream_index == -1 )
+	{
+		return std::unexpected( idhan::ModuleError( "No video stream found" ) );
+	}
+
+	// Calculate 10% duration and seek to it
+	const auto duration { av_rescale_q(
+		format_context->duration, AV_TIME_BASE_Q, format_context->streams[ video_stream_index ]->time_base ) };
+	const auto target_timestamp { static_cast< int64_t >( duration * 0.10 ) };
+
+	if ( av_seek_frame( format_context.get(), video_stream_index, target_timestamp, AVSEEK_FLAG_BACKWARD ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to seek to timestamp" ) );
+	}
+
+	// Find and open codec
+	const AVCodec* codec = avcodec_find_decoder( codec_params->codec_id );
+	if ( !codec )
+	{
+		return std::unexpected( idhan::ModuleError( "Codec not found" ) );
+	}
+
+	std::unique_ptr< AVCodecContext, AVCodecContextDeleter > codec_context( avcodec_alloc_context3( codec ) );
+	if ( !codec_context )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to allocate codec context" ) );
+	}
+
+	if ( avcodec_parameters_to_context( codec_context.get(), codec_params ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to copy codec params" ) );
+	}
+
+	if ( avcodec_open2( codec_context.get(), codec, nullptr ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to open codec" ) );
+	}
+
+	// Read frames until we get a valid video frame
+	const std::unique_ptr< AVPacket, AVPacketDeleter > packet( av_packet_alloc() );
+	const std::unique_ptr< AVFrame, AVFrameDeleter > frame( av_frame_alloc() );
+
+	if ( !packet || !frame )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to allocate packet or frame" ) );
+	}
+
+	bool frame_decoded = false;
+
+	while ( av_read_frame( format_context.get(), packet.get() ) >= 0 )
+	{
+		if ( packet->stream_index == video_stream_index )
+		{
+			if ( avcodec_send_packet( codec_context.get(), packet.get() ) >= 0 )
+			{
+				if ( avcodec_receive_frame( codec_context.get(), frame.get() ) >= 0 )
+				{
+					frame_decoded = true;
+					av_packet_unref( packet.get() );
+					break;
+				}
+			}
+		}
+		av_packet_unref( packet.get() );
+	}
+
+	if ( !frame_decoded )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to decode frame" ) );
+	}
+
+	// Convert frame to RGB24 using swscale
+	std::unique_ptr< SwsContext, SwsContextDeleter > sws_context( sws_getContext(
+		codec_context->width,
+		codec_context->height,
+		codec_context->pix_fmt,
+		codec_context->width,
+		codec_context->height,
+		AV_PIX_FMT_RGB24,
+		SWS_BILINEAR,
+		nullptr,
+		nullptr,
+		nullptr ) );
+
+	if ( !sws_context )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to create swscale context" ) );
+	}
+
+	std::unique_ptr< AVFrame, AVFrameDeleter > rgb_frame( av_frame_alloc() );
+	if ( !rgb_frame )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to allocate RGB frame" ) );
+	}
+
+	rgb_frame->format = AV_PIX_FMT_RGB24;
+	rgb_frame->width = codec_context->width;
+	rgb_frame->height = codec_context->height;
+
+	if ( av_frame_get_buffer( rgb_frame.get(), 0 ) < 0 )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to allocate RGB frame buffer" ) );
+	}
+
+	sws_scale(
+		sws_context.get(),
+		frame->data,
+		frame->linesize,
+		0,
+		codec_context->height,
+		rgb_frame->data,
+		rgb_frame->linesize );
+
+	// Create packed buffer without alignment padding
+	const size_t packed_line_size = rgb_frame->width * 3;
+	const size_t packed_size = packed_line_size * rgb_frame->height;
+	std::vector< unsigned char > packed_data( packed_size );
+
+	// Copy data line by line to remove padding
+	for ( int y = 0; y < rgb_frame->height; ++y )
+	{
+		std::memcpy(
+			packed_data.data() + ( y * packed_line_size ),
+			rgb_frame->data[ 0 ] + ( y * rgb_frame->linesize[ 0 ] ),
+			packed_line_size );
+	}
+
+	std::unique_ptr< VipsImage, VipsImageDeleter > image( vips_image_new_from_memory(
+		packed_data.data(), packed_size, rgb_frame->width, rgb_frame->height, 3, VIPS_FORMAT_UCHAR ) );
+
+	if ( !image )
+	{
+		return std::unexpected( idhan::ModuleError { "Failed to create VIPS image" } );
+	}
+
+	const float source_aspect { static_cast< float >( rgb_frame->width ) / static_cast< float >( rgb_frame->height ) };
+	const float target_aspect { static_cast< float >( width ) / static_cast< float >( height ) };
+
+	if ( target_aspect > source_aspect )
+		width = static_cast< std::size_t >( static_cast< float >( height ) * source_aspect );
+	else
+		height = static_cast< std::size_t >( static_cast< float >( width ) / source_aspect );
+
+	VipsImage* resized { nullptr };
+	if ( vips_resize(
+			 image.get(),
+			 &resized,
+			 static_cast< double >( width ) / static_cast< double >( vips_image_get_width( image.get() ) ),
+			 nullptr ) )
+	{
+		return std::unexpected( idhan::ModuleError { "Failed to resize image" } );
+	}
+	std::unique_ptr< VipsImage, VipsImageDeleter > resized_image( resized );
+
+	// Encode to PNG
+	void* output_buffer { nullptr };
+	std::size_t output_length { 0 };
+	if ( vips_pngsave_buffer( resized_image.get(), &output_buffer, &output_length, nullptr ) )
+	{
+		return std::unexpected( idhan::ModuleError( "Failed to encode PNG" ) );
+	}
+
+	const std::vector< std::byte > output(
+		static_cast< std::byte* >( output_buffer ), static_cast< std::byte* >( output_buffer ) + output_length );
+	g_free( output_buffer );
+
+	return idhan::ThumbnailerModuleI::ThumbnailInfo { .data = std::move( output ), .width = width, .height = height };
+}

IDHANModules/premade/FFMPEGThumbnailer.hpp

@@ -0,0 +1,24 @@
+//
+// Created by kj16609 on 11/13/25.
+//
+#pragma once
+
+#include "ThumbnailerModule.hpp"
+
+class FFMPEGThumbnailer final : public idhan::ThumbnailerModuleI
+{
+  public:
+
+	std::string_view name() override;
+
+	idhan::ModuleVersion version() override;
+
+	std::vector< std::string_view > handleableMimes() override;
+
+	std::expected< ThumbnailInfo, idhan::ModuleError > createThumbnail(
+		const void* data,
+		std::size_t length,
+		std::size_t width,
+		std::size_t height,
+		std::string mime_name ) override;
+};

IDHANModules/premade/ImageVipsMetadata.cpp

@@ -19,14 +19,14 @@ std::vector< std::string_view > ImageVipsMetadata::handleableMimes()
 }
 
 std::expected< MetadataInfo, ModuleError > ImageVipsMetadata::parseFile(
-	void* data,
+	const void* data,
 	const std::size_t length,
 	const std::string mime_name )
 {
 	VipsImage* image;
 	if ( const auto it = VIPS_FUNC_MAP.find( mime_name ); it != VIPS_FUNC_MAP.end() )
 	{
-		if ( it->second( data, length, &image, nullptr ) != 0 )
+		if ( it->second( const_cast< void* >( data ), length, &image, nullptr ) != 0 )
 		{
 			return std::unexpected( ModuleError { "Failed to load image" } );
 		}

IDHANModules/premade/ImageVipsMetadata.hpp

@@ -17,7 +17,7 @@ class ImageVipsMetadata final : public idhan::MetadataModuleI
 
 	std::vector< std::string_view > handleableMimes() override;
 	std::expected< idhan::MetadataInfo, idhan::ModuleError > parseFile(
-		void* data,
+		const void* data,
 		std::size_t length,
 		std::string mime_name ) override;
 

IDHANModules/premade/ImageVipsThumbnailer.cpp

@@ -17,7 +17,7 @@ std::vector< std::string_view > ImageVipsThumbnailer::handleableMimes()
 }
 
 std::expected< ThumbnailerModuleI::ThumbnailInfo, ModuleError > ImageVipsThumbnailer::createThumbnail(
-	void* data,
+	const void* data,
 	const std::size_t length,
 	std::size_t width,
 	std::size_t height,
@@ -26,7 +26,7 @@ std::expected< ThumbnailerModuleI::ThumbnailInfo, ModuleError > ImageVipsThumbna
 	VipsImage* image;
 	if ( const auto it = VIPS_FUNC_MAP.find( mime_name ); it != VIPS_FUNC_MAP.end() )
 	{
-		if ( it->second( data, length, &image, nullptr ) != 0 )
+		if ( it->second( const_cast< void* >( data ), length, &image, nullptr ) != 0 )
 		{
 			return std::unexpected( ModuleError { "Failed to load image" } );
 		}

IDHANModules/premade/ImageVipsThumbnailer.hpp

@@ -9,7 +9,7 @@ class ImageVipsThumbnailer : public idhan::ThumbnailerModuleI
 	std::vector< std::string_view > handleableMimes() override;
 
 	std::expected< ThumbnailInfo, idhan::ModuleError > createThumbnail(
-		void* data,
+		const void* data,
 		std::size_t length,
 		std::size_t width,
 		std::size_t height,

IDHANModules/premade/PsdMetadata.cpp

@@ -0,0 +1,592 @@
+//
+// Created by kj16609 on 11/12/25.
+//
+#include "PsdMetadata.hpp"
+
+#include <vips/vips8>
+
+#include <algorithm>
+#include <cstring>
+
+#include "ModuleBase.hpp"
+
+namespace
+{
+
+std::uint16_t readUint16BE( const std::uint8_t* data )
+{
+	return ( static_cast< std::uint16_t >( data[ 0 ] ) << 8 ) | ( static_cast< std::uint16_t >( data[ 1 ] ) << 0 );
+}
+
+std::uint32_t readUint32BE( const std::uint8_t* data )
+{
+	return ( static_cast< std::uint32_t >( data[ 0 ] ) << 24 ) | ( static_cast< std::uint32_t >( data[ 1 ] ) << 16 )
+	     | ( static_cast< std::uint32_t >( data[ 2 ] ) << 8 ) | ( static_cast< std::uint32_t >( data[ 3 ] ) << 0 );
+}
+
+float readFloat32BE( const std::uint8_t* data )
+{
+	std::uint32_t bits { readUint32BE( data ) };
+	return *reinterpret_cast< float* >( &bits );
+}
+
+struct PSDHeader
+{
+	std::uint16_t channels;
+	std::uint32_t height;
+	std::uint32_t width;
+	std::uint16_t depth;
+	std::uint16_t colorMode;
+};
+
+std::optional< PSDHeader > parsePSDHeader( const std::uint8_t* data, const std::size_t length )
+{
+	if ( length < 26 ) return std::nullopt;
+
+	if ( memcmp( data, "8BPS", 4 ) != 0 ) return std::nullopt;
+
+	const std::uint16_t version { readUint16BE( data + 4 ) };
+	if ( version != 1 ) return std::nullopt; // TODO: support v2 "large" format
+
+	return { {
+		.channels = readUint16BE( data + 12 ),
+		.height = readUint32BE( data + 14 ),
+		.width = readUint32BE( data + 18 ),
+		.depth = readUint16BE( data + 22 ),
+		.colorMode = readUint16BE( data + 24 ),
+	} };
+}
+
+void unpackScanline(
+	const std::uint8_t* buffer,
+	const std::size_t bufferLength,
+	std::uint8_t* output,
+	const std::size_t outputLength )
+{
+	std::size_t inputIdx { 0 };
+	std::size_t outputIdx { 0 };
+
+	while ( outputIdx < outputLength && inputIdx < bufferLength )
+	{
+		const std::uint8_t headerByte { buffer[ inputIdx++ ] };
+		if ( headerByte > 128 )
+		{
+			const std::uint16_t repeatCount { static_cast< std::uint16_t >( 257 - headerByte ) };
+			if ( inputIdx >= bufferLength ) break;
+			const std::uint8_t value { buffer[ inputIdx++ ] };
+			for ( std::uint16_t r = 0; r < repeatCount && outputIdx < outputLength; ++r ) output[ outputIdx++ ] = value;
+		}
+		else if ( headerByte < 128 )
+		{
+			const std::uint16_t literalLength { static_cast< std::uint16_t >( headerByte + 1 ) };
+			for ( std::uint16_t c = 0; c < literalLength && outputIdx < outputLength && inputIdx < bufferLength; ++c )
+				output[ outputIdx++ ] = buffer[ inputIdx++ ];
+		}
+	}
+}
+
+std::vector< std::uint8_t > unpackRaster(
+	const std::uint8_t* buffer,
+	std::size_t& offset,
+	const std::size_t dataLength,
+	const std::uint32_t width,
+	const std::uint32_t height,
+	const std::uint16_t channels )
+{
+	const std::size_t scanlineCountsSize { height * channels * 2 };
+	if ( offset + scanlineCountsSize > dataLength )
+	{
+		return {};
+	}
+
+	std::vector< std::uint16_t > scanlineLengths( height * channels );
+	for ( std::size_t i = 0; i < scanlineLengths.size(); ++i )
+	{
+		scanlineLengths[ i ] = readUint16BE( buffer + offset + i * 2 );
+	}
+	offset += scanlineCountsSize;
+
+	const std::size_t planeSize { static_cast< std::size_t >( width ) * height };
+	std::vector< std::uint8_t > planarData( planeSize * channels );
+
+	std::size_t outputOffset { 0 };
+	for ( const std::uint16_t scanlineLength : scanlineLengths )
+	{
+		const std::uint16_t compressedLength { scanlineLength };
+
+		if ( offset + compressedLength > dataLength )
+		{
+			return {};
+		}
+
+		unpackScanline( buffer + offset, compressedLength, planarData.data() + outputOffset, width );
+
+		offset += compressedLength;
+		outputOffset += width;
+	}
+
+	return planarData;
+}
+
+std::vector< std::uint8_t > convert16to8bit( const std::vector< std::uint8_t >& buffer, const std::size_t pixelCount )
+{
+	if ( buffer.size() < pixelCount * 2 )
+	{
+		return {};
+	}
+	std::vector< std::uint8_t > result( pixelCount );
+	for ( std::size_t i = 0; i < pixelCount; ++i )
+		result[ i ] = static_cast< std::uint8_t >( readUint16BE( &buffer[ i * 2 ] ) >> 8 );
+	return result;
+}
+
+std::vector< std::uint8_t > convert32to8bit( const std::vector< std::uint8_t >& buffer, const std::size_t pixelCount )
+{
+	if ( buffer.size() < pixelCount * 4 )
+	{
+		return {};
+	}
+	std::vector< std::uint8_t > result( pixelCount );
+	for ( std::size_t i = 0; i < pixelCount; ++i )
+		result[ i ] =
+			static_cast< std::uint8_t >( std::clamp( readFloat32BE( &buffer[ i * 4 ] ), 0.0f, 1.0f ) * 255.0f );
+	return result;
+}
+
+std::vector< std::uint8_t > convertToTargetDepth(
+	const std::vector< std::uint8_t >& buffer,
+	const std::uint16_t depth,
+	const std::size_t pixelCount )
+{
+	switch ( depth )
+	{
+		case 8:
+			return buffer;
+		case 16:
+			return convert16to8bit( buffer, pixelCount );
+		case 32:
+			return convert32to8bit( buffer, pixelCount );
+		default:
+			return {};
+	}
+}
+
+std::expected< std::vector< std::uint8_t >, idhan::ModuleError > convertCMYKtoInterleavedRGB(
+	const std::basic_string_view< std::uint8_t > cmyk,
+	const std::size_t pixelCount )
+{
+	std::vector< std::uint8_t > rgb( pixelCount * 3 );
+	for ( std::size_t i = 0; i < pixelCount; ++i )
+	{
+		const std::uint8_t c { cmyk[ ( pixelCount * 0 ) + i ] };
+		const std::uint8_t m { cmyk[ ( pixelCount * 1 ) + i ] };
+		const std::uint8_t y { cmyk[ ( pixelCount * 2 ) + i ] };
+		const std::uint8_t k { cmyk[ ( pixelCount * 3 ) + i ] };
+		// Obviously, this does not take ICC profiles into account, but hopefully it is a good enough first approximation.
+		rgb[ i * 3 + 0 ] = static_cast< std::uint8_t >( ( 255 - c ) * ( 255 - k ) / 255 );
+		rgb[ i * 3 + 1 ] = static_cast< std::uint8_t >( ( 255 - m ) * ( 255 - k ) / 255 );
+		rgb[ i * 3 + 2 ] = static_cast< std::uint8_t >( ( 255 - y ) * ( 255 - k ) / 255 );
+	}
+	return rgb;
+}
+
+std::expected< std::vector< std::uint8_t >, idhan::ModuleError > convertGrayscaleToInterleavedRGB(
+	const std::basic_string_view< std::uint8_t > gray,
+	const std::size_t pixelCount )
+{
+	std::vector< std::uint8_t > rgb( pixelCount * 3 );
+	for ( std::size_t i = 0; i < pixelCount; ++i )
+	{
+		const std::uint8_t value { gray[ i ] };
+		rgb[ i * 3 + 0 ] = value;
+		rgb[ i * 3 + 1 ] = value;
+		rgb[ i * 3 + 2 ] = value;
+	}
+	return rgb;
+}
+
+std::expected< std::vector< std::uint8_t >, idhan::ModuleError > convertIndexedToInterleavedRGB(
+	const std::basic_string_view< std::uint8_t > indexed,
+	const std::size_t pixelCount,
+	const std::basic_string_view< std::uint8_t > colorTable )
+{
+	if ( colorTable.length() < 0x300 )
+	{
+		return std::unexpected( idhan::ModuleError { "Short color table" } );
+	}
+	std::vector< std::uint8_t > rgb( pixelCount * 3 );
+	for ( std::size_t i = 0; i < pixelCount; ++i )
+	{
+		const std::uint8_t value { indexed[ i ] };
+		rgb[ i * 3 + 0 ] = colorTable[ value + 0x000 ];
+		rgb[ i * 3 + 1 ] = colorTable[ value + 0x100 ];
+		rgb[ i * 3 + 2 ] = colorTable[ value + 0x200 ];
+	}
+	return rgb;
+}
+
+std::expected< std::vector< std::uint8_t >, idhan::ModuleError > convertPlanarRGBToInterleavedRGB(
+	const std::basic_string_view< std::uint8_t > planarData,
+	const std::size_t pixelCount,
+	const std::uint16_t channels )
+{
+	std::vector< std::uint8_t > interleaved( pixelCount * 3 );
+	for ( std::size_t c = 0; c < std::min< std::size_t >( channels, 3 ); ++c )
+	{
+		const std::uint8_t* planeData { planarData.data() + ( c * pixelCount ) };
+		for ( std::size_t i = 0; i < pixelCount; ++i )
+		{
+			interleaved[ i * 3 + c ] = planeData[ i ];
+		}
+	}
+	return interleaved;
+}
+
+std::expected< std::vector< std::uint8_t >, idhan::ModuleError > convertPlanarToInterleavedRGB(
+	const std::basic_string_view< std::uint8_t > planarData,
+	const std::uint16_t colorMode,
+	const std::uint32_t width,
+	const std::uint32_t height,
+	const std::uint16_t channels,
+	const std::basic_string_view< std::uint8_t > colorTable )
+{
+	const std::size_t pixelCount { static_cast< std::size_t >( width ) * height };
+
+	switch ( colorMode )
+	{
+		case 1: // Grayscale
+			return convertGrayscaleToInterleavedRGB( planarData, pixelCount );
+		case 2: // Indexed
+			return convertIndexedToInterleavedRGB( planarData, pixelCount, colorTable );
+		case 3: // RGB
+			return convertPlanarRGBToInterleavedRGB( planarData, pixelCount, channels );
+		case 4: // CMYK
+			return convertCMYKtoInterleavedRGB( planarData, pixelCount );
+		default:
+			return std::unexpected( idhan::ModuleError { "Unsupported color mode" } );
+	}
+}
+
+std::uint32_t countPSDLayers( const std::uint8_t* data, std::size_t length )
+{
+	// Start past the file header.
+	std::size_t offset { 26 };
+
+	// Then, skip the color mode data.
+	if ( offset + 4 > length ) return 0;
+	const std::uint32_t colorModeLength { readUint32BE( data + offset ) };
+	offset += 4 + colorModeLength;
+
+	// Next, skip image resources section.
+	if ( offset + 4 > length ) return 0;
+	const std::uint32_t resourcesLength { readUint32BE( data + offset ) };
+	offset += 4 + resourcesLength;
+
+	// Read layer and mask info
+	if ( offset + 4 > length ) return 0;
+	const std::uint32_t layerMaskLength { readUint32BE( data + offset ) };
+	offset += 4;
+	if ( layerMaskLength == 0 ) return 0;
+	const std::size_t layerMaskEnd { offset + layerMaskLength };
+	if ( layerMaskEnd > length ) return 0;
+
+	// Read layer info length
+	if ( offset + 4 > length ) return 0;
+	const std::uint32_t layerInfoLength { readUint32BE( data + offset ) };
+	offset += 4;
+	if ( layerInfoLength == 0 ) return 0;
+
+	// Read layer count
+	if ( offset + 2 > length ) return 0;
+	const std::int16_t rawLayerCount { static_cast< int16_t >( readUint16BE( data + offset ) ) };
+	offset += 2;
+
+	const std::uint16_t layerCount { std::abs< std::uint16_t >( rawLayerCount ) };
+	std::uint32_t realLayerCount = 0;
+
+	// Parse each layer to determine groupType
+	for ( std::uint16_t i = 0; i < layerCount; ++i )
+	{
+		// Skip bounds (top, left, bottom, right)
+		if ( offset + 16 > length ) return realLayerCount;
+		offset += 16;
+
+		// Read channel count
+		if ( offset + 2 > length ) return realLayerCount;
+		const std::uint16_t channelCount { readUint16BE( data + offset ) };
+		offset += 2;
+
+		// Skip channel info
+		if ( offset + channelCount * 6 > length ) return realLayerCount;
+		offset += channelCount * 6;
+
+		// Skip signature, blend mode, opacity, clipping, flags, filler
+		if ( offset + 12 > length ) return realLayerCount;
+		offset += 12;
+
+		// Read extra data length
+		if ( offset + 4 > length ) return realLayerCount;
+		const std::uint32_t extraDataLength { readUint32BE( data + offset ) };
+		offset += 4;
+
+		const std::size_t extraDataEnd { offset + extraDataLength };
+		if ( extraDataEnd > length ) return realLayerCount;
+
+		// Skip mask data length + mask data
+		if ( offset + 4 > length ) return realLayerCount;
+		const std::uint32_t maskLength { readUint32BE( data + offset ) };
+		offset += 4 + maskLength;
+
+		// Skip blending ranges
+		if ( offset + 4 > length ) return realLayerCount;
+		const std::uint32_t blendingRangesLength { readUint32BE( data + offset ) };
+		offset += 4 + blendingRangesLength;
+
+		// Skip layer name
+		if ( offset >= length ) return realLayerCount;
+		const std::uint8_t nameLen { data[ offset ] };
+		offset += 1 + ( ( nameLen + 1 + 3 ) & ~3 ); // Padded to 4 bytes
+
+		// Parse additional layer info to find "lsct" (layer section divider)
+		std::uint32_t groupType { 0 }; // Default to NORMA}
+
+		while ( offset + 12 <= extraDataEnd && offset + 12 <= length )
+		{
+			if ( memcmp( data + offset, "8BIM", 4 ) != 0 && memcmp( data + offset, "8B64", 4 ) != 0 ) break;
+			offset += 4;
+
+			char key[ 5 ] = { 0 };
+			memcpy( key, data + offset, 4 );
+			offset += 4;
+
+			const std::uint32_t dataLength { readUint32BE( data + offset ) };
+			offset += 4;
+
+			const std::size_t dataEnd = offset + dataLength;
+			if ( dataEnd > extraDataEnd || dataEnd > length ) break;
+
+			if ( memcmp( key, "lsct", 4 ) == 0 && dataLength >= 4 )
+			{
+				groupType = readUint32BE( data + offset );
+			}
+
+			offset = dataEnd;
+		}
+
+		// Move to end of extra data
+		offset = extraDataEnd;
+
+		if ( groupType != 3 ) // Ignore layer section dividers.
+		{
+			realLayerCount++;
+		}
+	}
+
+	return realLayerCount;
+}
+
+} // namespace
+
+std::vector< std::string_view > PsdMetadata::handleableMimes()
+{
+	return { "application/psd" };
+}
+
+std::string_view PsdMetadata::name()
+{
+	return "PSD Metadata Parser";
+}
+
+idhan::ModuleVersion PsdMetadata::version()
+{
+	return { .m_major = 1, .m_minor = 0, .m_patch = 0 };
+}
+
+std::expected< idhan::MetadataInfo, idhan::ModuleError > PsdMetadata::parseFile(
+	const void* data,
+	const std::size_t length,
+	[[maybe_unused]] std::string mime_name )
+{
+	const auto* bytes { static_cast< const std::uint8_t* >( data ) };
+
+	const auto header { parsePSDHeader( bytes, length ) };
+	if ( !header )
+	{
+		return std::unexpected( idhan::ModuleError { "Invalid PSD header" } );
+	}
+
+	idhan::MetadataInfo generic_metadata {};
+	idhan::MetadataInfoImageProject project_metadata {};
+
+	project_metadata.image_info.width = static_cast< int >( header->width );
+	project_metadata.image_info.height = static_cast< int >( header->height );
+	project_metadata.image_info.channels = static_cast< std::uint8_t >( header->channels );
+	project_metadata.layers = countPSDLayers( bytes, length );
+
+	generic_metadata.m_simple_type = idhan::SimpleMimeType::IMAGE_PROJECT;
+	generic_metadata.m_metadata = project_metadata;
+
+	return generic_metadata;
+}
+
+std::vector< std::string_view > PsdThumbnailer::handleableMimes()
+{
+	return { "application/psd" };
+}
+
+std::string_view PsdThumbnailer::name()
+{
+	return "PSD Thumbnailer Parser";
+}
+
+idhan::ModuleVersion PsdThumbnailer::version()
+{
+	return { .m_major = 1, .m_minor = 0, .m_patch = 0 };
+}
+
+std::expected< idhan::ThumbnailerModuleI::ThumbnailInfo, idhan::ModuleError > PsdThumbnailer::createThumbnail(
+	const void* data,
+	const std::size_t length,
+	std::size_t width,
+	std::size_t height,
+	[[maybe_unused]] std::string mime_name )
+{
+	const auto bytes { static_cast< const std::uint8_t* >( data ) };
+
+	const auto header { parsePSDHeader( bytes, length ) };
+	if ( !header )
+	{
+		return std::unexpected( idhan::ModuleError { "Invalid PSD header" } );
+	}
+
+	if ( header->depth != 8 && header->depth != 16 && header->depth != 32 )
+	{
+		return std::unexpected( idhan::ModuleError { "Unsupported bit depth" } );
+	}
+
+	std::size_t offset { 26 };
+
+	if ( offset + 4 > length ) return std::unexpected( idhan::ModuleError { "Truncated file" } );
+	const std::uint32_t colorModeLength { readUint32BE( bytes + offset ) };
+	offset += 4;
+
+	if ( offset + colorModeLength > length ) return std::unexpected( idhan::ModuleError { "Truncated file" } );
+	const std::basic_string_view colorTable { bytes + offset, colorModeLength };
+	offset += colorModeLength;
+
+	if ( offset + 4 > length ) return std::unexpected( idhan::ModuleError { "Truncated file" } );
+	const std::uint32_t resourcesLength { readUint32BE( bytes + offset ) };
+	offset += 4 + resourcesLength;
+
+	if ( offset + 4 > length ) return std::unexpected( idhan::ModuleError { "Truncated file" } );
+	const std::uint32_t layerMaskLength { readUint32BE( bytes + offset ) };
+	offset += 4 + layerMaskLength;
+
+	if ( offset + 2 > length ) return std::unexpected( idhan::ModuleError { "Truncated file" } );
+	const std::uint16_t compression { readUint16BE( bytes + offset ) };
+	offset += 2;
+
+	const std::size_t bytesPerSample { static_cast< std::size_t >( header->depth / 8 ) };
+	const std::size_t planeSize { static_cast< std::size_t >( header->width ) * header->height };
+	std::vector< std::uint8_t > planarData;
+
+	switch ( compression )
+	{
+		case 0: // Uncompressed
+			{
+				std::size_t expectedSize { planeSize * header->channels * bytesPerSample };
+
+				if ( offset + expectedSize > length )
+				{
+					return std::unexpected( idhan::ModuleError { "Insufficient image data" } );
+				}
+
+				planarData.assign( bytes + offset, bytes + offset + expectedSize );
+			}
+			break;
+		case 1: // PackBits
+			{
+				planarData = unpackRaster( bytes, offset, length, header->width, header->height, header->channels );
+				if ( planarData.empty() )
+				{
+					return std::unexpected( idhan::ModuleError { "Failed to decompress RLE data" } );
+				}
+			}
+			break;
+		default:
+			return std::unexpected( idhan::ModuleError { "Unsupported compression method" } );
+	}
+
+	std::size_t totalPixels { planeSize * header->channels };
+	std::vector< std::uint8_t > planar8bit { convertToTargetDepth( planarData, header->depth, totalPixels ) };
+	if ( planar8bit.empty() )
+	{
+		return std::unexpected( idhan::ModuleError { "Failed to convert bit depth" } );
+	}
+
+	auto interleavedRGB { convertPlanarToInterleavedRGB(
+		std::basic_string_view( planar8bit.data(), planar8bit.size() ),
+		header->colorMode,
+		header->width,
+		header->height,
+		header->channels,
+		colorTable ) };
+
+	if ( !interleavedRGB.has_value() )
+	{
+		return std::unexpected( interleavedRGB.error() );
+	}
+
+	VipsImage* image { vips_image_new_from_memory(
+		interleavedRGB->data(),
+		interleavedRGB->size(),
+		static_cast< int >( header->width ),
+		static_cast< int >( header->height ),
+		3,
+		VIPS_FORMAT_UCHAR ) };
+
+	if ( !image )
+	{
+		return std::unexpected( idhan::ModuleError { "Failed to create image from PSD data" } );
+	}
+
+	const float source_aspect { static_cast< float >( header->width ) / static_cast< float >( header->height ) };
+	const float target_aspect { static_cast< float >( width ) / static_cast< float >( height ) };
+
+	if ( target_aspect > source_aspect )
+		width = static_cast< std::size_t >( static_cast< float >( height ) * source_aspect );
+	else
+		height = static_cast< std::size_t >( static_cast< float >( width ) / source_aspect );
+
+	VipsImage* resized { nullptr };
+	if ( vips_resize(
+			 image,
+			 &resized,
+			 static_cast< double >( width ) / static_cast< double >( vips_image_get_width( image ) ),
+			 nullptr ) )
+	{
+		g_object_unref( image );
+		return std::unexpected( idhan::ModuleError { "Failed to resize image" } );
+	}
+	g_object_unref( image );
+
+	void* output_buffer { nullptr };
+	std::size_t output_length { 0 };
+	if ( vips_pngsave_buffer( resized, &output_buffer, &output_length, nullptr ) )
+	{
+		g_object_unref( resized );
+		return std::unexpected( idhan::ModuleError { "Failed to save thumbnail" } );
+	}
+	g_object_unref( resized );
+
+	std::vector< std::byte > output(
+		static_cast< std::byte* >( output_buffer ), static_cast< std::byte* >( output_buffer ) + output_length );
+	g_free( output_buffer );
+
+	ThumbnailInfo info {};
+	info.data = std::move( output );
+	info.width = width;
+	info.height = height;
+
+	return info;
+}

IDHANModules/premade/PsdMetadata.hpp

@@ -0,0 +1,44 @@
+//
+// Created by kj16609 on 11/12/25.
+//
+#pragma once
+#include "MetadataModule.hpp"
+#include "ThumbnailerModule.hpp"
+
+class PsdMetadata final : public idhan::MetadataModuleI
+{
+  public:
+
+	PsdMetadata() = default;
+
+	std::vector< std::string_view > handleableMimes() override;
+
+	std::string_view name() override;
+
+	idhan::ModuleVersion version() override;
+
+	std::expected< idhan::MetadataInfo, idhan::ModuleError > parseFile(
+		const void* data,
+		std::size_t length,
+		std::string mime_name ) override;
+};
+
+class PsdThumbnailer final : public idhan::ThumbnailerModuleI
+{
+  public:
+
+	PsdThumbnailer() = default;
+
+	std::vector< std::string_view > handleableMimes() override;
+
+	std::string_view name() override;
+
+	idhan::ModuleVersion version() override;
+
+	std::expected< ThumbnailInfo, idhan::ModuleError > createThumbnail(
+		const void* data,
+		std::size_t length,
+		std::size_t width,
+		std::size_t height,
+		std::string mime_name ) override;
+};

IDHANModules/premade/entry.cpp

@@ -12,15 +12,20 @@
 #include <memory>
 #include <vector>
 
+#include "FFMPEGMetadata.hpp"
+#include "FFMPEGThumbnailer.hpp"
 #include "ImageVipsMetadata.hpp"
 #include "ImageVipsThumbnailer.hpp"
+#include "PsdMetadata.hpp"
 
 using namespace idhan;
 
 std::vector< std::shared_ptr< IDHANModule > > getModules()
 {
 	std::vector< std::shared_ptr< IDHANModule > > ret {
-		std::make_shared< ImageVipsMetadata >(), std::make_shared< ImageVipsThumbnailer >()
+		std::make_shared< ImageVipsMetadata >(), std::make_shared< ImageVipsThumbnailer >(),
+		std::make_shared< PsdMetadata >(),       std::make_shared< PsdThumbnailer >(),
+		std::make_shared< FFMPEGMetadata >(),    std::make_shared< FFMPEGThumbnailer >(),
 	};
 
 	return ret;

IDHANModules/premade/ffmpeg.hpp

@@ -0,0 +1,87 @@
+//
+// Created by kj16609 on 11/14/25.
+//
+#pragma once
+
+#include <libavutil/error.h>
+#include <spdlog/spdlog.h>
+
+extern "C" {
+#include <libavformat/avio.h>
+}
+
+#include <cstdint>
+#include <string_view>
+
+namespace idhan
+{
+namespace log = spdlog;
+}
+
+struct OpaqueInfo
+{
+	std::string_view m_data;
+	std::int64_t m_cursor { 0 };
+};
+
+inline int readFunction( void* opaque, std::uint8_t* buffer, int buffer_size )
+{
+	auto& buffer_view { *static_cast< OpaqueInfo* >( opaque ) };
+
+	const bool cursor_oob { buffer_view.m_cursor > buffer_view.m_data.size() };
+	if ( cursor_oob ) return AVERROR_EOF;
+
+	auto* data { buffer_view.m_data.data() };
+
+	data += buffer_view.m_cursor;
+	const std::int64_t size { static_cast< std::int64_t >( buffer_view.m_data.size() ) - buffer_view.m_cursor };
+	const std::int64_t min_size { std::min( size, static_cast< std::int64_t >( buffer_size ) ) };
+
+	if ( min_size == 0 ) return AVERROR_EOF;
+
+	std::memcpy( buffer, data, min_size );
+
+	buffer_view.m_cursor += min_size;
+
+	return static_cast< int >( min_size );
+}
+
+inline std::int64_t seekFunction( void* opaque, std::int64_t offset, int whence )
+{
+	auto& buffer_view { *static_cast< OpaqueInfo* >( opaque ) };
+
+	idhan::log::info( "Asked to seek from whence {} and offset {}", whence, offset );
+	switch ( whence )
+	{
+		case SEEK_SET:
+			idhan::log::info( "Asked to seek to specific offset {}", offset );
+			buffer_view.m_cursor = offset;
+			break;
+		case SEEK_CUR:
+			idhan::log::info( "Asked to seek to an +{} from cursor", offset );
+			buffer_view.m_cursor += offset;
+			break;
+		case SEEK_END:
+			idhan::log::info( "Asked to seek to end" );
+			buffer_view.m_cursor = static_cast< std::int64_t >( buffer_view.m_data.size() ) + offset;
+			break;
+		case AVSEEK_SIZE:
+			idhan::log::info( "Asked to seek size" );
+			return static_cast< std::int64_t >( buffer_view.m_data.size() );
+		default:
+			{
+				idhan::log::info( "Asked to seek to whence that ended in default" );
+				return -1;
+			}
+	}
+
+	if ( buffer_view.m_cursor < 0 )
+		buffer_view.m_cursor = 0;
+	else if ( buffer_view.m_cursor >= buffer_view.m_data.size() )
+		buffer_view.m_cursor = buffer_view.m_data.size();
+
+	return buffer_view.m_cursor;
+}
+
+inline static std::vector< std::string_view >
+	ffmpeg_handleable_mimes { "video/mp4", "video/webm", "video/mpeg", "video/quicktime" };

IDHANServer/CMakeLists.txt

@@ -2,7 +2,6 @@ project(IDHANServer LANGUAGES CXX C)
 
 AddFGLExecutable(IDHANServer ${CMAKE_CURRENT_SOURCE_DIR}/src)
 
-
 target_sources(IDHANServer PRIVATE ${MIGRATION_SOURCE})
 
 # Gui is needed for QImage for whatever reason

IDHANServer/src/ServerContext.cpp

@@ -58,8 +58,14 @@ void ServerContext::setupCORSSupport() const
 		} );
 
 	drogon::app().registerPostHandlingAdvice(
-		[]( [[maybe_unused]] const drogon::HttpRequestPtr& request, const drogon::HttpResponsePtr& response )
-		{ addCORSHeaders( response ); } );
+		[ this ]( [[maybe_unused]] const drogon::HttpRequestPtr& request, const drogon::HttpResponsePtr& response )
+		{
+			if ( args.testmode )
+				log::info( "Finished Handling query: {}:{}", request->getMethodString(), request->getPath() );
+			else
+				log::debug( "Finished Handling query: {}:{}", request->getMethodString(), request->getPath() );
+			addCORSHeaders( response );
+		} );
 }
 
 void exceptionHandler( const std::exception& e, const drogon::HttpRequestPtr& request, ResponseFunction&& callback )

IDHANServer/src/ServerContext.hpp

@@ -13,7 +13,7 @@
 #include <memory>
 
 #include "ConnectionArguments.hpp"
-#include "filesystem/ClusterManager.hpp"
+#include "filesystem/clusters/ClusterManager.hpp"
 #include "modules/ModuleLoader.hpp"
 
 namespace idhan

IDHANServer/src/api/APIMaintenance.hpp

@@ -33,9 +33,12 @@ class APIMaintenance : public drogon::HttpController< APIMaintenance >
 	drogon::Task< drogon::HttpResponsePtr > integrityCheck( drogon::HttpRequestPtr request );
 
 	drogon::Task< drogon::HttpResponsePtr > parseMime( drogon::HttpRequestPtr request );
+	drogon::Task< drogon::HttpResponsePtr > createThumbnail( drogon::HttpRequestPtr request );
 	drogon::Task< drogon::HttpResponsePtr > reloadMime( drogon::HttpRequestPtr request );
 	drogon::Task< drogon::HttpResponsePtr > listParsers( drogon::HttpRequestPtr request );
 
+	drogon::Task< drogon::HttpResponsePtr > purgeThumbnails( drogon::HttpRequestPtr request );
+
   public:
 
 	METHOD_LIST_BEGIN
@@ -44,12 +47,15 @@ class APIMaintenance : public drogon::HttpController< APIMaintenance >
 	// ADD_METHOD_TO( IDHANMaintenanceAPI::postgresqlStorage, "/db/stats/chart" );
 	ADD_METHOD_TO( APIMaintenance::postgresqlStorageSunData, "/db/stats/sunburst" );
 
-	ADD_METHOD_TO( APIMaintenance::parseMime, "/mime/parse_file" );
+	ADD_METHOD_TO( APIMaintenance::parseMime, "/mime/parse" );
+	ADD_METHOD_TO( APIMaintenance::createThumbnail, "/mime/generate_thumbnail" );
 	ADD_METHOD_TO( APIMaintenance::reloadMime, "/mime/reload" );
 	ADD_METHOD_TO( APIMaintenance::listParsers, "/mime/parsers" );
 
 	ADD_METHOD_TO( APIMaintenance::integrityCheck, "/integrity" );
 
+	ADD_METHOD_TO( APIMaintenance::purgeThumbnails, "/purge/thumbnails" );
+
 	// ADD_METHOD_TO( APIMaintenance::test, "/test" );
 
 	METHOD_LIST_END

IDHANServer/src/api/ClusterAPI.hpp

@@ -6,9 +6,9 @@
 #include <drogon/HttpController.h>
 #include <drogon/utils/coroutine.h>
 
+#include "threading/ExpectedTask.hpp"
 #include "IDHANTypes.hpp"
 #include "db/dbTypes.hpp"
-#include "helpers/ExpectedTask.hpp"
 
 namespace idhan::api
 {

IDHANServer/src/api/FileRelationshipsAPI.hpp

@@ -0,0 +1,26 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+#pragma once
+#include "drogon/HttpController.h"
+
+namespace idhan::api
+{
+
+class FileRelationshipsAPI : public drogon::HttpController< FileRelationshipsAPI >
+{
+	drogon::Task< drogon::HttpResponsePtr > setBetterDuplicate( drogon::HttpRequestPtr request );
+	drogon::Task< drogon::HttpResponsePtr > addAlternative( drogon::HttpRequestPtr request );
+
+  public:
+
+	METHOD_LIST_BEGIN
+
+	ADD_METHOD_TO( FileRelationshipsAPI::setBetterDuplicate, "/relationships/duplicates/add" );
+
+	ADD_METHOD_TO( FileRelationshipsAPI::addAlternative, "/relationships/alternatives/add" );
+
+	METHOD_LIST_END
+};
+
+} // namespace idhan::api

IDHANServer/src/api/cluster/add.cpp

@@ -4,9 +4,9 @@
 
 #include <fstream>
 
+#include "../../filesystem/clusters/ClusterManager.hpp"
 #include "api/ClusterAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
-#include "filesystem/ClusterManager.hpp"
 #include "logging/log.hpp"
 
 namespace idhan::api

IDHANServer/src/api/cluster/scan.cpp

@@ -2,21 +2,22 @@
 // Created by kj16609 on 3/20/25.
 //
 
+#include "../../filesystem/io/IOUring.hpp"
 #include "MetadataModule.hpp"
 #include "api/ClusterAPI.hpp"
-#include "api/helpers/ExpectedTask.hpp"
 #include "api/helpers/createBadRequest.hpp"
-#include "api/helpers/records.hpp"
+#include "api/helpers/helpers.hpp"
 #include "crypto/SHA256.hpp"
 #include "fgl/size.hpp"
-#include "filesystem/IOUring.hpp"
-#include "filesystem/utility.hpp"
+#include "filesystem/filesystem.hpp"
 #include "fixme.hpp"
 #include "hyapi/helpers.hpp"
 #include "logging/log.hpp"
-#include "metadata/parseMetadata.hpp"
+#include "metadata/metadata.hpp"
 #include "mime/FileInfo.hpp"
 #include "mime/MimeDatabase.hpp"
+#include "records/records.hpp"
+#include "threading/ExpectedTask.hpp"
 
 namespace idhan::api
 {
@@ -24,7 +25,7 @@ ExpectedTask< RecordID > adoptOrphan( FileIOUring io_uring, DbClientPtr db )
 {
 	const auto data { co_await io_uring.readAll() };
 	const auto sha256 { SHA256::hash( data.data(), data.size() ) };
-	const auto record_result { co_await helpers::createRecord( sha256, db ) };
+	const auto record_result { co_await idhan::helpers::createRecord( sha256, db ) };
 
 	co_return record_result;
 }
@@ -33,9 +34,9 @@ struct ScanParams
 {
 	bool read_only:1 { true };
 	bool recompute_hash:1 { false };
-	bool scan_mime:1 { false };
+	bool scan_mime:1 { true };
 	bool rescan_mime:1 { false };
-	bool scan_metadata:1 { false };
+	bool scan_metadata:1 { true };
 	bool rescan_metadata:1 { false };
 	bool stop_on_fail:1 { false };
 	bool adopt_orphans:1 { false };
@@ -68,10 +69,10 @@ static ScanParams extractScanParams( const drogon::HttpRequestPtr& request )
 	p.adopt_orphans = request->getOptionalParameter< bool >( "adopt_orphans" ).value_or( false );
 	p.remove_missing_files = request->getOptionalParameter< bool >( "remove_missing_files" ).value_or( false );
 
-	p.scan_metadata = p.scan_metadata || p.adopt_orphans; // orphans will need to be scanned for metadata
-	p.scan_mime = p.scan_mime || p.scan_metadata; // mime is needed for metadata
-	p.recompute_hash = p.recompute_hash || p.adopt_orphans;
-	p.recompute_hash = p.recompute_hash || p.read_only;
+	p.scan_metadata |= p.adopt_orphans; // orphans will need to be scanned for metadata
+	p.scan_mime |= p.scan_metadata; // mime is needed for metadata
+	p.recompute_hash |= p.adopt_orphans;
+	p.recompute_hash |= p.read_only;
 	// if read only then we need to recompute the hash because the file path can't be trusted anymore
 
 	return p;
@@ -84,22 +85,24 @@ class ScanContext
 
 	ScanParams m_params {};
 	std::string m_mime_name {};
+	SHA256 m_sha256 {};
 
 	static constexpr auto INVALID_RECORD { std::numeric_limits< RecordID >::max() };
 	RecordID m_record_id { INVALID_RECORD };
 
 	ClusterID m_cluster_id;
+	std::filesystem::path m_cluster_path;
 
-	ExpectedTask< SHA256 > checkSHA256( FileIOUring uring, std::filesystem::path bad_dir );
+	ExpectedTask< SHA256 > checkSHA256( std::filesystem::path bad_dir );
 
-	ExpectedTask< RecordID > checkRecord( SHA256 sha256, DbClientPtr db );
+	ExpectedTask< RecordID > checkRecord( DbClientPtr db );
 
 	ExpectedTask< void > cleanupDoubleClusters( ClusterID found_cluster_id, DbClientPtr db );
 	drogon::Task<> updateFileModifiedTime( drogon::orm::DbClientPtr db );
 	ExpectedTask< void > insertFileInfo( drogon::orm::DbClientPtr db );
 
 	ExpectedTask< void > checkCluster( DbClientPtr db );
-	ExpectedTask< bool > hasMime( DbClientPtr db );
+	drogon::Task< bool > hasMime( DbClientPtr db );
 
 	ExpectedTask<> scanMime( DbClientPtr db );
 
@@ -108,11 +111,16 @@ class ScanContext
 
   public:
 
-	ScanContext( const std::filesystem::path& file_path, const ClusterID cluster_id, const ScanParams params ) :
+	ScanContext(
+		const std::filesystem::path& file_path,
+		const ClusterID cluster_id,
+		const std::filesystem::path& cluster_path,
+		const ScanParams params ) :
 	  m_path( file_path ),
 	  m_size( std::filesystem::file_size( file_path ) ),
 	  m_params( params ),
-	  m_cluster_id( cluster_id )
+	  m_cluster_id( cluster_id ),
+	  m_cluster_path( cluster_path )
 	{}
 
 	ExpectedTask<> scan( std::filesystem::path bad_dir, DbClientPtr db );
@@ -129,59 +137,56 @@ drogon::Task< drogon::HttpResponsePtr > ClusterAPI::scan( drogon::HttpRequestPtr
 
 	scan_params.read_only = scan_params.force_readonly || result[ 0 ][ "read_only" ].as< bool >();
 
-	const std::filesystem::path folder_path { result[ 0 ][ "folder_path" ].as< std::string >() };
+	const std::filesystem::path cluster_path { result[ 0 ][ "folder_path" ].as< std::string >() };
 
-	const auto bad_dir { folder_path / "bad" };
+	const auto bad_dir { cluster_path / "bad" };
 
 	std::vector< drogon::Task< std::expected< void, drogon::HttpResponsePtr > > > scan_tasks {};
 
 	std::filesystem::path last_scanned { "" };
 
-	auto dir_itterator { std::filesystem::recursive_directory_iterator( folder_path ) };
-	const auto end { std::filesystem::recursive_directory_iterator() };
-
 	std::vector< ExpectedTask<> > awaiters {};
 
-	while ( dir_itterator != end )
+	for ( const auto& folder : std::filesystem::directory_iterator( cluster_path ) )
 	{
-		const auto entry { *dir_itterator };
+		if ( !folder.is_directory() ) continue;
 
-		const auto& file_path { entry.path() };
+		if ( folder.path() == bad_dir ) continue;
 
-		if ( file_path == bad_dir )
+		for ( const auto& file : std::filesystem::directory_iterator( folder ) )
 		{
-			dir_itterator.disable_recursion_pending();
+			const auto entry { file };
+
+			const auto& file_path { entry.path() };
+
+			log::info( "Scanner hitting path: {}", file_path.string() );
+
+			if ( !entry.is_regular_file() )
+			{
+				continue;
+			}
+
+			// ignore thumbnails
+			if ( file_path.extension() == ".thumbnail" )
+			{
+				continue;
+			}
+
+			if ( file_path.parent_path() != last_scanned )
+			{
+				last_scanned = file_path.parent_path();
+				log::info( "Scanning {}", last_scanned.string() );
+			}
+
+			ScanContext ctx { file_path, cluster_id, cluster_path, scan_params };
+
+			const std::expected< void, drogon::HttpResponsePtr > file_result { co_await ctx.scan( bad_dir, db ) };
+
+			if ( scan_params.stop_on_fail && !file_result )
+			{
+				co_return file_result.error();
+			};
 		}
-
-		if ( !entry.is_regular_file() )
-		{
-			++dir_itterator;
-			continue;
-		}
-
-		// ignore thumbnails
-		if ( file_path.extension() == ".thumbnail" )
-		{
-			++dir_itterator;
-			continue;
-		}
-
-		if ( file_path.parent_path() != last_scanned )
-		{
-			last_scanned = file_path.parent_path();
-			log::info( "Scanning {}", last_scanned.string() );
-		}
-
-		ScanContext ctx { file_path, cluster_id, scan_params };
-
-		const std::expected< void, drogon::HttpResponsePtr > file_result { co_await ctx.scan( bad_dir, db ) };
-
-		if ( scan_params.stop_on_fail && !file_result )
-		{
-			co_return file_result.error();
-		};
-
-		++dir_itterator;
 	}
 
 	co_await drogon::when_all( std::move( scan_tasks ) );
@@ -196,9 +201,10 @@ drogon::Task< drogon::HttpResponsePtr > ClusterAPI::scan( drogon::HttpRequestPtr
  * @param bad_dir Directory to put failed files, Such as ones that have the wrong filename
  * @return
  */
-ExpectedTask< SHA256 > ScanContext::checkSHA256( FileIOUring uring, const std::filesystem::path bad_dir )
+ExpectedTask< SHA256 > ScanContext::checkSHA256( const std::filesystem::path bad_dir )
 {
 	const auto file_stem { m_path.stem().string() };
+	FileIOUring uring { m_path };
 
 	auto sha256_e { m_params.trust_filename ? SHA256::fromHex( file_stem ) : co_await SHA256::hashCoro( uring ) };
 
@@ -231,6 +237,7 @@ ExpectedTask< SHA256 > ScanContext::checkSHA256( FileIOUring uring, const std::f
 		{
 			if ( !m_params.read_only )
 			{
+				std::filesystem::create_directories( bad_dir );
 				const auto new_path { bad_dir / m_path.filename() };
 
 				// try to fix the mistake
@@ -244,6 +251,15 @@ ExpectedTask< SHA256 > ScanContext::checkSHA256( FileIOUring uring, const std::f
 					new_path.string() ) );
 			}
 		}
+		catch ( std::exception& e )
+		{
+			co_return std::unexpected( createInternalError(
+				"When scanning file at {} it was detected that the filename does not match the sha256 "
+				"{}. There was an error that prevented this from being fixed: {}",
+				m_path.string(),
+				sha256_hex,
+				e.what() ) );
+		}
 		catch ( ... )
 		{
 			co_return std::unexpected( createInternalError(
@@ -257,21 +273,21 @@ ExpectedTask< SHA256 > ScanContext::checkSHA256( FileIOUring uring, const std::f
 	co_return *sha256_e;
 }
 
-ExpectedTask< RecordID > ScanContext::checkRecord( const SHA256 sha256, drogon::orm::DbClientPtr db )
+ExpectedTask< RecordID > ScanContext::checkRecord( drogon::orm::DbClientPtr db )
 {
 	const auto search_result {
-		co_await db->execSqlCoro( "SELECT record_id FROM records WHERE sha256 = $1", sha256.toVec() )
+		co_await db->execSqlCoro( "SELECT record_id FROM records WHERE sha256 = $1", m_sha256.toVec() )
 	};
 
 	if ( search_result.empty() && m_params.adopt_orphans )
 	{
 		const auto insert_result {
-			co_await db->execSqlCoro( "INSERT INTO records (sha256) VALUES ($1) RETURNING record_id", sha256.toVec() )
+			co_await db->execSqlCoro( "INSERT INTO records (sha256) VALUES ($1) RETURNING record_id", m_sha256.toVec() )
 		};
 
 		if ( insert_result.empty() )
 		{
-			co_return std::unexpected( createInternalError( "Failed to create a record for hash {}", sha256.hex() ) );
+			co_return std::unexpected( createInternalError( "Failed to create a record for hash {}", m_sha256.hex() ) );
 		}
 
 		co_return insert_result[ 0 ][ 0 ].as< RecordID >();
@@ -343,6 +359,7 @@ ExpectedTask<> ScanContext::insertFileInfo( drogon::orm::DbClientPtr db )
 
 ExpectedTask<> ScanContext::checkCluster( drogon::orm::DbClientPtr db )
 {
+	log::debug( "Verifying that the record is in the correct cluster" );
 	FGL_ASSERT( m_record_id != INVALID_RECORD, "Invalid record" );
 	const auto file_info {
 		co_await db->execSqlCoro( "SELECT cluster_id, modified_time FROM file_info WHERE record_id = $1", m_record_id )
@@ -371,16 +388,43 @@ ExpectedTask<> ScanContext::checkCluster( drogon::orm::DbClientPtr db )
 		return_unexpected_error( result );
 	}
 
+	// now check if the file is in the right path
+	const auto current_parent { m_path.parent_path() };
+	const auto expected_cluster_subfolder { filesystem::getFileFolder( m_sha256 ) };
+	const auto expected_parent_path { m_cluster_path / expected_cluster_subfolder };
+
+	if ( current_parent != expected_parent_path )
+	{
+		log::warn(
+			"Expected file {} to be in path {} but was found in {} instead (Record {})",
+			m_path.filename().string(),
+			expected_parent_path.string(),
+			current_parent.string(),
+			m_record_id );
+
+		if ( !m_params.read_only )
+		{
+			const auto new_path { expected_parent_path / m_path.filename() };
+			log::info( "Moving file {} to {}", new_path.string(), new_path.string() );
+
+			std::filesystem::create_directories( expected_parent_path );
+
+			std::filesystem::rename( m_path, new_path );
+
+			m_path = new_path;
+		}
+	}
+
 	co_return {};
 }
 
-ExpectedTask< bool > ScanContext::hasMime( DbClientPtr db )
+drogon::Task< bool > ScanContext::hasMime( DbClientPtr db )
 {
 	auto current_mime { co_await db->execSqlCoro(
 		"SELECT mime_id, name FROM file_info JOIN mime USING (mime_id) WHERE record_id = $1 AND mime_id IS NOT NULL",
 		m_record_id ) };
 
-	if ( !current_mime.empty() )
+	if ( !current_mime.empty() && !current_mime[ 0 ][ "mime_id" ].isNull() )
 	{
 		m_mime_name = current_mime[ 0 ][ 1 ].as< std::string >();
 		co_return true;
@@ -395,11 +439,13 @@ ExpectedTask<> ScanContext::scanMime( DbClientPtr db )
 	FileIOUring file_io { m_path };
 
 	// skip checking if we have a mime if we are going to rescan it
-	if ( !m_params.rescan_mime && co_await hasMime( db ) )
+	if ( ( !m_params.rescan_mime ) && co_await hasMime( db ) )
 	{
+		log::debug( "Skipping metadata scan because it already had metadata and rescan_mime was set to false" );
 		co_return {};
 	}
 
+	log::debug( "Starting metadata scan for {} (Record {})", m_path.filename().string(), m_record_id );
 	const auto mime_string_e { co_await mime::getMimeDatabase()->scan( file_io ) };
 
 	const auto mtime { filesystem::getLastWriteTime( m_path ) };
@@ -411,9 +457,10 @@ ExpectedTask<> ScanContext::scanMime( DbClientPtr db )
 		if ( extension_str.starts_with( "." ) ) extension_str = extension_str.substr( 1 );
 
 		log::warn(
-			"During a cluster scan file {} failed to be detected by any mime parsers; It has been added despite this and has an extension override of \'{}\'",
-			m_path.string(),
-			extension_str );
+			"During a cluster scan file {} failed to be detected by any mime parsers; It has been added despite this and has an extension override of \'{}\' (Record {})",
+			m_path.filename().string(),
+			extension_str,
+			m_record_id );
 
 		co_await db->execSqlCoro(
 			"INSERT INTO file_info (record_id, size, extension, modified_time) VALUES ($1, $2, $3, $4) ON CONFLICT (record_id) DO UPDATE SET extension = $3, mime_id = NULL",
@@ -455,8 +502,10 @@ ExpectedTask<> ScanContext::scanMetadata( DbClientPtr db )
 	// No mime was found in the previous step
 	if ( m_mime_name.empty() )
 	{
-		co_return std::unexpected(
-			createInternalError( "Unable to determine metadata parser for {}: No mime found", m_record_id ) );
+		co_return std::unexpected( createInternalError(
+			"Unable to determine metadata parser for {} (Record {}): No mime found",
+			m_path.filename().string(),
+			m_record_id ) );
 	}
 
 	if ( !m_params.rescan_metadata )
@@ -474,7 +523,7 @@ ExpectedTask<> ScanContext::scanMetadata( DbClientPtr db )
 		}
 	}
 
-	const std::shared_ptr< MetadataModuleI > metadata_parser { co_await findBestParser( m_mime_name ) };
+	const std::shared_ptr< MetadataModuleI > metadata_parser { co_await metadata::findBestParser( m_mime_name ) };
 
 	// No parser was found
 	if ( !metadata_parser )
@@ -493,7 +542,7 @@ ExpectedTask<> ScanContext::scanMetadata( DbClientPtr db )
 
 	if ( metadata_e )
 	{
-		co_await updateRecordMetadata( m_record_id, db, *metadata_e );
+		co_await metadata::updateRecordMetadata( m_record_id, db, *metadata_e );
 	}
 	else
 	{
@@ -535,10 +584,11 @@ ExpectedTask< void > ScanContext::checkExtension( DbClientPtr db )
 	if ( expected_extension != file_extension )
 	{
 		log::warn(
-			"When scanning record {}. It was detected that the extension did not match it's mime, Expected {} got {}",
-			m_record_id,
+			"When scanning {} it was detected that the extension did not match it's mime, Expected {} got {} (Record {})",
+			m_path.filename().string(),
 			expected_extension,
-			file_extension );
+			file_extension,
+			m_record_id );
 
 		if ( !m_params.read_only && m_params.fix_extensions )
 		{
@@ -560,8 +610,6 @@ drogon::Task< std::expected< void, drogon::HttpResponsePtr > > ScanContext::scan
 	const std::filesystem::path bad_dir,
 	drogon::orm::DbClientPtr db )
 {
-	FileIOUring io_uring { m_path };
-
 	log::debug( "Scanning file: {}", m_path.string() );
 
 	if ( m_size == 0 )
@@ -569,10 +617,12 @@ drogon::Task< std::expected< void, drogon::HttpResponsePtr > > ScanContext::scan
 			"When scanning file: {} it was detected that it has a filesize of zero!", m_path.string() ) );
 
 	// check that the sha256 matches the sha256 name of the file
-	const auto sha256_e { co_await checkSHA256( io_uring, bad_dir ) };
+	const auto sha256_e { co_await checkSHA256( bad_dir ) };
 	return_unexpected_error( sha256_e );
 
-	const auto record_e { co_await checkRecord( *sha256_e, db ) };
+	m_sha256 = *sha256_e;
+
+	const auto record_e { co_await checkRecord( db ) };
 	if ( !record_e ) co_return std::unexpected( record_e.error() );
 	return_unexpected_error( record_e );
 	m_record_id = *record_e;
@@ -582,24 +632,30 @@ drogon::Task< std::expected< void, drogon::HttpResponsePtr > > ScanContext::scan
 	// check if the record has been identified in a cluster before
 	const auto cluster_e { co_await checkCluster( db ) };
 
-	if ( m_params.scan_mime )
+	bool has_mime_info { co_await hasMime( db ) };
+
+	if ( ( m_params.scan_mime && !has_mime_info ) || m_params.rescan_mime )
 	{
 		log::debug( "Scanning mime for file {}", m_path.string() );
 		const auto mime_e { co_await scanMime( db ) };
 		if ( !mime_e )
 		{
 			const auto msg( hyapi::helpers::extractHttpResponseErrorMessage( mime_e.error() ) );
-			log::warn( "Failed to process mime for record {} at path {}: {}", m_record_id, m_path.string(), msg );
+			log::warn( "Failed to process mime for {} (Record {}): {}", m_path.filename().string(), m_record_id, msg );
 			co_return std::unexpected( createInternalError(
-				"Failed to process mime for record {} at path {}: {}", m_record_id, m_path.string(), msg ) );
+				"Failed to process mime for {} (Record {}): {}", m_path.filename().string(), m_record_id, msg ) );
 		}
+		has_mime_info = co_await hasMime( db );
 	}
 
-	// extension check
-	const auto extenion_result { co_await checkExtension( db ) };
-	return_unexpected_error( extenion_result );
+	if ( has_mime_info )
+	{
+		// extension check
+		const auto extenion_result { co_await checkExtension( db ) };
+		return_unexpected_error( extenion_result );
+	}
 
-	if ( m_params.scan_metadata )
+	if ( ( m_params.scan_metadata || m_params.rescan_metadata ) && has_mime_info )
 	{
 		log::debug( "Scanning metadata for file {}", m_path.string() );
 		const auto metadata_e { co_await scanMetadata( db ) };

IDHANServer/src/api/file/fetchFile.cpp

@@ -4,11 +4,12 @@
 
 #include <regex>
 
+#include "ServerContext.hpp"
 #include "api/RecordAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "api/helpers/helpers.hpp"
 #include "crypto/SHA256.hpp"
-#include "filesystem/utility.hpp"
+#include "filesystem/filesystem.hpp"
 #include "logging/log.hpp"
 
 namespace idhan::api
@@ -17,15 +18,20 @@ namespace idhan::api
 drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchFile( drogon::HttpRequestPtr request, RecordID record_id )
 {
 	const auto db { drogon::app().getFastDbClient() };
-	const auto path_e { co_await filesystem::getFilepath( record_id, db ) };
+	const auto path_e { co_await filesystem::getRecordPath( record_id, db ) };
 	if ( !path_e ) co_return path_e.error();
 
 	if ( !std::filesystem::exists( *path_e ) )
 	{
 		log::warn( "Expected file at location {} for record {} but no file was found", path_e->string(), record_id );
-		co_return createInternalError( "File was expected but not found. Possible data loss" );
+		co_return createInternalError(
+			"File not found at expected location. Record ID: {}, Path: {}. This may indicate data corruption or file system issues.",
+			record_id,
+			path_e->string() );
 	}
 
+	const std::size_t file_size { std::filesystem::file_size( *path_e ) };
+
 	// Check if this is a head request
 	if ( request->isHead() )
 	{
@@ -33,15 +39,31 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchFile( drogon::HttpReques
 
 		// add to response header that we support partial requests
 		response->addHeader( "Accept-Ranges", "bytes" );
-		response->addHeader( "Content-Length", std::to_string( std::filesystem::file_size( *path_e ) ) );
+
+		response->addHeader( "Content-Length", std::to_string( file_size ) );
+
+		const auto mime_info {
+			co_await db->execSqlCoro( "SELECT mime.name as mime_name FROM file_info JOIN mime USING (mime_id)" )
+		};
+
+		if ( mime_info.empty() )
+		{
+			response->setContentTypeString( "application/octet-stream" );
+			// response->addHeader( "Content-Type", "application/octet-stream" );
+		}
+		else
+		{
+			response->setContentTypeString( mime_info[ 0 ][ "mime_name" ].as< std::string >() );
+			// response->addHeader( "Content-Type", mime_info[ 0 ][ "mime_name" ].as< std::string >() );
+		}
+
+		response->setPassThrough( true );
 
 		co_return response;
 	}
 
 	// Get the header for ranges if supplied
 
-	const std::size_t file_size { std::filesystem::file_size( *path_e ) };
-
 	// Get the header for ranges if supplied
 	const auto& range_header { request->getHeader( "Range" ) };
 	std::size_t begin { 0 };
@@ -49,12 +71,16 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchFile( drogon::HttpReques
 
 	// This is stupid but apparently valid
 	constexpr auto full_range { "bytes=0-" };
-	if ( !range_header.empty() && range_header != full_range )
+
+	const bool has_range_header { !range_header.empty() };
+	const bool is_full_range { has_range_header && ( range_header == full_range ) };
+	if ( !is_full_range && has_range_header )
 	{
-		static const std::regex range_pattern { R"(bytes=(\d*)-(\d*)?)" };
+		constexpr auto regex_pattern { R"(bytes=(\d*)-(\d*)?)" };
+		static const std::regex regex { regex_pattern };
 		std::smatch range_match {};
 
-		if ( std::regex_match( range_header, range_match, range_pattern ) )
+		if ( std::regex_match( range_header, range_match, regex ) )
 		{
 			if ( range_match.size() != 3 )
 			{
@@ -65,14 +91,20 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchFile( drogon::HttpReques
 			try
 			{
 				if ( range_match[ 1 ].matched )
+				{
+					log::debug( "Regex range header match 1: {}", range_match[ 1 ].str() );
 					begin = static_cast< std::size_t >( std::stoull( range_match[ 1 ].str() ) );
+				}
 				if ( range_match[ 2 ].matched )
+				{
+					log::debug( "Regex range header match 2: {}", range_match[ 2 ].str() );
 					end = static_cast< std::size_t >( std::stoull( range_match[ 2 ].str() ) );
+				}
 			}
 			catch ( std::exception& e )
 			{
-				log::error( "Error with range header: {}", e.what() );
-				co_return createBadRequest( "Invalid Range Header" );
+				log::error( "Error with range header: {}, Header was {}", e.what(), range_header );
+				co_return createBadRequest( "Error with range header: {}, Header was {}", e.what(), range_header );
 			}
 
 			// Ensure the range is valid
@@ -80,14 +112,15 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchFile( drogon::HttpReques
 		}
 		else
 		{
-			co_return createBadRequest( "Invalid Range Header Format" );
+			co_return createBadRequest( "Invalid Range Header Format Regex failed: {}", regex_pattern );
 		}
 	}
 
 	if ( request->getOptionalParameter< bool >( "download" ).value_or( false ) )
 	{
 		// send the file as a download instead of letting the browser try to display it
-		co_return drogon::HttpResponse::newFileResponse( path_e->string(), path_e->filename().string() );
+		const auto response { drogon::HttpResponse::newFileResponse( path_e->string(), path_e->filename().string() ) };
+		co_return response;
 	}
 
 	auto response { drogon::HttpResponse::newFileResponse( path_e->string(), begin, end - begin ) };

IDHANServer/src/api/file/fetchInfo.cpp

@@ -6,75 +6,12 @@
 #include "api/RecordAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "crypto/SHA256.hpp"
-#include "metadata/parseMetadata.hpp"
+#include "fgl/defines.hpp"
+#include "metadata/metadata.hpp"
 
 namespace idhan::api
 {
 
-drogon::Task< std::expected< void, drogon::HttpResponsePtr > > addImageInfo(
-	Json::Value& root,
-	const RecordID record_id,
-	DbClientPtr db )
-{
-	const auto metadata { co_await db->execSqlCoro( "SELECT * FROM image_metadata WHERE record_id = $1", record_id ) };
-
-	if ( metadata.empty() )
-		co_return std::unexpected( createInternalError( "Could not find image metadata for record {}", record_id ) );
-
-	root[ "width" ] = metadata[ 0 ][ "width" ].as< std::uint32_t >();
-	root[ "height" ] = metadata[ 0 ][ "height" ].as< std::uint32_t >();
-	root[ "channels" ] = metadata[ 0 ][ "channels" ].as< std::uint32_t >();
-
-	co_return std::expected< void, drogon::HttpResponsePtr >();
-}
-
-drogon::Task< std::expected< void, drogon::HttpResponsePtr > > addFileSpecificInfo(
-	Json::Value& root,
-	const RecordID record_id,
-	DbClientPtr db )
-{
-	auto simple_mime_result {
-		co_await db->execSqlCoro( "SELECT simple_mime_type FROM metadata WHERE record_id = $1", record_id )
-	};
-
-	if ( simple_mime_result.empty() ) // Could not find any mime info for this record, Try parsing for it.
-	{
-		const auto parsed_metadata { co_await tryParseRecordMetadata( record_id, db ) };
-
-		if ( !parsed_metadata ) co_return std::unexpected( parsed_metadata.error() );
-
-		simple_mime_result =
-			co_await db->execSqlCoro( "SELECT simple_mime_type FROM metadata WHERE record_id = $1", record_id );
-	}
-
-	if ( simple_mime_result.empty() )
-		co_return std::unexpected( createInternalError( "Failed to get simple mime type for record {}", record_id ) );
-
-	const SimpleMimeType simple_mime_type { simple_mime_result[ 0 ][ "simple_mime_type" ].as< std::uint16_t >() };
-
-	switch ( simple_mime_type )
-	{
-		case SimpleMimeType::IMAGE:
-			{
-				const auto result { co_await addImageInfo( root, record_id, db ) };
-				if ( !result ) co_return std::unexpected( result.error() );
-				break;
-			}
-		case SimpleMimeType::VIDEO:
-			break;
-		case SimpleMimeType::ANIMATION:
-			break;
-		case SimpleMimeType::AUDIO:
-			break;
-		case SimpleMimeType::NONE:
-			[[fallthrough]];
-		default:
-			break;
-	}
-
-	co_return std::expected< void, drogon::HttpResponsePtr >();
-}
-
 drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchInfo(
 	[[maybe_unused]] drogon::HttpRequestPtr request,
 	RecordID record_id )
@@ -92,7 +29,7 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchInfo(
 
 	const auto file_info { co_await db->execSqlCoro( "SELECT * FROM file_info WHERE record_id = $1", record_id ) };
 
-	if ( !file_info.empty() )
+	if ( !file_info.empty() && !file_info[ 0 ][ "mime_id" ].isNull() )
 	{
 		root[ "size" ] = file_info[ 0 ][ "size" ].as< std::size_t >();
 
@@ -102,7 +39,7 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchInfo(
 		root[ "mime" ] = mime_info[ 0 ][ "name" ].as< std::string >();
 		root[ "extension" ] = mime_info[ 0 ][ "best_extension" ].as< std::string >();
 
-		co_await addFileSpecificInfo( root, record_id, db );
+		co_await metadata::addFileSpecificInfo( root, record_id, db );
 	}
 
 	co_return drogon::HttpResponse::newHttpJsonResponse( root );
@@ -111,7 +48,7 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchInfo(
 drogon::Task< drogon::HttpResponsePtr > RecordAPI::parseFile( drogon::HttpRequestPtr request, RecordID record_id )
 {
 	const auto db { drogon::app().getDbClient() };
-	const auto parse_result { co_await tryParseRecordMetadata( record_id, db ) };
+	const auto parse_result { co_await metadata::tryParseRecordMetadata( record_id, db ) };
 	if ( !parse_result ) co_return parse_result.error();
 
 	co_return co_await fetchInfo( request, record_id );

IDHANServer/src/api/file/fetchThumbnail.cpp

@@ -4,18 +4,19 @@
 
 #include <fstream>
 
+#include "../../filesystem/io/IOUring.hpp"
 #include "api/RecordAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "api/helpers/helpers.hpp"
 #include "crypto/SHA256.hpp"
 #include "drogon/HttpAppFramework.h"
 #include "drogon/utils/coroutine.h"
-#include "filesystem/IOUring.hpp"
 #include "logging/ScopedTimer.hpp"
 #include "modules/ModuleLoader.hpp"
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wsuggest-override"
+#include "filesystem/filesystem.hpp"
 #include "paths.hpp"
 #include "trantor/utils/ConcurrentTaskQueue.h"
 #pragma GCC diagnostic pop
@@ -70,37 +71,30 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchThumbnail( drogon::HttpR
 		// We must generate the thumbnail
 		auto thumbnailers { modules::ModuleLoader::instance().getThumbnailerFor( mime_name ) };
 
-		if ( thumbnailers.size() == 0 )
+		if ( thumbnailers.empty() )
 		{
 			co_return createBadRequest( "No thumbnailer for mime type {} provided by modules", mime_name );
 		}
 
 		auto& thumbnailer { thumbnailers[ 0 ] };
 
-		const auto record_path { co_await helpers::getRecordPath( record_id, db ) };
-
-		if ( !record_path ) co_return record_path.error();
-
 		// FileMappedData data { record_path.value() };
-		FileIOUring io_uring { record_path.value() };
+		auto io_uring_e { co_await filesystem::getIOForRecord( record_id, db ) };
+		if ( !io_uring_e ) co_return io_uring_e.error();
+		auto& io_uring { io_uring_e.value() };
 
 		//TODO: Allow requesting a specific thumbnail size
 		std::size_t height { 256 };
 		std::size_t width { 256 };
 
-		std::vector< std::byte > data { co_await io_uring.readAll() };
+		const auto& [ data, data_size ] = io_uring.mmapReadOnly();
 
-		const auto thumbnail_info {
-			thumbnailer->createThumbnail( data.data(), data.size(), width, height, mime_name )
-		};
+		const auto thumbnail_info { thumbnailer->createThumbnail( data, data_size, width, height, mime_name ) };
 
 		if ( !thumbnail_info ) co_return createInternalError( "Thumbnailer had an error: {}", thumbnail_info.error() );
 
 		std::filesystem::create_directories( thumbnail_location_e.value().parent_path() );
 
-		// const auto& thumbnail_data = thumbnail_info.value().data;
-		auto thumbnail_data { std::make_shared< std::vector< std::byte > >( thumbnail_info.value().data ) };
-
 		const auto& thumbnail_location { thumbnail_location_e.value() };
 
 		std::filesystem::create_directories( thumbnail_location.parent_path() );
@@ -108,11 +102,12 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::fetchThumbnail( drogon::HttpR
 
 		log::debug( "Writing thumbnail to {}", thumbnail_location.string() );
 
-		co_await io_uring_write.write( *thumbnail_data );
+		co_await io_uring_write.write( thumbnail_info->data );
 	}
 
-	auto response { drogon::HttpResponse::newFileResponse(
-		thumbnail_location_e.value(), thumbnail_location_e.value().filename(), drogon::ContentType::CT_IMAGE_PNG ) };
+	auto response {
+		drogon::HttpResponse::newFileResponse( thumbnail_location_e.value(), "", drogon::ContentType::CT_IMAGE_PNG )
+	};
 
 	const auto duration { std::chrono::hours( 1 ) };
 

IDHANServer/src/api/file/importFile.cpp

@@ -2,15 +2,15 @@
 // Created by kj16609 on 11/15/24.
 //
 
+#include "../../filesystem/clusters/ClusterManager.hpp"
+#include "../../records/records.hpp"
 #include "api/ImportAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
-#include "api/helpers/records.hpp"
 #include "codes/ImportCodes.hpp"
 #include "crypto/SHA256.hpp"
 #include "db/drogonArrayBind.hpp"
-#include "filesystem/ClusterManager.hpp"
 #include "logging/log.hpp"
-#include "metadata/parseMetadata.hpp"
+#include "metadata/metadata.hpp"
 #include "mime/MimeDatabase.hpp"
 
 namespace idhan::api
@@ -160,7 +160,7 @@ drogon::Task< drogon::HttpResponsePtr > ImportAPI::importFile( const drogon::Htt
 
 	const auto response { drogon::HttpResponse::newHttpJsonResponse( root ) };
 
-	co_await tryParseRecordMetadata( record_id, db );
+	co_await metadata::tryParseRecordMetadata( record_id, db );
 
 	co_return response;
 }

IDHANServer/src/api/helpers/checkContentType.cpp

@@ -4,7 +4,7 @@
 
 #include "checkContentType.hpp"
 
-namespace idhan
+namespace idhan::api::helpers
 {
 //! Responds with that the content type is unsupported or unknown
 void checkContentType(
@@ -23,4 +23,4 @@ void checkContentType(
 
 	callback( drogon::HttpResponse::newHttpJsonResponse( json ) );
 }
-} // namespace idhan
+} // namespace idhan::api::helpers

IDHANServer/src/api/helpers/checkContentType.hpp

@@ -6,7 +6,7 @@
 
 #include "ResponseCallback.hpp"
 
-namespace idhan
+namespace idhan::api::helpers
 {
 //! Responds with that the content type is unsupported or unknown
 void checkContentType(

IDHANServer/src/api/helpers/getTagDomainID.cpp

@@ -8,7 +8,7 @@
 namespace idhan::api::helpers
 {
 
-std::expected< TagDomainID, drogon::HttpResponsePtr > getTagDomainID( drogon::HttpRequestPtr request )
+std::expected< TagDomainID, drogon::HttpResponsePtr > getTagDomainIDParameter( const drogon::HttpRequestPtr& request )
 {
 	try
 	{

IDHANServer/src/api/helpers/helpers.hpp

@@ -3,21 +3,8 @@
 //
 #pragma once
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Weffc++"
-#pragma GCC diagnostic ignored "-Wredundant-tags"
-#pragma GCC diagnostic ignored "-Wcast-qual"
-#pragma GCC diagnostic ignored "-Wold-style-cast"
-#pragma GCC diagnostic ignored "-Wnoexcept"
-#pragma GCC diagnostic ignored "-Wredundant-decls"
-#pragma GCC diagnostic ignored "-Wuseless-cast"
-#pragma GCC diagnostic ignored "-Wnoexcept"
-#pragma GCC diagnostic ignored "-Wswitch-enum"
-#pragma GCC diagnostic ignored "-Wshadow"
 #include <drogon/HttpResponse.h>
-#include <drogon/orm/DbClient.h>
 #include <drogon/utils/coroutine.h>
-#pragma GCC diagnostic pop
 
 #include <expected>
 #include <vector>
@@ -28,11 +15,7 @@
 namespace idhan::api::helpers
 {
 
-drogon::Task< std::expected< std::filesystem::path, drogon::HttpResponsePtr > > getRecordPath(
-	RecordID record_id,
-	DbClientPtr db );
-
-std::expected< TagDomainID, drogon::HttpResponsePtr > getTagDomainID( drogon::HttpRequestPtr request );
+std::expected< TagDomainID, drogon::HttpResponsePtr > getTagDomainIDParameter( const drogon::HttpRequestPtr& request );
 
 constexpr std::chrono::seconds default_max_age {
 	std::chrono::duration_cast< std::chrono::seconds >( std::chrono::years( 1 ) )

IDHANServer/src/api/helpers/urls.cpp

@@ -1,37 +0,0 @@
-//
-// Created by kj16609 on 7/24/25.
-//
-
-#include "urls.hpp"
-
-#include "createBadRequest.hpp"
-
-namespace idhan::helpers
-{
-drogon::Task< std::expected< UrlID, drogon::HttpResponsePtr > > findOrCreateUrl( const std::string url, DbClientPtr db )
-{
-	UrlID url_id { INVALID_URL_ID };
-	std::size_t tries { 0 };
-
-	do
-	{
-		tries += 1;
-		if ( tries > 16 ) co_return std::unexpected( createBadRequest( "Too many URL creation attempts" ) );
-		const auto search_result { co_await db->execSqlCoro( "SELECT url_id FROM urls WHERE url = $1", url ) };
-
-		if ( !search_result.empty() )
-		{
-			url_id = search_result[ 0 ][ 0 ].as< UrlID >();
-			break;
-		}
-
-		const auto insert { co_await db->execSqlCoro(
-			"INSERT INTO urls (url) VALUES ($1) ON CONFLICT DO NOTHING RETURNING url_id", url ) };
-
-		if ( !insert.empty() ) url_id = insert[ 0 ][ 0 ].as< UrlID >();
-	}
-	while ( url_id == INVALID_URL_ID );
-
-	co_return url_id;
-}
-} // namespace idhan::helpers

IDHANServer/src/api/jobs/rescanMetadata.cpp

@@ -4,7 +4,8 @@
 
 #include "IDHANTypes.hpp"
 #include "api/APIMaintenance.hpp"
-#include "metadata/parseMetadata.hpp"
+#include "logging/log.hpp"
+#include "metadata/metadata.hpp"
 
 namespace idhan::api
 {
@@ -19,9 +20,11 @@ drogon::Task< drogon::HttpResponsePtr > APIMaintenance::rescanMetadata(
 	{
 		const auto record_id { row[ "record_id" ].as< RecordID >() };
 
-		co_await tryParseRecordMetadata( record_id, db );
+		co_await metadata::tryParseRecordMetadata( record_id, db );
 	}
 
+	log::info( "Finished scanning metadata for records" );
+
 	co_return drogon::HttpResponse::newHttpJsonResponse( Json::Value() );
 }
 

IDHANServer/src/api/maintenance/generateThumbnail.cpp

@@ -0,0 +1,71 @@
+//
+// Created by kj16609 on 10/21/25.
+//
+
+#include "api/APIMaintenance.hpp"
+#include "api/helpers/createBadRequest.hpp"
+#include "mime/MimeDatabase.hpp"
+#include "modules/ModuleLoader.hpp"
+
+namespace idhan::api
+{
+
+drogon::Task< drogon::HttpResponsePtr > APIMaintenance::createThumbnail( drogon::HttpRequestPtr request )
+{
+	if ( request->contentType() != drogon::CT_APPLICATION_OCTET_STREAM )
+		co_return createBadRequest(
+			"Content type must be octet-stream was {}", static_cast< int >( request->contentType() ) );
+	const auto request_data { request->getBody() };
+
+	if ( request_data.empty() )
+	{
+		Json::Value error;
+		error[ "error" ] = "No data provided in POST request";
+		co_return drogon::HttpResponse::newHttpJsonResponse( error );
+	}
+
+	const auto mime_str { co_await mime::getMimeDatabase()->scan( request_data ) };
+
+	if ( !mime_str )
+	{
+		Json::Value response;
+		response[ "success" ] = false;
+		response[ "error" ] = "Failed to parse mime type";
+		co_return drogon::HttpResponse::newHttpJsonResponse( response );
+	}
+
+	auto thumbnailers { modules::ModuleLoader::instance().getThumbnailerFor( *mime_str ) };
+
+	if ( thumbnailers.empty() )
+	{
+		Json::Value response;
+		response[ "success" ] = false;
+		response[ "error" ] = "Failed to find thumbnailer for mime";
+		co_return drogon::HttpResponse::newHttpJsonResponse( response );
+	}
+
+	//grab first thumbnailer
+	auto thumbnailer { thumbnailers.at( 0 ) };
+
+	const auto thumbnail_data {
+		thumbnailer->createThumbnail( request_data.data(), request_data.size(), 128, 128, *mime_str )
+	};
+
+	if ( !thumbnailer )
+	{
+		Json::Value response;
+		response[ "success" ] = false;
+		response[ "error" ] = "Failed to parse thumbnail type";
+		co_return drogon::HttpResponse::newHttpJsonResponse( response );
+	}
+
+	const auto& thumb_info { *thumbnail_data };
+
+	auto response = drogon::HttpResponse::newHttpResponse();
+	response->setContentTypeCode( drogon::CT_IMAGE_PNG );
+	response->setBody(
+		std::string( reinterpret_cast< const char* >( thumb_info.data.data() ), thumb_info.data.size() ) );
+	co_return response;
+}
+
+} // namespace idhan::api

IDHANServer/src/api/maintenance/parseMime.cpp

@@ -5,6 +5,7 @@
 #include "api/APIMaintenance.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "mime/MimeDatabase.hpp"
+#include "modules/ModuleLoader.hpp"
 
 namespace idhan::api
 {
@@ -12,7 +13,8 @@ namespace idhan::api
 drogon::Task< drogon::HttpResponsePtr > APIMaintenance::parseMime( drogon::HttpRequestPtr request )
 {
 	if ( request->contentType() != drogon::CT_APPLICATION_OCTET_STREAM )
-		co_return createBadRequest( "Content type must be octet-stream" );
+		co_return createBadRequest(
+			"Content type must be octet-stream was {}", static_cast< int >( request->contentType() ) );
 	const auto request_data { request->getBody() };
 
 	if ( request_data.empty() )
@@ -36,6 +38,20 @@ drogon::Task< drogon::HttpResponsePtr > APIMaintenance::parseMime( drogon::HttpR
 	response[ "success" ] = true;
 	response[ "mime" ] = mime_str.value();
 
+	auto metadata_modules { modules::ModuleLoader::instance().getParserFor( *mime_str ) };
+
+	response[ "metadata_modules" ] = {};
+
+	for ( const auto& metadata_module : metadata_modules )
+	{
+		Json::Value metadata_obj {};
+		metadata_obj[ "name" ] = std::string( metadata_module->name() );
+
+		auto metadata_info { metadata_module->parseFile( request_data.data(), request_data.size(), *mime_str ) };
+
+		response[ "metadata_modules" ].append( std::move( metadata_obj ) );
+	}
+
 	co_return drogon::HttpResponse::newHttpJsonResponse( response );
 }
 

IDHANServer/src/api/maintenance/purgeThumbnails.cpp

@@ -0,0 +1,102 @@
+//
+// Created by kj16609 on 11/14/25.
+//
+
+#include "../APIMaintenance.hpp"
+
+#include "../../paths.hpp"
+#include "../../logging/log.hpp"
+
+#include <filesystem>
+
+namespace idhan::api
+{
+
+drogon::Task< drogon::HttpResponsePtr > APIMaintenance::purgeThumbnails( drogon::HttpRequestPtr request )
+{
+	try
+	{
+		const auto thumbnails_path { getThumbnailsPath() };
+
+		if ( !std::filesystem::exists( thumbnails_path ) )
+		{
+			log::warn( "Thumbnails directory does not exist: {}", thumbnails_path.string() );
+
+			Json::Value response;
+			response[ "success" ] = true;
+			response[ "message" ] = "Thumbnails directory does not exist";
+			response[ "deleted_count" ] = 0;
+
+			auto resp { drogon::HttpResponse::newHttpJsonResponse( response ) };
+			resp->setStatusCode( drogon::k200OK );
+			co_return resp;
+		}
+
+		std::size_t deleted_count { 0 };
+		std::size_t failed_count { 0 };
+
+		log::info( "Starting thumbnail purge from: {}", thumbnails_path.string() );
+
+		// Iterate through all files in the thumbnails directory
+		for ( const auto& entry : std::filesystem::recursive_directory_iterator( thumbnails_path ) )
+		{
+			if ( entry.is_regular_file() )
+			{
+				try
+				{
+					std::filesystem::remove( entry.path() );
+					++deleted_count;
+					log::trace( "Deleted thumbnail: {}", entry.path().string() );
+				}
+				catch ( const std::filesystem::filesystem_error& e )
+				{
+					++failed_count;
+					log::error( "Failed to delete thumbnail {}: {}", entry.path().string(), e.what() );
+				}
+			}
+		}
+
+		// Clean up empty directories
+		try
+		{
+			for ( const auto& entry : std::filesystem::recursive_directory_iterator( thumbnails_path ) )
+			{
+				if ( entry.is_directory() && std::filesystem::is_empty( entry.path() ) )
+				{
+					std::filesystem::remove( entry.path() );
+					log::trace( "Removed empty directory: {}", entry.path().string() );
+				}
+			}
+		}
+		catch ( const std::filesystem::filesystem_error& e )
+		{
+			log::warn( "Error cleaning up empty directories: {}", e.what() );
+		}
+
+		log::info( "Thumbnail purge complete. Deleted: {}, Failed: {}", deleted_count, failed_count );
+
+		Json::Value response;
+		response[ "success" ] = true;
+		response[ "deleted_count" ] = static_cast< Json::UInt64 >( deleted_count );
+		response[ "failed_count" ] = static_cast< Json::UInt64 >( failed_count );
+		response[ "message" ] = "Thumbnails purged successfully";
+
+		auto resp { drogon::HttpResponse::newHttpJsonResponse( response ) };
+		resp->setStatusCode( drogon::k200OK );
+		co_return resp;
+	}
+	catch ( const std::exception& e )
+	{
+		log::error( "Error purging thumbnails: {}", e.what() );
+
+		Json::Value response;
+		response[ "success" ] = false;
+		response[ "error" ] = e.what();
+
+		auto resp { drogon::HttpResponse::newHttpJsonResponse( response ) };
+		resp->setStatusCode( drogon::k500InternalServerError );
+		co_return resp;
+	}
+}
+
+} // namespace idhan::api

IDHANServer/src/api/record/createRecord.cpp

@@ -2,10 +2,10 @@
 // Created by kj16609 on 11/17/24.
 //
 
+#include "../../records/records.hpp"
 #include "IDHANTypes.hpp"
 #include "api/RecordAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
-#include "api/helpers/records.hpp"
 #include "crypto/SHA256.hpp"
 #include "fgl/defines.hpp"
 #include "logging/ScopedTimer.hpp"

IDHANServer/src/api/record/tags/addTags.cpp

@@ -243,7 +243,7 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::addTags(
 
 	if ( !tag_pair_ids ) co_return tag_pair_ids.error();
 
-	const auto tag_domain_id { helpers::getTagDomainID( request ) };
+	const auto tag_domain_id { helpers::getTagDomainIDParameter( request ) };
 
 	if ( !tag_domain_id ) co_return tag_domain_id.error();
 
@@ -271,7 +271,7 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::addMultipleTags( drogon::Http
 	if ( !json[ "records" ].isArray() )
 		co_return createBadRequest( "Invalid json: Array of ids called 'records' must be present." );
 
-	const auto tag_domain_id { helpers::getTagDomainID( request ) };
+	const auto tag_domain_id { helpers::getTagDomainIDParameter( request ) };
 
 	if ( !tag_domain_id ) co_return tag_domain_id.error();
 

IDHANServer/src/api/record/urls/addUrls.cpp

@@ -4,10 +4,10 @@
 
 #include <expected>
 
+#include "../../../urls/urls.hpp"
 #include "IDHANTypes.hpp"
 #include "api/RecordAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
-#include "api/helpers/urls.hpp"
 
 namespace idhan::api
 {
@@ -27,13 +27,20 @@ drogon::Task< drogon::HttpResponsePtr > RecordAPI::addUrls( drogon::HttpRequestP
 	{
 		const auto url_id { co_await helpers::findOrCreateUrl( url.asString(), db ) };
 
-		if ( !url_id ) co_return url_id.error();
+		if ( !url_id )
+		{
+			log::error( "Failed to find or create url: {}", url.asString() );
+			co_return url_id.error();
+		}
 
 		co_await db->execSqlCoro(
-			"INSERT INTO url_mappings (url_id, record_id) VALUES ($1, $2)", url_id.value(), record_id );
+			"INSERT INTO url_mappings (url_id, record_id) VALUES ($1, $2) ON CONFLICT DO NOTHING", *url_id, record_id );
 	}
 
-	co_return drogon::HttpResponse::newHttpResponse();
+	Json::Value result {};
+	result[ "status" ] = drogon::k200OK;
+
+	co_return drogon::HttpResponse::newHttpJsonResponse( result );
 }
 
 } // namespace idhan::api

IDHANServer/src/api/record/urls/removeUrls.cpp

@@ -2,10 +2,10 @@
 // Created by kj16609 on 7/24/25.
 //
 
+#include "../../../urls/urls.hpp"
 #include "IDHANTypes.hpp"
 #include "api/RecordAPI.hpp"
 #include "api/helpers/createBadRequest.hpp"
-#include "api/helpers/urls.hpp"
 
 namespace idhan::api
 {

IDHANServer/src/api/record/urls/urls.hpp

@@ -5,10 +5,10 @@
 #include <expected>
 
 #include "IDHANTypes.hpp"
-#include "api/helpers/ExpectedTask.hpp"
 #include "db/dbTypes.hpp"
 #include "drogon/orm/DbClient.h"
 #include "drogon/utils/coroutine.h"
+#include "threading/ExpectedTask.hpp"
 
 namespace idhan::helpers
 {

IDHANServer/src/api/relationships/addAlternative.cpp

@@ -0,0 +1,124 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+
+#include "IDHANTypes.hpp"
+#include "api/FileRelationshipsAPI.hpp"
+#include "api/helpers/createBadRequest.hpp"
+#include "db/drogonArrayBind.hpp"
+
+namespace idhan::api
+{
+
+using GroupID = idhan::Integer;
+
+drogon::Task< GroupID > createNewGroup( DbClientPtr db )
+{
+	const auto group_result {
+		co_await db->execSqlCoro( "INSERT INTO alternative_groups DEFAULT VALUES RETURNING group_id" )
+	};
+
+	const auto group_id { group_result[ 0 ][ 0 ].as< GroupID >() };
+	co_return group_id;
+}
+
+drogon::Task<> addItemsToNewGroup( std::vector< RecordID > record_ids, DbClientPtr db )
+{
+	const auto group_id { co_await createNewGroup( db ) };
+	co_await db->execSqlCoro(
+		"INSERT INTO alternative_group_members (group_id, record_id) VALUES ($1, UNNEST($2::" RECORD_PG_TYPE_NAME
+		"[]))",
+		group_id,
+		std::move( record_ids ) );
+}
+
+drogon::Task<> addItemsToExistingGroup(
+	std::vector< RecordID > record_ids,
+	const std::vector< GroupID > group_ids,
+	drogon::orm::DbClientPtr db )
+{
+	const auto group_id { group_ids[ 0 ] };
+
+	co_await db->execSqlCoro(
+		"INSERT INTO alternative_group_members (group_id, record_id) VALUES ($1, UNNEST($2::" RECORD_PG_TYPE_NAME
+		"[])) ON CONFLICT (group_id, record_id) DO NOTHING",
+		group_id,
+		std::move( record_ids ) );
+}
+
+drogon::Task<> addItemsToExistingGroupsMerge(
+	std::vector< RecordID > record_ids,
+	std::vector< GroupID > group_ids,
+	drogon::orm::DbClientPtr db )
+{
+	const auto group_id { co_await createNewGroup( db ) };
+
+	co_await db->execSqlCoro(
+		"UPDATE alternative_group_members SET group_id = $1 WHERE group_id = ANY($2)",
+		group_id,
+		std::forward< std::vector< GroupID > >( group_ids ) );
+
+	co_await db->execSqlCoro(
+		"INSERT INTO alternative_group_members (group_id, record_id) VALUES ($1, UNNEST($2::" RECORD_PG_TYPE_NAME
+		"[])) ON CONFLICT (group_id, record_id) DO NOTHING",
+		group_id,
+		std::forward< std::vector< RecordID > >( record_ids ) );
+
+	co_await db->execSqlCoro(
+		"DELETE FROM alternative_groups WHERE group_id = ANY($1)",
+		std::forward< std::vector< GroupID > >( group_ids ) );
+}
+
+drogon::Task< drogon::HttpResponsePtr > FileRelationshipsAPI::addAlternative( drogon::HttpRequestPtr request )
+{
+	auto db { drogon::app().getDbClient() };
+
+	const auto json_ptr { request->getJsonObject() };
+	if ( !json_ptr ) co_return createBadRequest( "Expected json body" );
+
+	const auto& json { *json_ptr };
+
+	if ( !json.isArray() ) co_return createBadRequest( "Expected json array of integers" );
+
+	std::vector< RecordID > record_ids {};
+
+	for ( const auto& id : json )
+	{
+		record_ids.emplace_back( id.as< RecordID >() );
+	}
+
+	const auto group_search { co_await db->execSqlCoro(
+		"SELECT DISTINCT group_id FROM alternative_group_members WHERE record_id = ANY($1)",
+		std::forward< std::vector< RecordID > >( record_ids ) ) };
+
+	std::vector< GroupID > group_ids {};
+
+	for ( const auto& id : group_search )
+	{
+		group_ids.emplace_back( id[ 0 ].as< GroupID >() );
+	}
+
+	std::ranges::sort( group_ids );
+	std::ranges::unique( group_ids );
+
+	const bool no_existing_groups { group_ids.empty() };
+	const bool existing_groups { not no_existing_groups };
+	const bool multiple_groups { not no_existing_groups && group_ids.size() > 1 };
+
+	if ( no_existing_groups )
+	{
+		co_await addItemsToNewGroup( record_ids, db );
+	}
+	else if ( existing_groups && !multiple_groups )
+	{
+		co_await addItemsToExistingGroup( record_ids, group_ids, db );
+	}
+	else if ( multiple_groups )
+	{
+		co_await addItemsToExistingGroupsMerge( record_ids, group_ids, db );
+	}
+
+	co_return drogon::HttpResponse::newHttpJsonResponse( {} );
+}
+
+} // namespace idhan::api

IDHANServer/src/api/relationships/setBetterDuplicate.cpp

@@ -0,0 +1,61 @@
+//
+// Created by kj16609 on 11/5/25.
+//
+
+#include "IDHANTypes.hpp"
+#include "api/FileRelationshipsAPI.hpp"
+#include "api/helpers/createBadRequest.hpp"
+
+namespace idhan::api
+{
+
+drogon::Task< drogon::HttpResponsePtr > setBetterDuplicateMultiple( const Json::Value json )
+{
+	auto db { drogon::app().getDbClient() };
+
+	for ( const auto& object : json )
+	{
+		if ( !object.isMember( "worse_id" ) || !object.isMember( "better_id" ) )
+			co_return createBadRequest( "Expected object to have `worse_id` and `better_id`" );
+
+		if ( !object[ "worse_id" ].isUInt() || !object[ "better_id" ].isUInt() )
+			co_return createBadRequest( "Expected `worse_id` and `better_id` to be unsigned integers" );
+
+		const RecordID worse_id { object[ "worse_id" ].as< RecordID >() };
+		const RecordID better_id { object[ "better_id" ].as< RecordID >() };
+
+		co_await db->execSqlCoro( "SELECT insert_duplicate_pair($1, $2)", worse_id, better_id );
+	}
+
+	co_return drogon::HttpResponse::newHttpJsonResponse( {} );
+}
+
+drogon::Task< drogon::HttpResponsePtr > setBetterDuplicateSingle( const Json::Value json )
+{
+	auto db { drogon::app().getDbClient() };
+
+	if ( !json[ "worse_id" ].isUInt() || !json[ "better_id" ].isUInt() )
+		co_return createBadRequest( "Expected `worse_id` and `better_id` to be unsigned integers" );
+
+	const RecordID worse_id { json[ "worse_id" ].as< RecordID >() };
+	const RecordID better_id { json[ "better_id" ].as< RecordID >() };
+
+	co_await db->execSqlCoro( "SELECT insert_duplicate_pair($1, $2)", worse_id, better_id );
+
+	co_return drogon::HttpResponse::newHttpJsonResponse( {} );
+}
+
+drogon::Task< drogon::HttpResponsePtr > FileRelationshipsAPI::setBetterDuplicate( const drogon::HttpRequestPtr request )
+{
+	const auto json_ptr { request->getJsonObject() };
+	if ( !json_ptr ) co_return createBadRequest( "Expected json body" );
+	const auto& json { *json_ptr };
+
+	if ( json.isArray() ) co_return co_await setBetterDuplicateMultiple( json );
+	if ( json.isObject() ) co_return co_await setBetterDuplicateSingle( json );
+
+	co_return createBadRequest(
+		"Expected json body of either array of objects, or a single object. Objects must have `better_id` and `worse_id`" );
+}
+
+} // namespace idhan::api

IDHANServer/src/api/search/search.cpp

@@ -31,7 +31,7 @@ drogon::Task< drogon::HttpResponsePtr > SearchAPI::search( drogon::HttpRequestPt
 
 	SearchBuilder builder {};
 
-	builder.setTags( tag_ids );
+	builder.setPositiveTags( tag_ids );
 
 	const auto result { co_await builder.query( db, tag_domain_ids ) };
 

IDHANServer/src/api/tags/alias/createAlias.cpp

@@ -29,7 +29,7 @@ drogon::Task< drogon::HttpResponsePtr > TagAPI::createTagAliases( drogon::HttpRe
 
 	const auto db { drogon::app().getDbClient() };
 
-	const auto tag_domain_id { helpers::getTagDomainID( request ) };
+	const auto tag_domain_id { helpers::getTagDomainIDParameter( request ) };
 
 	if ( !tag_domain_id ) co_return tag_domain_id.error();
 

IDHANServer/src/api/tags/autocompleteTag.cpp

@@ -15,7 +15,9 @@ drogon::Task< Json::Value > getSimilarTags(
 {
 	LOG_DEBUG << "Searching for tag \"{}\"" << search_value;
 
-	const auto wrapped_search_value { '%' + search_value + '%' };
+	const bool is_negative { search_value.starts_with( '-' ) };
+	const std::string real_search_value { is_negative ? search_value.substr( 1 ) : search_value };
+	const auto wrapped_search_value { format_ns::format( "%{}%", real_search_value ) };
 
 	constexpr std::size_t max_limit { 32 };
 
@@ -41,7 +43,7 @@ drogon::Task< Json::Value > getSimilarTags(
 		limit $3
 		)",
 		wrapped_search_value,
-		search_value,
+		real_search_value,
 		std::min( limit, max_limit ) ) };
 
 	Json::Value tags { Json::arrayValue };
@@ -50,8 +52,10 @@ drogon::Task< Json::Value > getSimilarTags(
 	{
 		Json::Value tag;
 
-		tag[ "value" ] = row[ "tag_text" ].as< std::string >();
-		tag[ "tag_text" ] = row[ "tag_text" ].as< std::string >();
+		const auto tag_text { row[ "tag_text" ].as< std::string >() };
+
+		tag[ "value" ] = tag_text;
+		tag[ "tag_text" ] = tag_text;
 
 		tag[ "similarity" ] = row[ "similarity" ].as< double >();
 		tag[ "tag_id" ] = row[ "tag_id" ].as< TagID >();

IDHANServer/src/api/tags/createTag.cpp

@@ -18,8 +18,8 @@ struct hash< std::pair< std::string, std::string > >
 };
 } // namespace std
 
+#include "threading/ExpectedTask.hpp"
 #include "api/TagAPI.hpp"
-#include "api/helpers/ExpectedTask.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "db/drogonArrayBind.hpp"
 #include "fgl/defines.hpp"

IDHANServer/src/api/tags/parents/createParents.cpp

@@ -30,7 +30,7 @@ drogon::Task< drogon::HttpResponsePtr > TagAPI::createTagParents( const drogon::
 
 	const auto db { drogon::app().getDbClient() };
 
-	const auto tag_domain_id { helpers::getTagDomainID( request ) };
+	const auto tag_domain_id { helpers::getTagDomainIDParameter( request ) };
 
 	if ( !tag_domain_id ) co_return tag_domain_id.error();
 

IDHANServer/src/api/version.cpp

@@ -2,12 +2,14 @@
 // Created by kj16609 on 11/8/24.
 //
 
+#include "version.hpp"
+
 #include <paths.hpp>
 
 #include "InfoAPI.hpp"
 #include "hyapi/constants/hydrus_version.hpp"
+#include "idhan/versions.hpp"
 #include "logging/log.hpp"
-#include "versions.hpp"
 
 namespace idhan::api
 {
@@ -19,10 +21,10 @@ drogon::Task< drogon::HttpResponsePtr > InfoAPI::version( [[maybe_unused]] drogo
 	Json::Value json;
 
 	json[ "idhan_server_version" ][ "string" ] =
-		format_ns::format( "{}.{}.{}", IDHAN_SERVER_MAJOR, IDHAN_SERVER_MINOR, IDHAN_SERVER_PATCH );
-	json[ "idhan_server_version" ][ "major" ] = IDHAN_SERVER_MAJOR;
-	json[ "idhan_server_version" ][ "minor" ] = IDHAN_SERVER_MINOR;
-	json[ "idhan_server_version" ][ "patch" ] = IDHAN_SERVER_PATCH;
+		format_ns::format( "{}.{}.{}", IDHAN_MAJOR_VERSION, IDHAN_MINOR_VERSION, IDHAN_PATCH_VERSION );
+	json[ "idhan_server_version" ][ "major" ] = IDHAN_MAJOR_VERSION;
+	json[ "idhan_server_version" ][ "minor" ] = IDHAN_MINOR_VERSION;
+	json[ "idhan_server_version" ][ "patch" ] = IDHAN_PATCH_VERSION;
 
 	json[ "idhan_api_version" ][ "string" ] =
 		format_ns::format( "{}.{}.{}", IDHAN_API_MAJOR, IDHAN_API_MINOR, IDHAN_API_PATCH );
@@ -37,6 +39,7 @@ drogon::Task< drogon::HttpResponsePtr > InfoAPI::version( [[maybe_unused]] drogo
 	json[ "commit" ] = FGL_GIT_COMMIT;
 	json[ "tag" ] = FGL_GIT_TAG;
 	json[ "build" ] = FGL_BUILD_TYPE;
+	json[ "build_on" ] = IDHAN_BUILD_DATE ", " IDHAN_BUILD_TIME;
 
 	co_return drogon::HttpResponse::newHttpJsonResponse( json );
 }

IDHANServer/src/api/version.hpp

@@ -0,0 +1,12 @@
+//
+// Created by kj16609 on 11/8/25.
+//
+#pragma once
+
+#include "idhan/versions.hpp"
+
+#define IDHAN_API_MAJOR 0
+#define IDHAN_API_MINOR 0
+#define IDHAN_API_PATCH 0
+
+#define IDHAN_API_VERSION MAKE_IDHAN_VERSION( IDHAN_API_MAJOR, IDHAN_API_MINOR, IDHAN_API_PATCH )

IDHANServer/src/core/record/getRecordSHA256.cpp

@@ -2,8 +2,8 @@
 // Created by kj16609 on 11/19/24.
 //
 
+#include "threading/ExpectedTask.hpp"
 #include "IDHANTypes.hpp"
-#include "api/helpers/ExpectedTask.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "crypto/SHA256.hpp"
 #include "drogon/HttpAppFramework.h"

IDHANServer/src/core/search/SearchBuilder.cpp

@@ -4,68 +4,200 @@
 
 #include "SearchBuilder.hpp"
 
+#include <ranges>
+
 #include "api/helpers/helpers.hpp"
 #include "db/drogonArrayBind.hpp"
+#include "drogon/HttpAppFramework.h"
 #include "fgl/defines.hpp"
 #include "logging/log.hpp"
+#include "tags/tags.hpp"
 
 namespace idhan
 {
 
-std::string SearchBuilder::construct(
-	const bool return_ids,
-	const bool return_hashes,
-	[[maybe_unused]] const bool filter_domains )
+void SearchBuilder::parseRangeSearch( RangeSearchInfo& target, std::string_view tag )
 {
-	// TODO: Sort tag ids to get the most out of each filter.
+	target.m_active = true;
 
-	std::string query {};
-	query.reserve( 1024 );
+	const bool is_greater_than { tag.contains( ">" ) };
+	const bool is_less_than { tag.contains( "<" ) };
+	const bool is_equal_to { tag.contains( "=" ) };
+	const bool is_not { tag.contains( "!" ) || tag.contains( "≠" ) }; // ew
+	const bool is_approximate { tag.contains( "~" ) };
 
-	if ( m_tags.empty() )
+	SearchOperation op { 0 };
+	if ( is_greater_than ) op |= SearchOperationFlags::GreaterThan;
+	if ( is_less_than ) op |= SearchOperationFlags::LessThan;
+	if ( is_equal_to ) op |= SearchOperationFlags::Equal;
+	if ( is_not ) op |= SearchOperationFlags::Not;
+	if ( is_approximate ) op |= SearchOperationFlags::Approximate;
+	target.operation = op;
+
+	log::debug( "Parsing range for {}", tag );
+
+	// find begining of number
+	const auto number_start { tag.find_first_of( "0123456789" ) };
+	const auto number_end { tag.find_last_of( "0123456789" ) };
+
+	const std::string number_substr { tag.substr( number_start, number_end ) };
+
+	log::debug( "Got number from \'{}\'", number_substr );
+
+	try
 	{
-		return "SELECT record_id FROM file_info WHERE mime_id IS NOT NULL";
+		std::size_t remaining_characters_pos { 0 };
+		target.count = std::stoull( number_substr, &remaining_characters_pos );
 	}
+	catch ( std::exception& e )
+	{
+		throw std::invalid_argument(
+			format_ns::format( "Failed to parse number using stoull: {}: {}", tag, e.what() ) );
+	}
+}
 
+std::unordered_map< TagID, std::string > SearchBuilder::createFilters(
+	const std::vector< TagID >& tag_ids,
+	const bool filter_domains )
+{
+	std::unordered_map< TagID, std::string > filters {};
+	filters.reserve( tag_ids.size() );
+
+	// 0 == filter_id, 1 == tag_id
+	// uses $1 for domains
 	constexpr std::string_view domain_filter_template {
-		"filter_{0} AS ( SELECT DISTINCT record_id FROM active_tag_mappings WHERE tag_id = {1} AND ideal_tag_id IS NULL AND tag_domain_id = ANY($1) UNION DISTINCT SELECT DISTINCT record_id FROM active_tag_mappings WHERE ideal_tag_id = {1} AND tag_domain_id = ANY($1) UNION DISTINCT SELECT DISTINCT record_id FROM active_tag_mappings_parents WHERE tag_id = {1} AND tag_domain_id = ANY($1) )"
-		// "filter_{0} AS (SELECT record_id FROM active_tag_mappings WHERE ideal_tag_id IS NULL AND tag_id = {1} AND tag_domain_id = "
-		// "ANY($1) UNION DISTINCT SELECT record_id FROM active_tag_mappings_parents WHERE tag_id = {1} AND tag_domain_id "
-		// "= ANY($1))"
+		"filter_{0} AS ( SELECT record_id FROM active_tag_mappings WHERE tag_id = {1} AND ideal_tag_id IS NULL AND tag_domain_id = ANY($1) UNION DISTINCT SELECT record_id FROM active_tag_mappings WHERE ideal_tag_id = {1} AND tag_domain_id = ANY($1) UNION DISTINCT SELECT record_id FROM active_tag_mappings_parents WHERE tag_id = {1} AND tag_domain_id = ANY($1) )"
 	};
 
+	// 0 == filter_id, 1 == tag_id
+	// Has no binds
 	constexpr std::string_view domainless_filter_template {
-		"filter_{0} AS ( SELECT DISTINCT record_id FROM active_tag_mappings WHERE tag_id = {1} AND ideal_tag_id IS NULL UNION DISTINCT SELECT DISTINCT record_id FROM active_tag_mappings WHERE ideal_tag_id = {1} UNION DISTINCT SELECT DISTINCT record_id FROM active_tag_mappings_parents WHERE tag_id = {1} )"
+		"filter_{0} AS ( SELECT record_id FROM active_tag_mappings WHERE tag_id = {1} AND ideal_tag_id IS NULL UNION DISTINCT SELECT record_id FROM active_tag_mappings WHERE ideal_tag_id = {1} UNION DISTINCT SELECT record_id FROM active_tag_mappings_parents WHERE tag_id = {1} )"
 	};
 
-	m_bind_domains = filter_domains;
-
-	query += "WITH ";
-
-	std::string final_filter { "final_filter AS (" };
-
-	for ( std::size_t i = 0; i < m_tags.size(); ++i )
+	for ( const auto& tag : tag_ids )
 	{
-		auto& tag { m_tags[ i ] };
+		const auto filled_template { filter_domains ? format_ns::format( domain_filter_template, tag, tag ) :
+			                                          format_ns::format( domainless_filter_template, tag, tag ) };
 
-		if ( filter_domains )
-			query += format_ns::format( domain_filter_template, i, tag );
-		else
-			query += format_ns::format( domainless_filter_template, i, tag );
-		final_filter += format_ns::format( "SELECT record_id FROM filter_{}", i );
-
-		if ( i + 1 != m_tags.size() )
-		{
-			query += ",\n";
-			// We have more to process
-			final_filter += " INTERSECT ";
-		}
+		filters.insert_or_assign( tag, filled_template );
 	}
 
-	query += ",\n";
-	query += final_filter;
-	query += ")\n";
+	return filters;
+}
 
+std::string SearchBuilder::buildPositiveFilter() const
+{
+	std::string positive_filter { "positive_filter AS (" };
+
+	if ( m_positive_tags.empty() )
+	{
+		// If there is no 'positive tags', we need to populate the positive filter with something to prevent it from returning nothing
+		positive_filter += "SELECT record_id FROM file_info WHERE mime_id IS NOT NULL),";
+		return positive_filter;
+	}
+
+	for ( auto itter = m_positive_tags.begin(); itter != m_positive_tags.end(); ++itter )
+	{
+		positive_filter += format_ns::format( "SELECT record_id FROM filter_{}", *itter );
+
+		if ( itter + 1 != m_positive_tags.end() )
+			positive_filter += " INTERSECT ";
+		else
+			positive_filter += "),";
+	}
+
+	return positive_filter;
+}
+
+std::string SearchBuilder::buildNegativeFilter() const
+{
+	std::string negative_filters { "negative_filter AS (" };
+
+	for ( auto itter = m_negative_tags.begin(); itter != m_negative_tags.end(); ++itter )
+	{
+		negative_filters += format_ns::format( "SELECT record_id FROM filter_{}", *itter );
+
+		if ( itter + 1 != m_negative_tags.end() )
+			negative_filters += " UNION DISTINCT ";
+		else
+			negative_filters += "),";
+	}
+
+	return negative_filters;
+}
+
+void SearchBuilder::generateOrderByClause( std::string& query ) const
+{
+	switch ( m_sort_type )
+	{
+		// DEFAULT and HY_* should not be used here.
+		default:
+			[[fallthrough]];
+		case SortType::FILESIZE:
+			query += " ORDER BY fm.size";
+			break;
+		case SortType::IMPORT_TIME:
+			query += " ORDER BY fm.cluster_store_time ";
+			break;
+		case SortType::RECORD_TIME:
+			query += " ORDER BY records.creation_time ";
+			break;
+	}
+}
+
+void SearchBuilder::determineJoinsForQuery( std::string& query )
+{
+	if ( m_duration_search == DurationSearchType::HasDuration )
+	{
+		m_required_joins.video_metadata |= true;
+	}
+
+	if ( m_duration_search == DurationSearchType::NoDuration )
+	{
+		m_required_joins.left_video_metadata |= true;
+	}
+
+	if ( m_width_search.m_active || m_height_search.m_active )
+	{
+		m_required_joins.left_image_metadata |= true;
+		m_required_joins.left_video_metadata |= true;
+	}
+
+	if ( m_required_joins.left_video_metadata )
+	{
+		query += " LEFT JOIN video_metadata USING (record_id)";
+	}
+
+	if ( m_required_joins.video_metadata && !m_required_joins.left_video_metadata )
+	{
+		query += " JOIN video_metadata USING (record_id)";
+	}
+
+	if ( m_required_joins.left_image_metadata )
+	{
+		query += " LEFT JOIN image_metadata USING (record_id)";
+	}
+
+	if ( m_required_joins.image_metadata && !m_required_joins.left_image_metadata )
+	{
+		query += " JOIN image_metadata USING (record_id)";
+	}
+
+	// determine any joins needed
+	if ( m_required_joins.records )
+	{
+		query += " JOIN records rc USING (record_id)";
+	}
+
+	if ( m_required_joins.file_info )
+	{
+		query += " JOIN file_info fm USING (record_id)";
+	}
+}
+
+void SearchBuilder::determineSelectClause( std::string& query, const bool return_ids, const bool return_hashes )
+{
 	// determine the SELECT
 	if ( return_ids && return_hashes )
 	{
@@ -84,40 +216,139 @@ std::string SearchBuilder::construct(
 		constexpr std::string_view select_record_id { " SELECT tm.record_id FROM final_filter tm" };
 		query += select_record_id;
 	}
+}
 
-	// determine any joins needed
-	if ( m_required_joins.records && false )
+void SearchBuilder::generateWhereClauses( std::string& query )
+{
+	// These are added after the join clauses
+	/*
+	// Not needed due to the JOIN being a filter
+	if ( m_duration_search == DurationSearchType::HasDuration )
 	{
-		query += " JOIN records rc ON rc.record_id = tm.record_id";
+		query += " WHERE vm_hd.duration IS NOT NULL";
+	}
+	*/
+
+	if ( m_duration_search == DurationSearchType::NoDuration )
+	{
+		query += " AND video_metadata.duration IS NULL";
 	}
 
-	if ( m_required_joins.file_info )
+	if ( m_height_search.m_active )
 	{
-		query += " JOIN file_info fm ON fm.record_id = tm.record_id";
+		const auto operation { m_height_search.operation };
+		if ( operation & SearchOperationFlags::Not )
+			query += " AND NOT";
+		else
+			query += " AND";
+
+		query += " COALESCE(image_metadata.height, video_metadata.height) ";
+
+		if ( operation & SearchOperationFlags::Equal )
+		{
+			query += "= ";
+		}
+
+		if ( operation & SearchOperationFlags::GreaterThan )
+		{
+			query += "> ";
+		}
+		else if ( operation & SearchOperationFlags::LessThan )
+		{
+			query += "< ";
+		}
+
+		query += std::to_string( m_height_search.count );
 	}
 
-	switch ( m_sort_type )
+	if ( m_width_search.m_active )
 	{
-		// DEFAULT and HY_* should not be used here.
-		default:
-			[[fallthrough]];
-		case SortType::FILESIZE:
-			query += " ORDER BY fm.size";
-			break;
-		case SortType::IMPORT_TIME:
-			query += " ORDER BY fm.cluster_store_time ";
-			break;
-		case SortType::RECORD_TIME:
-			query += " ORDER BY records.creation_time ";
-			break;
+		const auto operation { m_width_search.operation };
+		if ( operation & SearchOperationFlags::Not )
+			query += " AND NOT";
+		else
+			query += " AND";
+
+		query += " COALESCE(image_metadata.width, video_metadata.width) ";
+
+		if ( operation & SearchOperationFlags::Equal )
+		{
+			query += "= ";
+		}
+
+		if ( operation & SearchOperationFlags::GreaterThan )
+		{
+			query += "> ";
+		}
+		else if ( operation & SearchOperationFlags::LessThan )
+		{
+			query += "< ";
+		}
+
+		query += std::to_string( m_width_search.count );
 	}
+}
+
+std::string SearchBuilder::construct( const bool return_ids, const bool return_hashes, const bool filter_domains )
+{
+	// TODO: Sort tag ids to get the most out of each filter.
+
+	std::string query { "WITH " };
+	query.reserve( 1024 );
+
+	if ( m_positive_tags.empty() && m_negative_tags.empty() )
+	{
+		// return "SELECT record_id FROM file_info WHERE mime_id IS NOT NULL";
+	}
+
+	std::vector< TagID > filtered_tags {};
+	filtered_tags.reserve( 16 );
+	std::ranges::copy( m_positive_tags, std::back_inserter( filtered_tags ) );
+	std::ranges::copy( m_negative_tags, std::back_inserter( filtered_tags ) );
+	const auto filter_map { createFilters( filtered_tags, filter_domains ) };
+	const auto positive_filter { buildPositiveFilter() };
+	const auto negative_filter { buildNegativeFilter() };
+
+	std::string final_filter {};
+
+	if ( m_negative_tags.size() > 0 )
+	{
+		final_filter +=
+			"final_filter AS (SELECT record_id FROM positive_filter EXCEPT SELECT record_id FROM negative_filter)";
+	}
+	else
+	{
+		final_filter += "final_filter AS (SELECT DISTINCT record_id FROM positive_filter)";
+	}
+
+	m_bind_domains = filter_domains;
+
+	for ( const auto& filter : filter_map | std::views::values )
+	{
+		query += filter + ",";
+	}
+	query += positive_filter;
+	if ( m_negative_tags.size() > 0 ) query += negative_filter;
+	query += final_filter;
+
+	log::info( "{}", query );
+
+	determineSelectClause( query, return_ids, return_hashes );
+
+	determineJoinsForQuery( query );
+
+	query += " WHERE fm.mime_id IS NOT NULL";
+
+	generateWhereClauses( query );
+
+	generateOrderByClause( query );
 
 	query += ( m_order == SortOrder::ASC ? " ASC" : " DESC" );
 
 	return query;
 }
 
-SearchBuilder::SearchBuilder() : m_sort_type(), m_order(), m_tags(), m_display_mode()
+SearchBuilder::SearchBuilder() : m_sort_type(), m_order(), m_positive_tags(), m_display_mode()
 {}
 
 drogon::Task< drogon::orm::Result > SearchBuilder::query(
@@ -179,9 +410,252 @@ void SearchBuilder::addFileDomain( [[maybe_unused]] const FileDomainID value )
 	FGL_UNIMPLEMENTED();
 }
 
-void SearchBuilder::setTags( const std::vector< TagID >& vector )
+ExpectedTask< void > SearchBuilder::setTags( const std::vector< std::string >& tags )
 {
-	m_tags = std::move( vector );
+	std::vector< std::string > positive_tags {};
+	std::vector< std::string > negative_tags {};
+
+	for ( const auto& tag : tags )
+	{
+		if ( tag.starts_with( "-" ) )
+			negative_tags.push_back( tag.substr( 1 ) );
+		else
+			positive_tags.push_back( tag );
+	}
+
+	auto db { drogon::app().getDbClient() };
+
+	const auto positive_map { co_await mapTags( positive_tags, db ) };
+	const auto negative_map { co_await mapTags( negative_tags, db ) };
+	return_unexpected_error( positive_map );
+	return_unexpected_error( negative_map );
+
+	std::vector< TagID > positive_ids {};
+	for ( const auto& tag_id : *positive_map | std::views::values ) positive_ids.emplace_back( tag_id );
+	std::vector< TagID > negative_ids {};
+	for ( const auto& tag_id : *negative_map | std::views::values ) negative_ids.emplace_back( tag_id );
+
+	setPositiveTags( positive_ids );
+	setNegativeTags( negative_ids );
+
+	co_return {};
+}
+
+void SearchBuilder::setPositiveTags( const std::vector< TagID >& vector )
+{
+	m_positive_tags = vector;
+}
+
+void SearchBuilder::setNegativeTags( const std::vector< TagID >& tag_ids )
+{
+	m_negative_tags = tag_ids;
+}
+
+void SearchBuilder::setSystemTags( const std::vector< std::string >& vector )
+{
+	log::debug( "Got {} system tags", vector.size() );
+	for ( const auto& tag : vector )
+	{
+		constexpr auto system_namespace { "system:" };
+		constexpr auto system_namespace_len { 7 };
+		if ( !tag.starts_with( system_namespace ) )
+			throw std::invalid_argument( format_ns::format( "Invalid system namespace: {}", tag ) );
+
+		const std::string_view system_subtag { std::string_view { tag }.substr( system_namespace_len ) };
+
+		log::debug( "Got system tag \'{}\'", system_subtag );
+
+		// system:everything
+		if ( system_subtag == "everything" )
+		{
+			m_search_everything = true;
+			continue;
+		}
+		// system:inbox
+		// system:archive
+		// system:has duration
+		if ( system_subtag == "has duration" )
+		{
+			m_duration_search = DurationSearchType::HasDuration;
+			continue;
+		}
+		// system:no duration
+		if ( system_subtag == "no duration" )
+		{
+			m_duration_search = DurationSearchType::NoDuration;
+			continue;
+		}
+		// system:is the best quality file of its duplicate group
+		// system:is not the best quality file of its duplicate group
+		// system:has audio
+		if ( system_subtag == "has audio" )
+		{
+			m_audio_search = AudioSearchType::HasAudio;
+			continue;
+		}
+		// system:no audio
+		if ( system_subtag == "no audio" )
+		{
+			m_audio_search = AudioSearchType::NoAudio;
+			continue;
+		}
+		// system:has exif
+		if ( system_subtag == "has exif" )
+		{
+			m_exif_search = ExitSearchType::HasExif;
+			continue;
+		}
+		// system:no exif
+		if ( system_subtag == "no exif" )
+		{
+			m_exif_search = ExitSearchType::NoExif;
+			continue;
+		}
+		// system:has embedded metadata
+		// system:no embedded metadata
+		// system:has icc profile
+		// system:no icc profile
+		// system:has tags
+		if ( system_subtag == "has tags" )
+		{
+			m_has_tags_search = TagCountSearchType::HasTags;
+			continue;
+		}
+		// system:no tags // system:untagged // MERGED
+		if ( ( system_subtag == "no tags" ) || ( system_subtag == "untagged" ) )
+		{
+			m_has_tags_search = TagCountSearchType::NoTags;
+			continue;
+		}
+		// system:number of tags > 5 // system:number of tags ~= 10 // system:number of tags > 0
+		if ( system_subtag.starts_with( "number of tags" ) )
+		{
+			parseRangeSearch( m_tag_count_search, system_subtag );
+			continue;
+		}
+
+		// system:number of words < 2
+		// system:height = 600 // system:height > 900
+		if ( system_subtag.starts_with( "height" ) )
+		{
+			parseRangeSearch( m_height_search, system_subtag );
+			continue;
+		}
+		// system:width < 200 // system:width > 1000
+		if ( system_subtag.starts_with( "width" ) )
+		{
+			parseRangeSearch( m_width_search, system_subtag );
+			continue;
+		}
+		// system:filesize ~= 50 kilobytes // system:filesize > 10megabytes // system:filesize < 1 GB // system:filesize > 0 B
+		if ( system_subtag.starts_with( "filesize" ) )
+		{
+			continue;
+		}
+		// system:similar to abcdef01 abcdef02 abcdef03, abcdef04 with distance 3
+		// system:similar to abcdef distance 5
+		// system:limit = 100
+		if ( system_subtag.starts_with( "limit" ) )
+		{
+			parseRangeSearch( m_limit_search, system_subtag );
+			continue;
+		}
+		// system:filetype = image/jpg, image/png, apng
+		if ( system_subtag.starts_with( "filetype" ) )
+		{
+			continue;
+		}
+		// system:hash = abcdef01 abcdef02 abcdef03 (this does sha256)
+		// system:hash = abcdef01 abcdef02 md5
+		if ( system_subtag.starts_with( "hash" ) )
+		{
+			continue;
+		}
+		// system:modified date < 7 years 45 days 7h // system:modified date > 2011-06-04
+		// system:date modified > 7 years 2 months // system:date modified < 0 years 1 month 1 day 1 hour
+		if ( system_subtag.starts_with( "modified date" ) || system_subtag.starts_with( "date modified" ) )
+		{
+			continue;
+		}
+		// system:last viewed time < 7 years 45 days 7h
+		// system:last view time < 7 years 45 days 7h
+		// system:import time < 7 years 45 days 7h
+		// system:time imported < 7 years 45 days 7h // system:time imported > 2011-06-04
+		// system:time imported > 7 years 2 months // system:time imported < 0 years 1 month 1 day 1 hour
+		// system:time imported ~= 2011-1-3 // system:time imported ~= 1996-05-2
+		if ( system_subtag.starts_with( "time imported" ) )
+		{
+			continue;
+		}
+		// system:duration < 5 seconds
+		// system:duration ~= 600 msecs
+		// system:duration > 3 milliseconds
+		// system:file service is pending to my files
+		// system:file service currently in my files
+		// system:file service is not currently in my files
+		// system:file service is not pending to my files
+		// system:number of file relationships = 2 duplicates
+		// system:number of file relationships > 10 potential duplicates
+		// system:num file relationships < 3 alternates
+		// system:num file relationships > 3 false positives
+		// system:ratio is wider than 16:9
+		if ( system_subtag.starts_with( "ratio wider than" ) )
+		{
+			continue;
+		}
+		// system:ratio is 16:9
+		if ( system_subtag.starts_with( "ratio is" ) )
+		{
+			continue;
+		}
+		// system:ratio taller than 1:1
+		if ( system_subtag.starts_with( "ratio taller than" ) )
+		{
+			continue;
+		}
+		// system:num pixels > 50 px // system:num pixels < 1 megapixels // system:num pixels ~= 5 kilopixel
+		if ( system_subtag.starts_with( "num pixels" ) )
+		{
+			continue;
+		}
+		// system:views in media ~= 10
+		// system:views in preview < 10
+		// system:views > 0
+		// system:viewtime in client api < 1 days 1 hour 0 minutes
+		// system:viewtime in media, client api, preview ~= 1 day 30 hours 100 minutes 90s
+		// system:has url matching regex index\.php
+		// system:does not have a url matching regex index\.php
+		// system:has url https://somebooru.org/posts/123456
+		if ( system_subtag.starts_with( "has url" ) )
+		{
+			continue;
+		}
+		// system:does not have url https://somebooru.org/posts/123456
+		if ( system_subtag.starts_with( "does not have url" ) )
+		{
+			continue;
+		}
+		// system:has domain safebooru.com
+		// system:does not have domain safebooru.com
+		// system:has a url with class safebooru file page
+		// system:does not have a url with url class safebooru file page
+		// system:tag as number page < 5
+		// system:has notes
+		// system:no notes
+		// system:does not have notes
+		// system:num notes is 5
+		// system:num notes > 1
+		// system:has note with name note name
+		// system:no note with name note name
+		// system:does not have note with name note name
+		// system:has a rating for service_name
+		// system:does not have a rating for service_name
+		// system:rating for service_name > ⅗ (numerical services)
+		// system:rating for service_name is like (like/dislike services)
+		// system:rating for service_name = 13 (inc/dec services)
+
+		log::warn( "Unsupported system tag system: \'{}\'", system_subtag );
+	}
 }
 
 void SearchBuilder::setDisplay( const HydrusDisplayType type )

IDHANServer/src/core/search/SearchBuilder.hpp

@@ -3,10 +3,12 @@
 //
 #pragma once
 
+#include <expected>
 #include <string_view>
 
 #include "IDHANTypes.hpp"
 #include "SearchBuilder.hpp"
+#include "api/APIAuth.hpp"
 #include "db/dbTypes.hpp"
 #include "drogon/HttpRequest.h"
 #include "drogon/orm/DbClient.h"
@@ -156,15 +158,103 @@ class SearchBuilder
 	{
 		bool file_info { false };
 		bool records { false };
+
+		bool left_video_metadata { false };
+		bool video_metadata { false };
+
+		bool left_image_metadata { false };
+		bool image_metadata { false };
 	} m_required_joins {};
 
+	bool m_search_everything { false };
+
+	using SearchOperation = std::uint8_t;
+
+	enum SearchOperationFlags : SearchOperation
+	{
+		GreaterThan = 1 << 0, // >
+		LessThan = 1 << 1, // <
+		Equal = 1 << 2, // =
+		Not = 1 << 3, // !
+		// Approximate = 1 << 4, // ~
+		Approximate = Equal, // ~
+
+		// helpers
+		NotLessThan = Not | LessThan, // ~<
+		NotGreaterThan = Not | GreaterThan, // ~>
+
+		GreaterThanEqual = GreaterThan | Equal, // >=
+		LessThanEqual = LessThan | Equal, // <=
+		NotGreaterThanEqual = Not | GreaterThanEqual, // ~>=
+		NotLessThanEqual = Not | LessThanEqual, // ~<=
+
+		NotEqual = Not | Equal, // ~=
+	};
+
+	enum class DurationSearchType
+	{
+		DontCare = 0,
+		HasDuration,
+		NoDuration
+	} m_duration_search { DurationSearchType::DontCare };
+
+	enum class AudioSearchType
+	{
+		DontCare = 0,
+		HasAudio,
+		NoAudio
+	} m_audio_search { AudioSearchType::DontCare };
+
+	enum class ExitSearchType
+	{
+		DontCare = 0,
+		HasExif,
+		NoExif
+	} m_exif_search { ExitSearchType::DontCare };
+
+	enum class TagCountSearchType
+	{
+		DontCare = 0,
+		HasTags,
+		NoTags,
+		HasCount
+	} m_has_tags_search { TagCountSearchType::DontCare };
+
+	struct RangeSearchInfo
+	{
+		//! If true then this count and operation are put into effect
+		bool m_active { false };
+		std::size_t count { 0 };
+		SearchOperation operation { 0 };
+	};
+
+	static void parseRangeSearch( RangeSearchInfo& target, std::string_view tag );
+
+	RangeSearchInfo m_tag_count_search {};
+
+	RangeSearchInfo m_width_search {};
+	RangeSearchInfo m_height_search {};
+
+	RangeSearchInfo m_limit_search {};
+
 	SortType m_sort_type;
 	SortOrder m_order;
-	std::vector< TagID > m_tags;
+	std::vector< TagID > m_positive_tags;
+	std::vector< TagID > m_negative_tags {};
 
 	HydrusDisplayType m_display_mode;
 	bool m_bind_domains { false };
 
+	static std::unordered_map< TagID, std::string > createFilters(
+		const std::vector< TagID >& tag_ids,
+		bool filter_domains );
+	std::string buildPositiveFilter() const;
+	std::string buildNegativeFilter() const;
+
+	void generateOrderByClause( std::string& query ) const;
+	void determineJoinsForQuery( std::string& query );
+	void determineSelectClause( std::string& query, bool return_ids, bool return_hashes );
+	void generateWhereClauses( std::string& query );
 	/**
 	 * @brief Constructs a query to be used. $1 is expected to be an array of tag_domain_ids
 	 * @param return_ids
@@ -191,8 +281,12 @@ class SearchBuilder
 	void filterTagDomain( TagDomainID value );
 
 	void addFileDomain( FileDomainID value );
+	drogon::Task< std::expected< void, std::shared_ptr< drogon::HttpResponse > > > setTags(
+		const std::vector< std::string >& tags );
 
-	void setTags( const std::vector< TagID >& vector );
+	void setPositiveTags( const std::vector< TagID >& vector );
+	void setNegativeTags( const std::vector< TagID >& tag_ids );
+	void setSystemTags( const std::vector< std::string >& vector );
 
 	void setDisplay( HydrusDisplayType type );
 };

IDHANServer/src/crypto/SHA256.cpp

@@ -8,10 +8,10 @@
 #include <fstream>
 #include <istream>
 
+#include "../filesystem/io/IOUring.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "decodeHex.hpp"
 #include "fgl/defines.hpp"
-#include "filesystem/IOUring.hpp"
 
 namespace idhan
 {
@@ -29,6 +29,11 @@ SHA256::SHA256( const std::string_view& data ) : m_data()
 
 SHA256::SHA256( const drogon::orm::Field& field )
 {
+	if ( field.isNull() )
+	{
+		throw std::invalid_argument( "Field is null" );
+	}
+
 	const auto data { field.as< std::vector< char > >() };
 
 	FGL_ASSERT(
@@ -42,8 +47,7 @@ std::string SHA256::hex() const
 {
 	std::string str {};
 	str.reserve( m_data.size() );
-	for ( std::size_t i = 0; i < m_data.size(); ++i )
-		str += format_ns::format( "{:02x}", static_cast< std::uint8_t >( m_data[ i ] ) );
+	for ( auto i : m_data ) str += format_ns::format( "{:02x}", static_cast< std::uint8_t >( i ) );
 	return str;
 }
 
@@ -138,17 +142,29 @@ SHA256 SHA256::hash( const std::byte* data, const std::size_t size )
 
 drogon::Task< SHA256 > SHA256::hashCoro( FileIOUring io_uring )
 {
-	const auto data { co_await io_uring.readAll() };
+	QCryptographicHash hasher { QCryptographicHash::Sha256 };
 
-	if ( data.empty() )
+	constexpr auto block_size { 1024 * 1024 };
+
+	for ( std::size_t i = 0; i < io_uring.size(); i += block_size )
 	{
-		log::warn(
-			"While reading file {}, The filesystem said the file was zero bytes, or the read failed! A following "
-			"warning might occur!",
-			io_uring.path().string() );
+		const auto data { co_await io_uring.read( i, block_size ) };
+
+		const QByteArrayView view { data.data(), static_cast< qsizetype >( data.size() ) };
+
+		hasher.addData( view );
 	}
 
-	co_return hash( data.data(), data.size() );
+	const auto result { hasher.result() };
+
+	std::vector< std::byte > out_data {};
+	out_data.resize( 256 / 8 );
+
+	FGL_ASSERT( out_data.size() == static_cast< std::size_t >( result.size() ), "Invalid size" );
+
+	std::memcpy( out_data.data(), result.data(), static_cast< std::size_t >( result.size() ) );
+
+	co_return SHA256::fromBuffer( out_data );
 }
 
 } // namespace idhan

IDHANServer/src/crypto/SHA256.hpp

@@ -35,8 +35,6 @@ class SHA256
 {
 	std::array< std::byte, ( 256 / 8 ) > m_data {};
 
-	SHA256() = delete;
-
 	explicit SHA256( const std::byte* data );
 	SHA256( const std::string_view& data );
 
@@ -48,7 +46,7 @@ class SHA256
 
   public:
 
-	static constexpr std::size_t size() { return ( 256 / 8 ); }
+	SHA256() = default;
 
 	SHA256( const drogon::orm::Field& field );
 
@@ -58,6 +56,8 @@ class SHA256
 	SHA256& operator=( SHA256&& other ) = default;
 	SHA256( SHA256&& other ) = default;
 
+	static constexpr std::size_t size() { return ( 256 / 8 ); }
+
 	std::array< std::byte, ( 256 / 8 ) > data() const { return m_data; }
 
 	//! Supplied so we can work with drogon until I figure out how the fuck to overload their operators.

IDHANServer/src/db/TagSearch.cpp

@@ -5,7 +5,7 @@
 
 #include <expected>
 
-#include "api/helpers/ExpectedTask.hpp"
+#include "threading/ExpectedTask.hpp"
 #include "api/helpers/createBadRequest.hpp"
 #include "drogon/HttpAppFramework.h"
 #include "logging/log.hpp"

IDHANServer/src/db/TagSearch.hpp

@@ -5,8 +5,8 @@
 #include <expected>
 #include <vector>
 
+#include "threading/ExpectedTask.hpp"
 #include "IDHANTypes.hpp"
-#include "api/helpers/ExpectedTask.hpp"
 #include "dbTypes.hpp"
 #include "drogon/HttpAppFramework.h"
 #include "drogon/orm/BaseBuilder.h"

IDHANServer/src/db/drogonArrayBind.cpp

@@ -159,6 +159,43 @@ std::vector< std::byte > createPgBinaryArray( std::vector< std::string >&& strin
 	return result;
 }
 
+std::vector< std::byte > createPgBinaryArray( const std::vector< std::string >&& strings )
+{
+	std::vector< std::byte > result {};
+
+	std::size_t string_sizes { 0 };
+	for ( const auto& str : strings ) string_sizes += str.size();
+
+	struct [[gnu::packed]] Element
+	{
+		std::uint32_t element_length { 0 };
+	};
+
+	static_assert( sizeof( Element ) == sizeof( std::uint32_t ), "Element is not sized properly" );
+
+	result.resize( sizeof( Header ) + ( sizeof( Element ) * strings.size() ) + string_sizes );
+	std::memset( result.data(), 0xFF, result.size() );
+
+	auto* header = reinterpret_cast< Header* >( result.data() );
+	header->num_dimensions = htonl( 1 ); // dimension count
+	header->data_offset = htonl( 0 ); // any nulls?
+	header->element_type_oid = htonl( OID_TEXT ); // element type
+	header->dimension_length = htonl( static_cast< uint32_t >( strings.size() ) ); // size of first dimension
+	header->lower_bound = htonl( 1 ); // offset of first dimension
+
+	std::byte* ptr = result.data() + sizeof( Header );
+	for ( const auto& str : strings )
+	{
+		auto& element = *reinterpret_cast< Element* >( ptr );
+		// const auto filtered_string { idhan::api::helpers::pgEscape( str ) };
+		element.element_length = htonl( static_cast< std::uint32_t >( str.size() ) );
+		std::memcpy( ptr + sizeof( Element ), str.data(), str.size() );
+		ptr += sizeof( Element ) + str.size();
+	}
+
+	return result;
+}
+
 std::vector< std::byte > createPgBinaryArray( std::set< std::string >&& strings )
 {
 	std::vector< std::byte > result {};

IDHANServer/src/db/pgEscape.cpp

@@ -7,7 +7,7 @@
 #include "fgl/defines.hpp"
 #include "logging/format_ns.hpp"
 
-namespace idhan::api::helpers
+namespace idhan::helpers
 {
 
 constexpr std::string pgEscapeI( const std::string& str )
@@ -57,4 +57,4 @@ FGL_FLATTEN std::string pgEscape( const std::string& str )
 	return pgEscapeI( str );
 }
 
-} // namespace idhan::api::helpers
+} // namespace idhan::helpers

IDHANServer/src/db/pgEscape.hpp

@@ -5,7 +5,7 @@
 
 #include <string>
 
-namespace idhan::api::helpers
+namespace idhan::helpers
 {
 std::string pgEscape( const std::string& s );
 }

IDHANServer/src/docs/importing/remote.puml

@@ -1,63 +0,0 @@
-@startuml
-'https://plantuml.com/sequence-diagram
-
-autonumber
-
-
-participant Client
-
-!pragma teoz true
-box "Server" #LightBlue
-participant Server
-box "Internal"
-queue ImporterQueue as Queue
-participant Importer...N as Importer
-participant MimeDatabase
-end box
-
-end box
-
-activate Client
-activate Server
-
-Client <-> Server: Auth
-
-Client -> Server: Submit file
-Server -> Server: Generate import UID
-
-Server -> Server: Wait for import queue to have space available
-Server -> Queue: Place file onto queue
-
-Server -> Client: Return UID
-
-deactivate Client
-
-Queue <-> Importer: Pull next import task
-Importer -> MimeDatabase: Request mime parse
-
-MimeDatabase -> Importer: Return mime info
-
-alt#Gold #LightBlue Mime detected
-Importer -> Importer: Continue import
-else #Pink Mime Unknown
-Importer -> Server: Invalidate UID
-end
-
-==  ==
-
-autonumber
-
-Client -> Server: Request import info
-alt#Gold #LightBlue Import success
-Server -> Client: Returns
-else #Pink UID Invalid/Error
-Server -> Client: Returns error
-else #Grey Import not completed
-Server -> Server: Wait for import to completed
-Server -> Client:
-end
-
-
-destroy Client
-
-@enduml