mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
fix(books): support accent-insensitive search
Install the PostgreSQL unaccent extension and use a custom immutable wrapper function to perform accent-insensitive ILIKE comparisons when searching for books, authors, series, and narrators. Add database indexes on the unaccented values to maintain query performance. Closes #640
This commit is contained in:
@@ -252,7 +252,12 @@ function outcomeClass(row: HardcoverImportPreviewRow): string {
|
||||
}
|
||||
|
||||
function normalizeSearch(value: string): string {
|
||||
return value.toLowerCase().replace(/\s+/g, ' ').trim()
|
||||
return value
|
||||
.normalize('NFD')
|
||||
.replace(/\p{Diacritic}/gu, '')
|
||||
.toLowerCase()
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
@@ -139,6 +139,15 @@ describe('HardcoverImportReviewModal', () => {
|
||||
expect(wrapper.text()).toContain('No rows match the current filters.')
|
||||
})
|
||||
|
||||
it('matches accented book metadata with an unaccented search term', async () => {
|
||||
const wrapper = mountModal(makePreview([makeRow({ localTitle: 'El Criticón', localAuthors: ['Baltasar Gracián'] })]))
|
||||
|
||||
await wrapper.get('input[type="search"]').setValue('gracian')
|
||||
|
||||
expect(wrapper.text()).toContain('El Criticón')
|
||||
expect(wrapper.text()).not.toContain('No rows match the current filters.')
|
||||
})
|
||||
|
||||
it('supports bulk selection controls across the current page', async () => {
|
||||
const wrapper = mountModal(
|
||||
makePreview([
|
||||
|
||||
@@ -4,4 +4,5 @@
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_trgm"; -- for fuzzy text search
|
||||
CREATE EXTENSION IF NOT EXISTS "unaccent"; -- for accent-insensitive search
|
||||
CREATE EXTENSION IF NOT EXISTS "vector"; -- for pgvector ANN search
|
||||
|
||||
+1
-1
@@ -130,7 +130,7 @@ docker compose -f docker-compose.dev.yml up -d --wait
|
||||
|
||||
This starts a PostgreSQL 16 container (with the `pgvector` extension) on port 5432. The `--wait` flag blocks until the database is healthy. Data is persisted in a named Docker volume across restarts.
|
||||
|
||||
> **Already have PostgreSQL running?** You can point `DATABASE_URL` in `server/.env` at your existing instance, but you will still need the `uuid-ossp`, `pg_trgm`, and `vector` extensions installed. Using Docker is the easiest path.
|
||||
> **Already have PostgreSQL running?** You can point `DATABASE_URL` in `server/.env` at your existing instance, but you will still need the `uuid-ossp`, `pg_trgm`, `unaccent`, and `vector` extensions installed. Using Docker is the easiest path.
|
||||
|
||||
#### Step 5 - Apply database migrations
|
||||
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@
|
||||
"start:debug": "nest start --debug --watch --config nest-cli.dev.json",
|
||||
"start:prod": "node dist/main",
|
||||
"db:generate": "drizzle-kit generate --name",
|
||||
"db:migrate": "drizzle-kit migrate",
|
||||
"db:migrate": "tsx --env-file-if-exists=.env src/scripts/install-postgres-extensions.ts && drizzle-kit migrate",
|
||||
"e2e:db:migrate": "DATABASE_URL=postgres://bookorbit:bookorbit@localhost:5432/bookorbit_e2e pnpm db:migrate",
|
||||
"db:migrate:runtime": "node dist/scripts/migrate.js",
|
||||
"e2e:db:prepare": "tsx src/scripts/db-prepare-e2e.ts",
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
const SEARCH_ENTRY_POINTS = [
|
||||
'modules/annotation/annotation.repository.ts',
|
||||
'modules/authors/authors.repository.ts',
|
||||
'modules/book-dock/book-dock.repository.ts',
|
||||
'modules/book/book-query-builder.service.ts',
|
||||
'modules/book/book.repository.ts',
|
||||
'modules/catalog/catalog.service.ts',
|
||||
'modules/custom-icon/custom-icon.repository.ts',
|
||||
'modules/entity-manager/strategies/author.strategy.ts',
|
||||
'modules/entity-manager/strategies/inline-entity.strategy.ts',
|
||||
'modules/entity-manager/strategies/junction-entity.strategy.ts',
|
||||
'modules/entity-manager/strategies/series.strategy.ts',
|
||||
'modules/migration/planner/matching.service.ts',
|
||||
'modules/opds/opds-book.service.ts',
|
||||
'modules/series/series.repository.ts',
|
||||
] as const;
|
||||
|
||||
describe('accent-insensitive search entry points', () => {
|
||||
it.each(SEARCH_ENTRY_POINTS)('%s uses the shared accent-insensitive predicate', (relativePath) => {
|
||||
const source = readFileSync(join(__dirname, '..', '..', relativePath), 'utf8');
|
||||
|
||||
expect(source).toContain('accentInsensitiveIlike');
|
||||
expect(source).not.toMatch(/\bilike\s*\(/i);
|
||||
expect(source).not.toMatch(/\bILIKE\b/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
import { PgDialect, pgTable, text } from 'drizzle-orm/pg-core';
|
||||
|
||||
import { accentInsensitiveIlike } from './accent-insensitive-search.utils';
|
||||
|
||||
const records = pgTable('records', {
|
||||
name: text('name'),
|
||||
});
|
||||
|
||||
describe('accentInsensitiveIlike', () => {
|
||||
it('applies unaccent to both the stored value and the search pattern', () => {
|
||||
const dialect = new PgDialect();
|
||||
|
||||
const query = dialect.sqlToQuery(accentInsensitiveIlike(records.name, '%gracian%'));
|
||||
|
||||
expect(query.sql).toBe('public.bookorbit_unaccent("records"."name") ILIKE public.bookorbit_unaccent($1)');
|
||||
expect(query.params).toEqual(['%gracian%']);
|
||||
});
|
||||
|
||||
it('keeps wildcard escaping parameterized', () => {
|
||||
const dialect = new PgDialect();
|
||||
|
||||
const query = dialect.sqlToQuery(accentInsensitiveIlike(records.name, '%100\\%\\_%'));
|
||||
|
||||
expect(query.params).toEqual(['%100\\%\\_%']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
import { SQL, SQLWrapper, sql } from 'drizzle-orm';
|
||||
|
||||
export function accentInsensitiveIlike(value: SQLWrapper, pattern: string): SQL {
|
||||
return sql`public.bookorbit_unaccent(${value}) ILIKE public.bookorbit_unaccent(${pattern})`;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE INDEX "book_series_name_unaccent_trgm_idx" ON "book_series" USING gin (public.bookorbit_unaccent("name") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "authors_name_unaccent_trgm_idx" ON "authors" USING gin (public.bookorbit_unaccent("name") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "bm_title_unaccent_trgm_idx" ON "book_metadata" USING gin (public.bookorbit_unaccent("title") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "bm_series_unaccent_trgm_idx" ON "book_metadata" USING gin (public.bookorbit_unaccent("series_name") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "bm_publisher_unaccent_trgm_idx" ON "book_metadata" USING gin (public.bookorbit_unaccent("publisher") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "bm_language_unaccent_trgm_idx" ON "book_metadata" USING gin (public.bookorbit_unaccent("language") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "genres_name_unaccent_trgm_idx" ON "genres" USING gin (public.bookorbit_unaccent("name") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "tags_name_unaccent_trgm_idx" ON "tags" USING gin (public.bookorbit_unaccent("name") gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "narrators_name_unaccent_trgm_idx" ON "narrators" USING gin (public.bookorbit_unaccent("name") gin_trgm_ops);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -330,6 +330,13 @@
|
||||
"when": 1783630311448,
|
||||
"tag": "0046_kobo-device-snapshots",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 47,
|
||||
"version": "7",
|
||||
"when": 1783791670862,
|
||||
"tag": "0047_accent-insensitive-search-indexes",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -85,15 +85,19 @@ export const bookMetadata = pgTable(
|
||||
},
|
||||
(t) => [
|
||||
index('bm_title_trgm_idx').using('gin', t.title.op('gin_trgm_ops')),
|
||||
index('bm_title_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.title}) gin_trgm_ops`),
|
||||
index('bm_title_lower_idx').on(sql`lower(${t.title})`),
|
||||
index('bm_title_book_id_idx').on(t.title, t.bookId),
|
||||
index('bm_series_trgm_idx').using('gin', t.seriesName.op('gin_trgm_ops')),
|
||||
index('bm_series_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.seriesName}) gin_trgm_ops`),
|
||||
index('bm_series_id_idx').on(t.seriesId),
|
||||
index('bm_series_id_index_book_id_idx').on(t.seriesId, t.seriesIndex, t.bookId),
|
||||
index('bm_series_name_lower_btrim_idx').on(sql`lower(btrim(${t.seriesName}))`),
|
||||
index('bm_publisher_trgm_idx').using('gin', t.publisher.op('gin_trgm_ops')),
|
||||
index('bm_publisher_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.publisher}) gin_trgm_ops`),
|
||||
index('bm_language_idx').on(t.language),
|
||||
index('bm_language_trgm_idx').using('gin', t.language.op('gin_trgm_ops')),
|
||||
index('bm_language_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.language}) gin_trgm_ops`),
|
||||
index('bm_published_date_idx').on(t.publishedDate),
|
||||
index('bm_published_year_idx').on(t.publishedYear),
|
||||
index('bm_published_date_sort_idx').on(sql`coalesce(${t.publishedDate}, make_date(${t.publishedYear}, 1, 1))`),
|
||||
@@ -147,7 +151,11 @@ export const authors = pgTable(
|
||||
hasPhoto: boolean('has_photo').notNull().default(false),
|
||||
lastEnrichedAt: timestamp('last_enriched_at', { withTimezone: true }),
|
||||
},
|
||||
(t) => [unique('authors_name_unique').on(t.name), index('authors_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops'))],
|
||||
(t) => [
|
||||
unique('authors_name_unique').on(t.name),
|
||||
index('authors_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops')),
|
||||
index('authors_name_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.name}) gin_trgm_ops`),
|
||||
],
|
||||
);
|
||||
|
||||
export const bookAuthors = pgTable(
|
||||
@@ -210,7 +218,10 @@ export const genres = pgTable(
|
||||
id: serial('id').primaryKey(),
|
||||
name: varchar('name', { length: 200 }).notNull().unique(),
|
||||
},
|
||||
(t) => [index('genres_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops'))],
|
||||
(t) => [
|
||||
index('genres_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops')),
|
||||
index('genres_name_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.name}) gin_trgm_ops`),
|
||||
],
|
||||
);
|
||||
|
||||
export const bookGenres = pgTable(
|
||||
@@ -232,7 +243,10 @@ export const tags = pgTable(
|
||||
id: serial('id').primaryKey(),
|
||||
name: varchar('name', { length: 200 }).notNull().unique(),
|
||||
},
|
||||
(t) => [index('tags_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops'))],
|
||||
(t) => [
|
||||
index('tags_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops')),
|
||||
index('tags_name_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.name}) gin_trgm_ops`),
|
||||
],
|
||||
);
|
||||
|
||||
export const bookTags = pgTable(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { index, integer, pgTable, primaryKey, serial, timestamp, varchar } from 'drizzle-orm/pg-core';
|
||||
|
||||
import { books } from './books';
|
||||
@@ -10,7 +11,10 @@ export const narrators = pgTable(
|
||||
sortName: varchar('sort_name', { length: 500 }),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
|
||||
},
|
||||
(t) => [index('narrators_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops'))],
|
||||
(t) => [
|
||||
index('narrators_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops')),
|
||||
index('narrators_name_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.name}) gin_trgm_ops`),
|
||||
],
|
||||
);
|
||||
|
||||
export const bookNarrators = pgTable(
|
||||
|
||||
@@ -18,6 +18,7 @@ export const bookSeries = pgTable(
|
||||
(t) => [
|
||||
uniqueIndex('book_series_normalized_name_uidx').on(t.normalizedName),
|
||||
index('book_series_name_trgm_idx').using('gin', t.name.op('gin_trgm_ops')),
|
||||
index('book_series_name_unaccent_trgm_idx').using('gin', sql`public.bookorbit_unaccent(${t.name}) gin_trgm_ops`),
|
||||
index('book_series_name_lower_idx').on(sql`lower(${t.name})`),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
eq,
|
||||
getTableColumns,
|
||||
gte,
|
||||
ilike,
|
||||
isNotNull,
|
||||
isNull,
|
||||
lte,
|
||||
@@ -22,6 +21,7 @@ import {
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../db/schema';
|
||||
import {
|
||||
annotationPositions,
|
||||
@@ -398,8 +398,8 @@ export class AnnotationRepository {
|
||||
const pattern = `%${term}%`;
|
||||
conditions.push(
|
||||
or(
|
||||
ilike(bookMetadata.title, pattern),
|
||||
sql`exists (select 1 from ${bookAuthors} inner join ${authors} on ${authors.id} = ${bookAuthors.authorId} where ${bookAuthors.bookId} = ${annotations.bookId} and ${authors.name} ilike ${pattern})`,
|
||||
accentInsensitiveIlike(bookMetadata.title, pattern),
|
||||
sql`exists (select 1 from ${bookAuthors} inner join ${authors} on ${authors.id} = ${bookAuthors.authorId} where ${bookAuthors.bookId} = ${annotations.bookId} and ${accentInsensitiveIlike(authors.name, pattern)})`,
|
||||
)!,
|
||||
);
|
||||
}
|
||||
@@ -476,7 +476,7 @@ export class AnnotationRepository {
|
||||
if (filters.chapter) conditions.push(eq(annotations.chapterTitle, filters.chapter));
|
||||
if (filters.search) {
|
||||
const pattern = `%${filters.search}%`;
|
||||
conditions.push(or(ilike(annotations.text, pattern), ilike(annotations.note, pattern))!);
|
||||
conditions.push(or(accentInsensitiveIlike(annotations.text, pattern), accentInsensitiveIlike(annotations.note, pattern))!);
|
||||
}
|
||||
if (filters.hasNote) conditions.push(sql`${annotations.note} is not null and ${annotations.note} <> ''`);
|
||||
if (filters.dateFrom) conditions.push(gte(annotations.createdAt, filters.dateFrom));
|
||||
@@ -504,7 +504,7 @@ export class AnnotationRepository {
|
||||
}
|
||||
if (filters.search) {
|
||||
const pattern = `%${filters.search}%`;
|
||||
conditions.push(or(ilike(annotations.text, pattern), ilike(annotations.note, pattern))!);
|
||||
conditions.push(or(accentInsensitiveIlike(annotations.text, pattern), accentInsensitiveIlike(annotations.note, pattern))!);
|
||||
}
|
||||
if (filters.chapter) {
|
||||
conditions.push(eq(annotations.chapterTitle, filters.chapter));
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { SQL, and, asc, desc, eq, ilike, inArray, isNull, max, or, sql } from 'drizzle-orm';
|
||||
import { SQL, and, asc, desc, eq, inArray, isNull, max, or, sql } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import type { ContentFilterRules } from '@bookorbit/types';
|
||||
import { buildContentFilterClauses } from '../../common/utils/content-filter-sql.utils';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import { DB } from '../../db';
|
||||
import { refreshPrimaryAuthorSortNamesForAuthors, refreshPrimaryAuthorSortNamesForBooks } from '../../db/book-author-sort-key';
|
||||
import * as schema from '../../db/schema';
|
||||
@@ -318,7 +319,7 @@ export class AuthorsRepository {
|
||||
}
|
||||
const query = params.q?.trim();
|
||||
if (query) {
|
||||
clauses.push(ilike(authors.name, `%${escapeLikePattern(query)}%`));
|
||||
clauses.push(accentInsensitiveIlike(authors.name, `%${escapeLikePattern(query)}%`));
|
||||
}
|
||||
if (params.hasPhoto !== undefined) {
|
||||
clauses.push(eq(authors.hasPhoto, params.hasPhoto));
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { and, asc, count, desc, eq, gt, ilike, inArray, isNull, notInArray, or, sum, type SQL } from 'drizzle-orm';
|
||||
import { and, asc, count, desc, eq, gt, inArray, isNull, notInArray, or, sum, type SQL } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../db/schema';
|
||||
import { bookDockFiles, type NewBookDockFileRow, type BookDockFileRow } from '../../db/schema';
|
||||
|
||||
@@ -197,7 +198,7 @@ export class BookDockRepository {
|
||||
} else if (status) {
|
||||
conditions.push(eq(bookDockFiles.status, status));
|
||||
}
|
||||
if (search) conditions.push(ilike(bookDockFiles.fileName, `%${search}%`));
|
||||
if (search) conditions.push(accentInsensitiveIlike(bookDockFiles.fileName, `%${search}%`));
|
||||
if (userId !== undefined && !isSuperuser) {
|
||||
conditions.push(or(eq(bookDockFiles.uploadedBy, userId), isNull(bookDockFiles.uploadedBy))!);
|
||||
}
|
||||
|
||||
@@ -26,11 +26,16 @@ vi.mock('drizzle-orm', () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../common/utils/accent-insensitive-search.utils', () => ({
|
||||
accentInsensitiveIlike: vi.fn((left: unknown, pattern: string) => ({ type: 'accentInsensitiveIlike', left, pattern })),
|
||||
}));
|
||||
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { ilike, sql } from 'drizzle-orm';
|
||||
import { sql } from 'drizzle-orm';
|
||||
|
||||
import { EMPTY_CONTENT_FILTER_RULES, FIELD_OPERATORS, type RuleField, type RuleOperator } from '@bookorbit/types';
|
||||
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import { BookQueryBuilder } from './book-query-builder.service';
|
||||
import { BookSortBuilder } from './book-sort-builder.service';
|
||||
|
||||
@@ -173,7 +178,7 @@ describe('BookQueryBuilder', () => {
|
||||
expect(where).toMatchObject({ type: 'and' });
|
||||
expect(where.clauses).toHaveLength(3);
|
||||
expect(where.clauses[2]).toMatchObject({ type: 'and' });
|
||||
expect(where.clauses[2].clauses[0]).toMatchObject({ type: 'ilike', pattern: '%Dune%' });
|
||||
expect(where.clauses[2].clauses[0]).toMatchObject({ type: 'accentInsensitiveIlike', pattern: '%Dune%' });
|
||||
});
|
||||
|
||||
it('applies content filter clauses when filters are active', () => {
|
||||
@@ -381,7 +386,7 @@ describe('BookQueryBuilder', () => {
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('builds one author subquery per includesAll value and uses ilike patterns', () => {
|
||||
it('builds one accent-insensitive author subquery per includesAll value', () => {
|
||||
const { builder, db } = makeBuilder();
|
||||
|
||||
builder.buildWhere(wrapRule({ type: 'rule', field: 'author', operator: 'includesAll', value: ['Frank', 'Herbert'] }) as never, {
|
||||
@@ -390,8 +395,8 @@ describe('BookQueryBuilder', () => {
|
||||
});
|
||||
|
||||
expect(db.select).toHaveBeenCalledTimes(2);
|
||||
expect(ilike).toHaveBeenCalledWith(expect.anything(), '%Frank%');
|
||||
expect(ilike).toHaveBeenCalledWith(expect.anything(), '%Herbert%');
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(expect.anything(), '%Frank%');
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(expect.anything(), '%Herbert%');
|
||||
});
|
||||
|
||||
it('handles empty includesAny sets by generating an always-false branch', () => {
|
||||
@@ -602,16 +607,26 @@ describe('field coverage smoke tests', () => {
|
||||
});
|
||||
|
||||
describe('buildQuickSearch', () => {
|
||||
it('produces an OR of title/series ilike and author/series/narrator exists subqueries', () => {
|
||||
it('passes an unaccented author query through every accent-insensitive text predicate', () => {
|
||||
const { builder } = makeBuilder();
|
||||
vi.mocked(accentInsensitiveIlike).mockClear();
|
||||
|
||||
builder.buildQuickSearch('gracian');
|
||||
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledTimes(5);
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(expect.anything(), '%gracian%');
|
||||
});
|
||||
|
||||
it('produces an OR of accent-insensitive title/series matches and author/series/narrator exists subqueries', () => {
|
||||
const { builder } = makeBuilder();
|
||||
|
||||
const result = builder.buildQuickSearch('tolkien') as any;
|
||||
|
||||
expect(result).toMatchObject({ type: 'or' });
|
||||
expect(result.clauses).toHaveLength(5);
|
||||
expect(result.clauses[0]).toMatchObject({ type: 'ilike', pattern: '%tolkien%' });
|
||||
expect(result.clauses[0]).toMatchObject({ type: 'accentInsensitiveIlike', pattern: '%tolkien%' });
|
||||
expect(result.clauses[1]).toMatchObject({ type: 'sql' });
|
||||
expect(result.clauses[2]).toMatchObject({ type: 'ilike', pattern: '%tolkien%' });
|
||||
expect(result.clauses[2]).toMatchObject({ type: 'accentInsensitiveIlike', pattern: '%tolkien%' });
|
||||
expect(result.clauses[3]).toMatchObject({ type: 'sql' });
|
||||
expect(result.clauses[4]).toMatchObject({ type: 'sql' });
|
||||
});
|
||||
@@ -621,8 +636,8 @@ describe('buildQuickSearch', () => {
|
||||
|
||||
const result = builder.buildQuickSearch('50% off') as any;
|
||||
|
||||
expect(result.clauses[0]).toMatchObject({ type: 'ilike', pattern: '%50\\% off%' });
|
||||
expect(result.clauses[2]).toMatchObject({ type: 'ilike', pattern: '%50\\% off%' });
|
||||
expect(result.clauses[0]).toMatchObject({ type: 'accentInsensitiveIlike', pattern: '%50\\% off%' });
|
||||
expect(result.clauses[2]).toMatchObject({ type: 'accentInsensitiveIlike', pattern: '%50\\% off%' });
|
||||
});
|
||||
|
||||
it('escapes underscore in q', () => {
|
||||
@@ -630,7 +645,7 @@ describe('buildQuickSearch', () => {
|
||||
|
||||
const result = builder.buildQuickSearch('book_one') as any;
|
||||
|
||||
expect(result.clauses[0]).toMatchObject({ type: 'ilike', pattern: '%book\\_one%' });
|
||||
expect(result.clauses[0]).toMatchObject({ type: 'accentInsensitiveIlike', pattern: '%book\\_one%' });
|
||||
});
|
||||
|
||||
it('calls db.select three times for author, series membership, and narrator exists subqueries', () => {
|
||||
@@ -692,11 +707,11 @@ describe('buildWhere with q', () => {
|
||||
|
||||
describe('textRuleToSql (via title)', () => {
|
||||
it.each([
|
||||
['contains', 'Dune', { type: 'ilike', pattern: '%Dune%' }],
|
||||
['contains', 'Dune', { type: 'accentInsensitiveIlike', pattern: '%Dune%' }],
|
||||
['notContains', 'Dune', { type: 'or' }],
|
||||
['startsWith', 'Dune', { type: 'ilike', pattern: 'Dune%' }],
|
||||
['endsWith', 'Dune', { type: 'ilike', pattern: '%Dune' }],
|
||||
['eq', 'Dune', { type: 'ilike', pattern: 'Dune' }],
|
||||
['startsWith', 'Dune', { type: 'accentInsensitiveIlike', pattern: 'Dune%' }],
|
||||
['endsWith', 'Dune', { type: 'accentInsensitiveIlike', pattern: '%Dune' }],
|
||||
['eq', 'Dune', { type: 'accentInsensitiveIlike', pattern: 'Dune' }],
|
||||
['notEq', 'Dune', { type: 'or' }],
|
||||
['isEmpty', undefined, { type: 'or' }],
|
||||
['isNotEmpty', undefined, { type: 'and' }],
|
||||
@@ -709,22 +724,22 @@ describe('textRuleToSql (via title)', () => {
|
||||
expect(getRuleSql(where)).toMatchObject(expected);
|
||||
});
|
||||
|
||||
it('notContains wraps an ilike in or(isNull, not())', () => {
|
||||
it('notContains wraps an accent-insensitive match in or(isNull, not())', () => {
|
||||
const { builder } = makeBuilder();
|
||||
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'title', operator: 'notContains', value: 'Dune' }) as never, BASE_CTX) as any;
|
||||
const clause = getRuleSql(where);
|
||||
expect(clause).toMatchObject({ type: 'or' });
|
||||
expect(clause.clauses[0]).toMatchObject({ type: 'isNull' });
|
||||
expect(clause.clauses[1]).toMatchObject({ type: 'not', value: { type: 'ilike', pattern: '%Dune%' } });
|
||||
expect(clause.clauses[1]).toMatchObject({ type: 'not', value: { type: 'accentInsensitiveIlike', pattern: '%Dune%' } });
|
||||
});
|
||||
|
||||
it('notEq wraps an exact ilike in or(isNull, not())', () => {
|
||||
it('notEq wraps an exact accent-insensitive match in or(isNull, not())', () => {
|
||||
const { builder } = makeBuilder();
|
||||
const where = builder.buildWhere(wrapRule({ type: 'rule', field: 'title', operator: 'notEq', value: 'Dune' }) as never, BASE_CTX) as any;
|
||||
const clause = getRuleSql(where);
|
||||
expect(clause).toMatchObject({ type: 'or' });
|
||||
expect(clause.clauses[0]).toMatchObject({ type: 'isNull' });
|
||||
expect(clause.clauses[1]).toMatchObject({ type: 'not', value: { type: 'ilike', pattern: 'Dune' } });
|
||||
expect(clause.clauses[1]).toMatchObject({ type: 'not', value: { type: 'accentInsensitiveIlike', pattern: 'Dune' } });
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { BadRequestException, Inject, Injectable } from '@nestjs/common';
|
||||
import { AnyColumn, SQL, and, eq, gt, gte, ilike, inArray, isNotNull, isNull, lt, lte, ne, not, or, sql } from 'drizzle-orm';
|
||||
import { AnyColumn, SQL, and, eq, gt, gte, inArray, isNotNull, isNull, lt, lte, ne, not, or, sql } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import type { CommunityRatingProvider, ContentFilterRules, GroupRule, ReadStatus, Rule, SortSpec } from '@bookorbit/types';
|
||||
import { DB } from '../../db';
|
||||
import { isDateKey, resolveTimeZone, toDateKeyInTimeZone } from '../../common/utils/timezone.utils';
|
||||
import { buildContentFilterClauses } from '../../common/utils/content-filter-sql.utils';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../db/schema';
|
||||
import { BookSortBuilder } from './book-sort-builder.service';
|
||||
import {
|
||||
@@ -86,7 +87,7 @@ export class BookQueryBuilder {
|
||||
.select({ one: sql`1` })
|
||||
.from(bookAuthors)
|
||||
.innerJoin(authors, eq(bookAuthors.authorId, authors.id))
|
||||
.where(and(eq(bookAuthors.bookId, books.id), ilike(authors.name, pattern))!);
|
||||
.where(and(eq(bookAuthors.bookId, books.id), accentInsensitiveIlike(authors.name, pattern))!);
|
||||
return sql`exists (${sq})`;
|
||||
})();
|
||||
|
||||
@@ -95,7 +96,7 @@ export class BookQueryBuilder {
|
||||
.select({ one: sql`1` })
|
||||
.from(bookNarrators)
|
||||
.innerJoin(narrators, eq(bookNarrators.narratorId, narrators.id))
|
||||
.where(and(eq(bookNarrators.bookId, books.id), ilike(narrators.name, pattern))!);
|
||||
.where(and(eq(bookNarrators.bookId, books.id), accentInsensitiveIlike(narrators.name, pattern))!);
|
||||
return sql`exists (${sq})`;
|
||||
})();
|
||||
|
||||
@@ -104,11 +105,17 @@ export class BookQueryBuilder {
|
||||
.select({ one: sql`1` })
|
||||
.from(bookSeriesMemberships)
|
||||
.innerJoin(bookSeries, eq(bookSeries.id, bookSeriesMemberships.seriesId))
|
||||
.where(and(eq(bookSeriesMemberships.bookId, books.id), ilike(bookSeries.name, pattern))!);
|
||||
.where(and(eq(bookSeriesMemberships.bookId, books.id), accentInsensitiveIlike(bookSeries.name, pattern))!);
|
||||
return sql`exists (${sq})`;
|
||||
})();
|
||||
|
||||
return or(ilike(bookMetadata.title, pattern), existsAuthor, ilike(bookMetadata.seriesName, pattern), existsSeries, existsNarrator)!;
|
||||
return or(
|
||||
accentInsensitiveIlike(bookMetadata.title, pattern),
|
||||
existsAuthor,
|
||||
accentInsensitiveIlike(bookMetadata.seriesName, pattern),
|
||||
existsSeries,
|
||||
existsNarrator,
|
||||
)!;
|
||||
}
|
||||
|
||||
buildOrderBy(sort: SortSpec[], userId?: number): SQL[] {
|
||||
@@ -198,17 +205,17 @@ export class BookQueryBuilder {
|
||||
}
|
||||
switch (operator) {
|
||||
case 'contains':
|
||||
return ilike(col, `%${escapeLikePattern(value!)}%`);
|
||||
return accentInsensitiveIlike(col, `%${escapeLikePattern(value!)}%`);
|
||||
case 'notContains':
|
||||
return or(isNull(col), not(ilike(col, `%${escapeLikePattern(value!)}%`)))!;
|
||||
return or(isNull(col), not(accentInsensitiveIlike(col, `%${escapeLikePattern(value!)}%`)))!;
|
||||
case 'startsWith':
|
||||
return ilike(col, `${escapeLikePattern(value!)}%`);
|
||||
return accentInsensitiveIlike(col, `${escapeLikePattern(value!)}%`);
|
||||
case 'endsWith':
|
||||
return ilike(col, `%${escapeLikePattern(value!)}`);
|
||||
return accentInsensitiveIlike(col, `%${escapeLikePattern(value!)}`);
|
||||
case 'eq':
|
||||
return ilike(col, escapeLikePattern(value!));
|
||||
return accentInsensitiveIlike(col, escapeLikePattern(value!));
|
||||
case 'notEq':
|
||||
return or(isNull(col), not(ilike(col, escapeLikePattern(value!))))!;
|
||||
return or(isNull(col), not(accentInsensitiveIlike(col, escapeLikePattern(value!))))!;
|
||||
case 'isEmpty':
|
||||
return or(isNull(col), eq(col, ''))!;
|
||||
case 'isNotEmpty':
|
||||
@@ -296,17 +303,17 @@ export class BookQueryBuilder {
|
||||
|
||||
switch (operator) {
|
||||
case 'contains':
|
||||
return existsSeries(ilike(bookSeries.name, `%${escapeLikePattern(value!)}%`));
|
||||
return existsSeries(accentInsensitiveIlike(bookSeries.name, `%${escapeLikePattern(value!)}%`));
|
||||
case 'notContains':
|
||||
return not(existsSeries(ilike(bookSeries.name, `%${escapeLikePattern(value!)}%`)));
|
||||
return not(existsSeries(accentInsensitiveIlike(bookSeries.name, `%${escapeLikePattern(value!)}%`)));
|
||||
case 'startsWith':
|
||||
return existsSeries(ilike(bookSeries.name, `${escapeLikePattern(value!)}%`));
|
||||
return existsSeries(accentInsensitiveIlike(bookSeries.name, `${escapeLikePattern(value!)}%`));
|
||||
case 'endsWith':
|
||||
return existsSeries(ilike(bookSeries.name, `%${escapeLikePattern(value!)}`));
|
||||
return existsSeries(accentInsensitiveIlike(bookSeries.name, `%${escapeLikePattern(value!)}`));
|
||||
case 'eq':
|
||||
return existsSeries(ilike(bookSeries.name, escapeLikePattern(value!)));
|
||||
return existsSeries(accentInsensitiveIlike(bookSeries.name, escapeLikePattern(value!)));
|
||||
case 'notEq':
|
||||
return not(existsSeries(ilike(bookSeries.name, escapeLikePattern(value!))));
|
||||
return not(existsSeries(accentInsensitiveIlike(bookSeries.name, escapeLikePattern(value!))));
|
||||
case 'isEmpty':
|
||||
return not(existsSeries());
|
||||
case 'isNotEmpty':
|
||||
@@ -460,13 +467,13 @@ export class BookQueryBuilder {
|
||||
switch (operator) {
|
||||
case 'includesAny':
|
||||
if (!values?.length) return sql`1 = 0`;
|
||||
return existsAuthor(or(...values.map((v) => ilike(authors.name, `%${v}%`)))!);
|
||||
return existsAuthor(or(...values.map((v) => accentInsensitiveIlike(authors.name, `%${v}%`)))!);
|
||||
case 'includesAll':
|
||||
if (!values?.length) return sql`1 = 0`;
|
||||
return and(...values.map((v) => existsAuthor(ilike(authors.name, `%${v}%`))))!;
|
||||
return and(...values.map((v) => existsAuthor(accentInsensitiveIlike(authors.name, `%${v}%`))))!;
|
||||
case 'excludesAll':
|
||||
if (!values?.length) return sql`1 = 1`;
|
||||
return not(existsAuthor(or(...values.map((v) => ilike(authors.name, `%${v}%`)))!));
|
||||
return not(existsAuthor(or(...values.map((v) => accentInsensitiveIlike(authors.name, `%${v}%`)))!));
|
||||
case 'isEmpty':
|
||||
return not(existsAuthor());
|
||||
case 'isNotEmpty':
|
||||
|
||||
@@ -6,6 +6,7 @@ import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
import type { ContentFilterRules, JumpBucketsResponse, SortSpec } from '@bookorbit/types';
|
||||
import { isAudioFormat, isComicFormat } from '@bookorbit/types';
|
||||
import { buildContentFilterClauses } from '../../common/utils/content-filter-sql.utils';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import { SeriesIdentityService } from '../../common/services/series-identity.service';
|
||||
import { SeriesMembershipService } from '../../common/services/series-membership.service';
|
||||
import { BookQueryBuilder } from './book-query-builder.service';
|
||||
@@ -996,14 +997,14 @@ export class BookRepository {
|
||||
.selectDistinct({ bookId: bookAuthors.bookId })
|
||||
.from(bookAuthors)
|
||||
.innerJoin(authors, eq(authors.id, bookAuthors.authorId))
|
||||
.where(sql`${authors.name} ILIKE ${pattern}`)
|
||||
.where(accentInsensitiveIlike(authors.name, pattern))
|
||||
.as('matched_authors');
|
||||
|
||||
const matchedSeries = this.db
|
||||
.selectDistinct({ bookId: bookSeriesMemberships.bookId })
|
||||
.from(bookSeriesMemberships)
|
||||
.innerJoin(bookSeries, eq(bookSeries.id, bookSeriesMemberships.seriesId))
|
||||
.where(sql`${bookSeries.name} ILIKE ${pattern}`)
|
||||
.where(accentInsensitiveIlike(bookSeries.name, pattern))
|
||||
.as('matched_series');
|
||||
|
||||
const contentFilterClauses = contentFilters ? buildContentFilterClauses(contentFilters, this.db) : [];
|
||||
@@ -1028,8 +1029,8 @@ export class BookRepository {
|
||||
inArray(books.libraryId, libraryIds),
|
||||
ne(books.status, 'processing'),
|
||||
or(
|
||||
sql`${bookMetadata.title} ILIKE ${pattern}`,
|
||||
sql`${bookMetadata.seriesName} ILIKE ${pattern}`,
|
||||
accentInsensitiveIlike(bookMetadata.title, pattern),
|
||||
accentInsensitiveIlike(bookMetadata.seriesName, pattern),
|
||||
isNotNull(matchedAuthors.bookId),
|
||||
isNotNull(matchedSeries.bookId),
|
||||
),
|
||||
|
||||
@@ -6,8 +6,13 @@ vi.mock('drizzle-orm', () => ({
|
||||
sql: vi.fn((parts: TemplateStringsArray, ...values: unknown[]) => ({ type: 'sql', parts, values })),
|
||||
}));
|
||||
|
||||
import { and, eq, ilike, isNotNull } from 'drizzle-orm';
|
||||
vi.mock('../../common/utils/accent-insensitive-search.utils', () => ({
|
||||
accentInsensitiveIlike: vi.fn((left: unknown, pattern: string) => ({ type: 'accentInsensitiveIlike', left, pattern })),
|
||||
}));
|
||||
|
||||
import { and, eq, isNotNull } from 'drizzle-orm';
|
||||
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import { authors, bookMetadata, bookSeries, collections, narrators } from '../../db/schema';
|
||||
import { CatalogService } from './catalog.service';
|
||||
|
||||
@@ -90,7 +95,7 @@ describe('CatalogService', () => {
|
||||
const result = await service.searchAuthors(' A%_\\B ');
|
||||
|
||||
expect(result).toEqual([{ name: 'A%_\\B' }]);
|
||||
expect(ilike).toHaveBeenCalledWith(authors.name, '%A\\%\\_\\\\B%');
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(authors.name, '%A\\%\\_\\\\B%');
|
||||
expect(selectChains[0]?.from).toHaveBeenCalledWith(authors);
|
||||
expect(selectChains[0]?.orderBy).toHaveBeenCalledWith(authors.name);
|
||||
expect(selectChains[0]?.limit).toHaveBeenCalledWith(15);
|
||||
@@ -116,7 +121,7 @@ describe('CatalogService', () => {
|
||||
|
||||
expect(result).toEqual([{ name: 'Orbit' }, { name: 'Tor' }]);
|
||||
expect(isNotNull).toHaveBeenCalledWith(bookMetadata.publisher);
|
||||
expect(ilike).toHaveBeenCalledWith(bookMetadata.publisher, '%or%');
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(bookMetadata.publisher, '%or%');
|
||||
expect(selectDistinctChains[0]?.from).toHaveBeenCalledWith(bookMetadata);
|
||||
expect(selectDistinctChains[0]?.limit).toHaveBeenCalledWith(15);
|
||||
});
|
||||
@@ -127,7 +132,7 @@ describe('CatalogService', () => {
|
||||
|
||||
await service.searchSeries('Expanse');
|
||||
|
||||
expect(ilike).toHaveBeenCalledWith(bookSeries.name, '%Expanse%');
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(bookSeries.name, '%Expanse%');
|
||||
expect(selectChains[0]?.from).toHaveBeenCalledWith(bookSeries);
|
||||
expect(selectChains[0]?.orderBy).toHaveBeenCalledWith(bookSeries.name);
|
||||
});
|
||||
@@ -139,7 +144,7 @@ describe('CatalogService', () => {
|
||||
await service.searchLanguages('English');
|
||||
|
||||
expect(isNotNull).toHaveBeenCalledWith(bookMetadata.language);
|
||||
expect(ilike).toHaveBeenCalledWith(bookMetadata.language, '%English%');
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(bookMetadata.language, '%English%');
|
||||
});
|
||||
|
||||
it('returns an empty result for blank collection terms to match other search endpoints', async () => {
|
||||
@@ -157,7 +162,7 @@ describe('CatalogService', () => {
|
||||
|
||||
expect(result).toEqual([{ name: 'Sci-Fi Favorites' }]);
|
||||
expect(eq).toHaveBeenCalledWith(collections.userId, 42);
|
||||
expect(ilike).toHaveBeenCalledWith(collections.name, '%sci\\_fi\\%%');
|
||||
expect(accentInsensitiveIlike).toHaveBeenCalledWith(collections.name, '%sci\\_fi\\%%');
|
||||
expect(and).toHaveBeenCalledTimes(1);
|
||||
expect(selectChains[0]?.from).toHaveBeenCalledWith(collections);
|
||||
expect(selectChains[0]?.where).toHaveBeenCalledWith(expect.objectContaining({ type: 'and' }));
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { and, eq, ilike, isNotNull } from 'drizzle-orm';
|
||||
import { and, eq, isNotNull } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../db/schema';
|
||||
import { authors, bookMetadata, bookSeries, collections, genres, narrators, tags } from '../../db/schema';
|
||||
|
||||
@@ -56,7 +57,7 @@ export class CatalogService {
|
||||
return this.db
|
||||
.select({ name: collections.name })
|
||||
.from(collections)
|
||||
.where(and(eq(collections.userId, userId), ilike(collections.name, pattern)))
|
||||
.where(and(eq(collections.userId, userId), accentInsensitiveIlike(collections.name, pattern)))
|
||||
.orderBy(collections.name)
|
||||
.limit(COLLECTION_SEARCH_LIMIT);
|
||||
}
|
||||
@@ -65,7 +66,12 @@ export class CatalogService {
|
||||
const pattern = this.toContainsPattern(q);
|
||||
if (!pattern) return Promise.resolve([]);
|
||||
|
||||
return this.db.select({ name: table.name }).from(table).where(ilike(table.name, pattern)).orderBy(table.name).limit(DEFAULT_SEARCH_LIMIT);
|
||||
return this.db
|
||||
.select({ name: table.name })
|
||||
.from(table)
|
||||
.where(accentInsensitiveIlike(table.name, pattern))
|
||||
.orderBy(table.name)
|
||||
.limit(DEFAULT_SEARCH_LIMIT);
|
||||
}
|
||||
|
||||
private searchByNameWithId(q: string, table: NamedTableWithId): Promise<SearchResultWithId[]> {
|
||||
@@ -75,7 +81,7 @@ export class CatalogService {
|
||||
return this.db
|
||||
.select({ id: table.id, name: table.name })
|
||||
.from(table)
|
||||
.where(ilike(table.name, pattern))
|
||||
.where(accentInsensitiveIlike(table.name, pattern))
|
||||
.orderBy(table.name)
|
||||
.limit(DEFAULT_SEARCH_LIMIT);
|
||||
}
|
||||
@@ -87,7 +93,7 @@ export class CatalogService {
|
||||
const rows = await this.db
|
||||
.selectDistinct({ name: column })
|
||||
.from(bookMetadata)
|
||||
.where(and(isNotNull(column), ilike(column, pattern)))
|
||||
.where(and(isNotNull(column), accentInsensitiveIlike(column, pattern)))
|
||||
.orderBy(column)
|
||||
.limit(DEFAULT_SEARCH_LIMIT);
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { asc, count, desc, eq, ilike, inArray, or, type SQL } from 'drizzle-orm';
|
||||
import { asc, count, desc, eq, inArray, or, type SQL } from 'drizzle-orm';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
import type { CustomIconSort } from '@bookorbit/types';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../db/schema';
|
||||
import type { CustomIconRow, NewCustomIcon } from '../../db/schema';
|
||||
|
||||
@@ -117,7 +118,7 @@ export class CustomIconRepository {
|
||||
const term = q?.trim();
|
||||
if (!term) return undefined;
|
||||
const pattern = `%${escapeLike(term)}%`;
|
||||
return or(ilike(schema.customIcons.name, pattern), ilike(schema.customIcons.slug, pattern));
|
||||
return or(accentInsensitiveIlike(schema.customIcons.name, pattern), accentInsensitiveIlike(schema.customIcons.slug, pattern));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { and, asc, count, desc, eq, exists, ilike, inArray, notExists, or, sql } from 'drizzle-orm';
|
||||
import { and, asc, count, desc, eq, exists, inArray, notExists, or, sql } from 'drizzle-orm';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../../db';
|
||||
@@ -7,6 +7,7 @@ import { refreshPrimaryAuthorSortNamesForBooks } from '../../../db/book-author-s
|
||||
import * as schema from '../../../db/schema';
|
||||
import { authors, bookAuthors, books, bookMetadata } from '../../../db/schema';
|
||||
import { buildContentFilterClauses } from '../../../common/utils/content-filter-sql.utils';
|
||||
import { accentInsensitiveIlike } from '../../../common/utils/accent-insensitive-search.utils';
|
||||
import {
|
||||
chooseCanonicalMetadataTextRow,
|
||||
normalizeMetadataText,
|
||||
@@ -138,7 +139,7 @@ export class AuthorStrategy implements EntityStrategy {
|
||||
async browse(params: BrowseParams): Promise<BrowseResult> {
|
||||
const bookCountExpr = sql<number>`count(distinct ${bookAuthors.bookId})::int`;
|
||||
|
||||
const nameCondition = params.search ? ilike(authors.name, `%${escapeLike(params.search)}%`) : undefined;
|
||||
const nameCondition = params.search ? accentInsensitiveIlike(authors.name, `%${escapeLike(params.search)}%`) : undefined;
|
||||
|
||||
const cfClauses = params.contentFilters ? buildContentFilterClauses(params.contentFilters, this.db) : [];
|
||||
const libraryBookIds =
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import type { InlineEntityType } from '@bookorbit/types';
|
||||
import { DB } from '../../../db';
|
||||
import { accentInsensitiveIlike } from '../../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../../db/schema';
|
||||
import { bookMetadata } from '../../../db/schema';
|
||||
import type {
|
||||
@@ -102,7 +103,7 @@ export abstract class InlineEntityStrategy implements EntityStrategy {
|
||||
|
||||
let searchFilter = sql``;
|
||||
if (params.search) {
|
||||
searchFilter = sql` AND bm.${f} ILIKE ${'%' + escapeLike(params.search) + '%'}`;
|
||||
searchFilter = sql` AND ${accentInsensitiveIlike(sql`bm.${f}`, '%' + escapeLike(params.search) + '%')}`;
|
||||
}
|
||||
|
||||
const idsLiteral = sql.raw(`(${params.libraryIds.join(',')})`);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BadRequestException, Inject, NotFoundException } from '@nestjs/common';
|
||||
import { and, asc, count, desc, eq, exists, ilike, inArray, notExists, or, sql } from 'drizzle-orm';
|
||||
import { and, asc, count, desc, eq, exists, inArray, notExists, or, sql } from 'drizzle-orm';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
import type { PgColumn, PgTableWithColumns } from 'drizzle-orm/pg-core';
|
||||
|
||||
@@ -7,6 +7,7 @@ import { DB } from '../../../db';
|
||||
import * as schema from '../../../db/schema';
|
||||
import { books, bookMetadata } from '../../../db/schema';
|
||||
import { buildContentFilterClauses } from '../../../common/utils/content-filter-sql.utils';
|
||||
import { accentInsensitiveIlike } from '../../../common/utils/accent-insensitive-search.utils';
|
||||
import type {
|
||||
BrowseParams,
|
||||
BrowseResult,
|
||||
@@ -132,7 +133,7 @@ export abstract class JunctionEntityStrategy implements EntityStrategy {
|
||||
.where(and(inArray(books.libraryId, params.libraryIds), ...cfClauses))
|
||||
: null;
|
||||
|
||||
const nameCondition = params.search ? (ilike(this.nameCol, `%${escapeLike(params.search)}%`) as any) : undefined;
|
||||
const nameCondition = params.search ? (accentInsensitiveIlike(this.nameCol, `%${escapeLike(params.search)}%`) as any) : undefined;
|
||||
const joinCondition = bookSubquery
|
||||
? and(eq(this.junctionEntityIdCol, this.entityIdCol), inArray(this.junctionBookIdCol, bookSubquery))
|
||||
: and(eq(this.junctionEntityIdCol, this.entityIdCol), sql`false`);
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { BadRequestException, Inject, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { SQL, and, asc, count, desc, eq, exists, ilike, inArray, notExists, or, sql } from 'drizzle-orm';
|
||||
import { SQL, and, asc, count, desc, eq, exists, inArray, notExists, or, sql } from 'drizzle-orm';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { buildContentFilterClauses } from '../../../common/utils/content-filter-sql.utils';
|
||||
import { accentInsensitiveIlike } from '../../../common/utils/accent-insensitive-search.utils';
|
||||
import { normalizeMetadataText, normalizeMetadataTextKey } from '../../../common/utils/metadata-text-normalize.utils';
|
||||
import { DB } from '../../../db';
|
||||
import * as schema from '../../../db/schema';
|
||||
@@ -118,7 +119,7 @@ export class SeriesStrategy implements EntityStrategy {
|
||||
.from(books)
|
||||
.where(and(inArray(books.libraryId, params.libraryIds), ...filterClauses))
|
||||
: null;
|
||||
const nameCondition = params.search ? (ilike(bookSeries.name, `%${escapeLike(params.search)}%`) as never) : undefined;
|
||||
const nameCondition = params.search ? (accentInsensitiveIlike(bookSeries.name, `%${escapeLike(params.search)}%`) as never) : undefined;
|
||||
const joinCondition = bookSubquery
|
||||
? and(eq(bookSeriesMemberships.seriesId, bookSeries.id), inArray(bookSeriesMemberships.bookId, bookSubquery))
|
||||
: and(eq(bookSeriesMemberships.seriesId, bookSeries.id), sql`false`);
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { and, eq, ilike, inArray, or, sql } from 'drizzle-orm';
|
||||
import { and, eq, inArray, or, sql } from 'drizzle-orm';
|
||||
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../../db';
|
||||
import { accentInsensitiveIlike } from '../../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../../db/schema';
|
||||
import type { SourceBook } from '../adapters/source-adapter.types';
|
||||
import type { PathMapping, PlannedBookMatch, PlannedUnresolvedBook, UnresolvedReasonCode } from './planner.types';
|
||||
@@ -267,13 +268,18 @@ export class MatchingService {
|
||||
return exactResult;
|
||||
}
|
||||
|
||||
const approxClauses = normalizedAuthors.map((author) => ilike(schema.authors.name, `%${escapeLike(author)}%`));
|
||||
const approxClauses = normalizedAuthors.map((author) => accentInsensitiveIlike(schema.authors.name, `%${escapeLike(author)}%`));
|
||||
const approxRows = await this.db
|
||||
.selectDistinct({ bookId: schema.bookMetadata.bookId })
|
||||
.from(schema.bookMetadata)
|
||||
.innerJoin(schema.bookAuthors, eq(schema.bookAuthors.bookId, schema.bookMetadata.bookId))
|
||||
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
|
||||
.where(and(sql`lower(${schema.bookMetadata.title}) = lower(${normalizedTitle})`, or(...approxClauses)!))
|
||||
.where(
|
||||
and(
|
||||
sql`lower(public.bookorbit_unaccent(${schema.bookMetadata.title})) = lower(public.bookorbit_unaccent(${normalizedTitle}))`,
|
||||
or(...approxClauses)!,
|
||||
),
|
||||
)
|
||||
.limit(2);
|
||||
const approxResult = toLookupResult(approxRows);
|
||||
if (approxResult.kind !== 'none') {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { ForbiddenException, Inject, Injectable } from '@nestjs/common';
|
||||
import { SQL, and, count, eq, ilike, inArray, or, sql } from 'drizzle-orm';
|
||||
import { SQL, and, count, eq, inArray, or, sql } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import * as schema from '../../db/schema';
|
||||
import {
|
||||
authors,
|
||||
@@ -246,7 +247,7 @@ export class OpdsBookService {
|
||||
.select({ one: sql`1` })
|
||||
.from(bookAuthors)
|
||||
.innerJoin(authors, eq(bookAuthors.authorId, authors.id))
|
||||
.where(and(eq(bookAuthors.bookId, books.id), ilike(authors.name, pattern))!);
|
||||
.where(and(eq(bookAuthors.bookId, books.id), accentInsensitiveIlike(authors.name, pattern))!);
|
||||
return sql`exists (${sq})`;
|
||||
})();
|
||||
|
||||
@@ -255,10 +256,15 @@ export class OpdsBookService {
|
||||
FROM ${bookSeriesMemberships}
|
||||
INNER JOIN ${bookSeries} ON ${bookSeries.id} = ${bookSeriesMemberships.seriesId}
|
||||
WHERE ${bookSeriesMemberships.bookId} = ${books.id}
|
||||
AND ${ilike(bookSeries.name, pattern)}
|
||||
AND ${accentInsensitiveIlike(bookSeries.name, pattern)}
|
||||
)`;
|
||||
|
||||
const clauses: SQL[] = [ilike(bookMetadata.title, pattern), existsAuthor, existsSeries, ilike(bookMetadata.seriesName, pattern)];
|
||||
const clauses: SQL[] = [
|
||||
accentInsensitiveIlike(bookMetadata.title, pattern),
|
||||
existsAuthor,
|
||||
existsSeries,
|
||||
accentInsensitiveIlike(bookMetadata.seriesName, pattern),
|
||||
];
|
||||
const normalizedIsbn = normalizeIsbnSearchTerm(term);
|
||||
if (normalizedIsbn) {
|
||||
clauses.push(or(eq(bookMetadata.isbn13, normalizedIsbn), eq(bookMetadata.isbn10, normalizedIsbn))!);
|
||||
@@ -362,7 +368,7 @@ export class OpdsBookService {
|
||||
const where: SQL[] = [inArray(books.libraryId, accessibleIds)];
|
||||
const term = opts.q?.trim();
|
||||
if (term) {
|
||||
where.push(ilike(authors.name, `%${term.replace(LIKE_SPECIAL_CHARS, '\\$&')}%`));
|
||||
where.push(accentInsensitiveIlike(authors.name, `%${term.replace(LIKE_SPECIAL_CHARS, '\\$&')}%`));
|
||||
}
|
||||
|
||||
const rows = await this.db
|
||||
@@ -396,7 +402,7 @@ export class OpdsBookService {
|
||||
const where: SQL[] = [inArray(books.libraryId, accessibleIds)];
|
||||
const term = opts.q?.trim();
|
||||
if (term) {
|
||||
where.push(ilike(bookSeries.name, `%${term.replace(LIKE_SPECIAL_CHARS, '\\$&')}%`));
|
||||
where.push(accentInsensitiveIlike(bookSeries.name, `%${term.replace(LIKE_SPECIAL_CHARS, '\\$&')}%`));
|
||||
}
|
||||
|
||||
const rows = await this.db
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { SQL, and, asc, count, desc, eq, ilike, inArray, isNotNull, sql } from 'drizzle-orm';
|
||||
import { SQL, and, asc, count, desc, eq, inArray, isNotNull, sql } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import type { ContentFilterRules } from '@bookorbit/types';
|
||||
import { buildContentFilterClauses } from '../../common/utils/content-filter-sql.utils';
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import { DB } from '../../db';
|
||||
import * as schema from '../../db/schema';
|
||||
import { authors, bookAuthors, bookMetadata, books, bookSeries, bookSeriesMemberships, userBookStatus } from '../../db/schema';
|
||||
@@ -47,7 +48,7 @@ export class SeriesRepository {
|
||||
return sql`${books.id} IN (
|
||||
SELECT ${bookAuthors.bookId} FROM ${bookAuthors}
|
||||
INNER JOIN ${authors} ON ${authors.id} = ${bookAuthors.authorId}
|
||||
WHERE ${ilike(authors.name, pattern)}
|
||||
WHERE ${accentInsensitiveIlike(authors.name, pattern)}
|
||||
)`;
|
||||
}
|
||||
|
||||
@@ -71,7 +72,7 @@ export class SeriesRepository {
|
||||
if (params.q) {
|
||||
const qPattern = `%${this.escapeLikePattern(params.q)}%`;
|
||||
const authorNameMatch = this.buildAuthorNameMatchCondition(qPattern);
|
||||
conditions.push(sql`(${ilike(bookSeries.name, qPattern)} OR ${authorNameMatch})`);
|
||||
conditions.push(sql`(${accentInsensitiveIlike(bookSeries.name, qPattern)} OR ${authorNameMatch})`);
|
||||
}
|
||||
|
||||
if (params.author) {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { Client } from 'pg';
|
||||
|
||||
import { installPostgresExtensions } from './postgres-extensions';
|
||||
|
||||
const DEFAULT_E2E_DATABASE_URL = 'postgres://bookorbit:bookorbit@localhost:5432/bookorbit_e2e';
|
||||
const CONNECT_RETRY_ATTEMPTS = 30;
|
||||
const CONNECT_RETRY_DELAY_MS = 1000;
|
||||
@@ -65,9 +67,7 @@ async function createDatabaseIfMissing(adminClient: Client, databaseName: string
|
||||
async function resetE2EDatabase(targetClient: Client): Promise<void> {
|
||||
await targetClient.query('DROP SCHEMA IF EXISTS public CASCADE');
|
||||
await targetClient.query('CREATE SCHEMA public');
|
||||
await targetClient.query('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"');
|
||||
await targetClient.query('CREATE EXTENSION IF NOT EXISTS pg_trgm');
|
||||
await targetClient.query('CREATE EXTENSION IF NOT EXISTS vector');
|
||||
await installPostgresExtensions(targetClient);
|
||||
await targetClient.query('CREATE SCHEMA IF NOT EXISTS drizzle');
|
||||
await targetClient.query('CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations (id serial PRIMARY KEY, hash text NOT NULL, created_at bigint)');
|
||||
await targetClient.query('DELETE FROM drizzle.__drizzle_migrations');
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Pool } from 'pg';
|
||||
|
||||
import { installPostgresExtensions } from './postgres-extensions';
|
||||
|
||||
async function run(): Promise<void> {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
if (!connectionString) throw new Error('DATABASE_URL is required');
|
||||
|
||||
const pool = new Pool({ connectionString });
|
||||
try {
|
||||
await installPostgresExtensions(pool);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
void run();
|
||||
@@ -4,6 +4,8 @@ import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
import { migrate } from 'drizzle-orm/node-postgres/migrator';
|
||||
import { Pool } from 'pg';
|
||||
|
||||
import { installPostgresExtensions } from './postgres-extensions';
|
||||
|
||||
function resolveMigrationsFolder(): string {
|
||||
const candidates = [
|
||||
join(__dirname, '..', '..', 'migrations'),
|
||||
@@ -33,11 +35,7 @@ async function runMigrations() {
|
||||
});
|
||||
|
||||
try {
|
||||
await pool.query(`
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_trgm";
|
||||
CREATE EXTENSION IF NOT EXISTS "vector";
|
||||
`);
|
||||
await installPostgresExtensions(pool);
|
||||
|
||||
const migrationsFolder = resolveMigrationsFolder();
|
||||
await migrate(drizzle(pool), { migrationsFolder });
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { installPostgresExtensions } from './postgres-extensions';
|
||||
|
||||
describe('installPostgresExtensions', () => {
|
||||
it('installs every extension required by application queries', async () => {
|
||||
const query = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
await installPostgresExtensions({ query });
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(1);
|
||||
const statement = query.mock.calls[0]?.[0] as string;
|
||||
expect(statement).toContain('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"');
|
||||
expect(statement).toContain('CREATE EXTENSION IF NOT EXISTS "pg_trgm"');
|
||||
expect(statement).toContain('CREATE EXTENSION IF NOT EXISTS "unaccent"');
|
||||
expect(statement).toContain('CREATE EXTENSION IF NOT EXISTS "vector"');
|
||||
expect(statement).toContain('CREATE OR REPLACE FUNCTION public.bookorbit_unaccent(value text)');
|
||||
expect(statement).toContain("public.unaccent('public.unaccent', value)");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,20 @@
|
||||
type SqlExecutor = {
|
||||
query(sql: string): Promise<unknown>;
|
||||
};
|
||||
|
||||
export async function installPostgresExtensions(executor: SqlExecutor): Promise<void> {
|
||||
await executor.query(`
|
||||
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
||||
CREATE EXTENSION IF NOT EXISTS "pg_trgm";
|
||||
CREATE EXTENSION IF NOT EXISTS "unaccent";
|
||||
CREATE EXTENSION IF NOT EXISTS "vector";
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.bookorbit_unaccent(value text)
|
||||
RETURNS text
|
||||
LANGUAGE sql
|
||||
IMMUTABLE
|
||||
PARALLEL SAFE
|
||||
STRICT
|
||||
AS $function$ SELECT public.unaccent('public.unaccent', value) $function$;
|
||||
`);
|
||||
}
|
||||
Reference in New Issue
Block a user