logging

.config/devenv.yml

@@ -28,6 +28,12 @@ redis:
 #  user:
 #  pass:
 
+#meilisearch
+# host: localhost
+# port: 7700
+# ssl: false
+# apiKey: ''
+
 id: 'aid'
 
 reservedUsernames:

.config/example.yml

@@ -82,6 +82,15 @@ redis:
 #  user:
 #  pass:
 
+#   ┌───────────────────────────┐
+#───┘ MeiliSearch configuration └─────────────────────────────
+
+#meilisearch:
+#  host: meilisearch
+#  port: 7700
+#  apiKey: ''
+#  ssl: false
+
 #   ┌───────────────┐
 #───┘ ID generation └───────────────────────────────────────────
 

.gitignore

@@ -33,6 +33,7 @@ coverage
 built
 db
 elasticsearch
+meili_data
 redis
 npm-debug.log
 *.pem

CALCKEY.md

@@ -124,6 +124,7 @@
 - Improve system emails
 - Mod mail
 - Focus trapping and button labels
+- Meilisearch
 
 ## Implemented (remote)
 

README.md

@@ -89,7 +89,8 @@ If you have access to a server that supports one of the sources below, I recomme
 
 - [FFmpeg](https://ffmpeg.org/) for video transcoding
 - Full text search (one of the following)
-  - 🦔 [Sonic](https://crates.io/crates/sonic-server) (recommended)
+  - 🦔 [Sonic](https://crates.io/crates/sonic-server)
+  - [MeiliSearch](https://www.meilisearch.com/)
   - [ElasticSearch](https://www.elastic.co/elasticsearch/)
 
 ### 🏗️ Build dependencies
@@ -140,7 +141,11 @@ psql postgres -c "create database calckey with encoding = 'UTF8';"
 
 In Calckey's directory, fill out the `db` section of `.config/default.yml` with the correct information, where the `db` key is `calckey`.
 
-## 🦔 Set up search
+## 🔎 Set up search
+
+### 🦔 Sonic
+
+Sonic is better suited for self hosters with smaller deployments. It's easier to use, uses almost no resources, and takes barely any any disk space.
 
 Follow sonic's [installation guide](https://github.com/valeriansaliou/sonic#installation)
 
@@ -148,6 +153,17 @@ If you use IPv4: in Sonic's directory, edit the `config.cfg` file to change `ine
 
 In Calckey's directory, fill out the `sonic` section of `.config/default.yml` with the correct information.
 
+### Meilisearch
+
+Meilisearch is better suited for larger deployments. It's faster but uses far more resources and disk space.
+
+Follow Meilisearch's [quick start guide](https://www.meilisearch.com/docs/learn/getting_started/quick_start)
+
+In Calckey's directory, fill out the `meilisearch` section of `.config/default.yml` with the correct information.
+
+### ElasticSearch
+
+Please don't use ElasticSearch unless you already have an ElasticSearch setup and want to continue using it for Calckey. ElasticSearch is slow, heavy, and offers very few benefits over Sonic/Meilisearch.
 
 ## 💅 Customize
 

chart/templates/_helpers.tpl

@@ -164,6 +164,15 @@ redis:
 #  collection: notes
 #  bucket: default
 
+#   ┌───────────────────────────┐
+#───┘ MeiliSearch configuration └─────────────────────────────
+
+#meilisearch:
+#  host: meilisearch
+#  port: 7700
+#  apiKey: ''
+#  ssl: false
+
 #   ┌─────────────────────────────┐
 #───┘ Elasticsearch configuration └─────────────────────────────
 

packages/backend/package.json

@@ -186,6 +186,7 @@
 		"execa": "6.1.0",
 		"json5": "2.2.3",
 		"json5-loader": "4.0.1",
+		"meilisearch": "^0.32.3",
 		"mocha": "10.2.0",
 		"pug": "3.0.2",
 		"strict-event-emitter-types": "2.0.0",

packages/backend/src/config/types.ts

@@ -39,6 +39,12 @@ export type Source = {
 		collection?: string;
 		bucket?: string;
 	};
+	meilisearch: {
+		host: string;
+		port: number;
+		apiKey?: string;
+		ssl?: boolean;
+	};
 
 	proxy?: string;
 	proxySmtp?: string;
@@ -81,6 +87,8 @@ export type Source = {
 
 	reservedUsernames?: string[];
 
+	logLevel?: "error" | "warn" | "info" | "debug" | "trace";
+
 	// Managed hosting stuff
 	maxUserSignups?: number;
 	isManagedHosting?: boolean;

packages/backend/src/db/meilisearch.ts

@@ -0,0 +1,23 @@
+import * as MeiliSearch from "meilisearch";
+import { dbLogger } from "./logger.js";
+
+import config from "@/config/index.js";
+
+const logger = dbLogger.createSubLogger("meilisearch", "gray", false);
+
+logger.info("Connecting to Meilisearch");
+
+const hasConfig =
+	config.meilisearch && (config.meilisearch.host || config.meilisearch.port);
+
+const host = hasConfig ? config.meilisearch.host ?? "localhost" : "";
+const port = hasConfig ? config.meilisearch.port ?? 7700 : 0;
+const ssl = hasConfig ? config.meilisearch.ssl ?? false : true;
+const apiKey = hasConfig ? config.meilisearch.apiKey ?? "" : "";
+
+export default hasConfig
+	? new MeiliSearch.MeiliSearch({
+			host: `${ssl ? "https" : "http"}://${host}:${port}`,
+			apiKey,
+	  })
+	: null;

packages/backend/src/server/api/endpoints/notes/search.ts

@@ -2,13 +2,17 @@ import { In } from "typeorm";
 import { Notes } from "@/models/index.js";
 import { Note } from "@/models/entities/note.js";
 import config from "@/config/index.js";
-import es from "../../../../db/elasticsearch.js";
-import sonic from "../../../../db/sonic.js";
+import es from "@/db/elasticsearch.js";
+import sonic from "@/db/sonic.js";
+import meilisearch from "@/db/meilisearch.js";
 import define from "../../define.js";
 import { makePaginationQuery } from "../../common/make-pagination-query.js";
 import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
 import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js";
 import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
+import { apiLogger } from "../../logger.js";
+
+const logger = apiLogger.createSubLogger("search", "blue", true);
 
 export const meta = {
 	tags: ["notes"],
@@ -61,7 +65,7 @@ export const paramDef = {
 } as const;
 
 export default define(meta, paramDef, async (ps, me) => {
-	if (es == null && sonic == null) {
+	if (es == null && sonic == null && meilisearch == null) {
 		const query = makePaginationQuery(
 			Notes.createQueryBuilder("note"),
 			ps.sinceId,
@@ -97,6 +101,36 @@ export default define(meta, paramDef, async (ps, me) => {
 		const notes: Note[] = await query.take(ps.limit).getMany();
 
 		return await Notes.packMany(notes, me);
+	} else if (meilisearch) {
+		//search in meilisearch
+		try {
+			const result = await meilisearch.index("notes").search(ps.query, {
+				limit: ps.limit,
+				offset: ps.offset,
+				filters: ps.userId,
+				// ? `userId = ${ps.userId}`
+				// : ps.channelId
+				// ? `channelId = ${ps.channelId}`
+				// : undefined,
+			});
+
+			const ids = result.hits.map((hit: { id: string }) => hit.id);
+
+			// Fetch the notes from the database until we have enough to satisfy the limit
+			const notes: Note[] = await Notes.find({
+				where: {
+					id: In(ids),
+				},
+				order: {
+					id: "DESC",
+				},
+			});
+
+			return await Notes.packMany(notes, me);
+		} catch (error) {
+			logger.error(error.message);
+			return [];
+		}
 	} else if (sonic) {
 		let start = 0;
 		const chunkSize = 100;

packages/backend/src/services/logger.ts

@@ -60,6 +60,26 @@ export default class Logger {
 		if (!this.store) store = false;
 		if (level === "debug") store = false;
 
+		// filter out logs based on config log level
+		if (config.logLevel === "error" && level !== "error") return;
+		if (config.logLevel === "warn" && level !== "error" && level !== "warning")
+			return;
+		if (
+			config.logLevel === "info" &&
+			level !== "error" &&
+			level !== "warning" &&
+			level !== "info"
+		)
+			return;
+		if (
+			config.logLevel === "debug" &&
+			level !== "error" &&
+			level !== "warning" &&
+			level !== "info" &&
+			level !== "debug"
+		)
+			return;
+
 		if (this.parentLogger) {
 			this.parentLogger.log(
 				level,

packages/backend/src/services/note/create.ts

@@ -1,6 +1,7 @@
 import * as mfm from "mfm-js";
-import es from "../../db/elasticsearch.js";
-import sonic from "../../db/sonic.js";
+import es from "@/db/elasticsearch.js";
+import sonic from "@/db/sonic.js";
+import meilisearch from "@/db/meilisearch.js";
 import {
 	publishMainStream,
 	publishNotesStream,
@@ -776,6 +777,25 @@ export async function index(note: Note): Promise<void> {
 			note.text,
 		);
 	}
+
+	if (meilisearch) {
+		await meilisearch.index("notes").addDocuments(
+			[
+				{
+					id: note.id.toString(),
+					text: note.text,
+					userId: note.userId,
+					userHost: note.userHost,
+					channelId: note.channelId,
+					cw: note.cw,
+					createdAt: note.createdAt.getTime(),
+				},
+			],
+			{
+				primaryKey: "id",
+			},
+		);
+	}
 }
 
 async function notifyToWatchersOfRenotee(

pnpm-lock.yaml

@@ -565,6 +565,9 @@ importers:
       json5-loader:
         specifier: 4.0.1
         version: 4.0.1(webpack@5.75.0)
+      meilisearch:
+        specifier: ^0.32.3
+        version: 0.32.3
       mocha:
         specifier: 10.2.0
         version: 10.2.0
@@ -10367,6 +10370,14 @@ packages:
     engines: {node: '>= 0.6'}
     dev: false
 
+  /meilisearch@0.32.3:
+    resolution: {integrity: sha512-EOgfBuRE5SiIPIpEDYe2HO0D7a4z5bexIgaAdJFma/dH5hx1kwO+u/qb2g3qKyjG+iA3l8MlmTj/Xd72uahaAw==}
+    dependencies:
+      cross-fetch: 3.1.5
+    transitivePeerDependencies:
+      - encoding
+    dev: true
+
   /meow@9.0.0:
     resolution: {integrity: sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==}
     engines: {node: '>=10'}