= Record<\n string,\n string | undefined\n >,\n >(\n target: RequestHandler,\n _context: ClassMethodDecoratorContext<\n This,\n (\n this: This,\n ...args: Parameters>\n ) => ReturnType>\n >,\n ) => {\n _context.addInitializer(function () {\n // eslint-disable-next-line no-invalid-this\n const proto = Object.getPrototypeOf(this); // will be bound to class\n if ( ! proto.__routes ) {\n proto.__routes = [];\n }\n proto.__routes.push({\n method,\n path,\n options: options as EndpointOptions | undefined,\n adminUsernames: adminUsernames\n ? [...adminUsernames, 'admin', 'system']\n : undefined,\n allowedAppIds,\n handler: target,\n });\n });\n };\n };\n};\n\n// HTTP method decorators\nexport const Get = createMethodDecorator('get');\nexport const Post = createMethodDecorator('post');\nexport const Put = createMethodDecorator('put');\nexport const Delete = createMethodDecorator('delete');\n// TODO DS: add others as needed (patch, etc)\n\nexport class HttpError extends Error {\n statusCode: number;\n constructor (statusCode: StatusCodes, message: string, cause?: unknown) {\n super(`${statusCode} - ${message}`, { cause });\n this.statusCode = statusCode;\n }\n}\n\n// Registers all routes from a decorated controller instance to an Express router\nexport class ExtensionController {\n logger?: Console;\n // TODO DS: make this work with other express-like routers\n registerRoutes () {\n const logger = this.logger || console;\n const prefix = Object.getPrototypeOf(this).__controllerPrefix || '';\n const adminsForController = Object.getPrototypeOf(this).__adminUsernames as\n | string[]\n | undefined;\n const allowedAppIdsForController = Object.getPrototypeOf(this).__allowedAppIds as\n | string[]\n | undefined;\n const routes: RouteMeta[] = Object.getPrototypeOf(this).__routes || [];\n for ( const route of routes ) {\n const fullPath = `${prefix}/${route.path}`.replace(/\\/+/g, '/');\n const adminsForRoute = route.adminUsernames\n ? adminsForController\n ? adminsForController.concat(route.adminUsernames)\n : route.adminUsernames\n : adminsForController\n ? adminsForController\n : undefined;\n const allowedAppIds = route.allowedAppIds\n ? allowedAppIdsForController\n ? allowedAppIdsForController.concat(route.allowedAppIds)\n : route.allowedAppIds\n : allowedAppIdsForController\n ? allowedAppIdsForController\n : undefined;\n\n if ( ! extension[route.method] ) {\n throw new Error(`Unsupported HTTP method: ${route.method}`);\n } else {\n logger.log(`Registering route: [${route.method.toUpperCase()}] ${fullPath}`);\n\n (extension[route.method] as RouterMethods[HttpMethod])(\n fullPath,\n route.options || {},\n async (req, res, next) => {\n try {\n if ( adminsForRoute || allowedAppIds ) {\n if ( ! req.actor ) {\n throw new HttpError(StatusCodes.UNAUTHORIZED, 'Unauthenticated');\n }\n }\n if ( adminsForRoute ) {\n if ( ! adminsForRoute.includes(req.actor!.type.user.username) ) {\n throw new HttpError(\n StatusCodes.FORBIDDEN,\n 'Only admins may request this resource.',\n );\n }\n }\n if ( allowedAppIds ) {\n if ( ( req.actor!.type?.app?.uid && !allowedAppIds.includes(req.actor!.type.app.uid) ) ) {\n throw new HttpError(\n StatusCodes.FORBIDDEN,\n 'This app may not request this resource.',\n );\n }\n }\n await route.handler.bind(this)(req, res, next);\n } catch ( error ) {\n if ( error instanceof HttpError ) {\n res.status(error.statusCode).send({ error: error.message });\n logger.warn('httpError:', error);\n return;\n }\n if ( error instanceof Error ) {\n res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: error.message });\n logger.error('Non-http error:', error);\n return;\n }\n res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: 'An unknown error occurred' });\n logger.error('An unknown error occurred:', error);\n }\n },\n );\n }\n }\n }\n}\n"
},
{
"path": "extensions/extensionController/src/index.ts",
"content": "import { Controller, Delete, ExtensionController, Get, HttpError, Post, Put } from './ExtensionController.js';\n\nextension.exports = {\n ExtensionController,\n Controller,\n Get,\n Put,\n Post,\n Delete,\n HttpError,\n};\n\nexport {\n Controller, Delete, ExtensionController, Get, HttpError, Post, Put,\n};\n"
},
{
"path": "extensions/extensionController/tsconfig.json",
"content": "{\n \"compilerOptions\": {\n \"target\": \"ES2024\",\n \"module\": \"nodenext\",\n \"moduleResolution\": \"nodenext\",\n \"strict\": true,\n \"forceConsistentCasingInFileNames\": true,\n \"skipLibCheck\": true,\n \"sourceMap\": true,\n \"noEmitOnError\": true,\n \"noImplicitAny\": false,\n \"allowJs\": true,\n \"checkJs\": false, \n },\n \"include\": [\n \"./**/*.ts\",\n \"./**/*.d.ts\"\n ],\n \"exclude\": [\n \"**/*.test.ts\",\n \"**/*.spec.ts\",\n \"**/test/**\",\n \"**/tests/**\",\n \"node_modules\",\n \"dist\",\n \"*.js\"\n ]\n}"
},
{
"path": "extensions/hellodriver/config.json",
"content": "{\n \"test\": \"yes I am a test\"\n}"
},
{
"path": "extensions/hellodriver/hellodriver.js",
"content": "const { kv } = extension.import('data');\n\nconst span = extension.span;\n\n/**\n * Here we create an interface called 'hello-world'. This interface\n * specifies that any implementation of 'hello-world' should implement\n * a method called `greet`. The greet method has a couple of optional\n * parameters including `subject` and `locale`. The `locale` parameter\n * is not implemented by the driver implementation in the proceeding\n * definition, showing how driver implementations don't always need\n * to support optional features.\n *\n * subject: the person to greet\n * locale: a standard locale string (ex: en_US.UTF-8)\n */\nextension.on('create.interfaces', event => {\n event.createInterface('hello-world', {\n description: 'Provides methods for generating greetings',\n methods: {\n greet: {\n description: 'Returns a greeting',\n parameters: {\n subject: {\n type: 'string',\n optional: true,\n },\n locale: {\n type: 'string',\n optional: true,\n },\n },\n },\n },\n });\n});\n\n/**\n * Here we register an implementation of the `hello-world` driver\n * interface. This implementation is called \"no-frills\" which is\n * the most basic reasonable implementation of the interface. The\n * default return value is \"Hello, World!\", but if subject is\n * provided it will be \"Hello, !\".\n *\n * This implementation can be called from puter.js like this:\n *\n * await puter.call('hello-world', 'no-frills', 'greet', { subject: 'Dave' });\n *\n * If you get an authorization error it's because the user you're\n * logged in as does not have permission to invoke the `no-frills`\n * implementation of `hello-world`. Users must be granted the following\n * permission to access this driver:\n *\n * service:no-frills:ii:hello-world\n *\n * The value of `` can be one of many \"special\" values\n * to demonstrate capabilities of drivers or extensions, including:\n * - `%fail%`: simulate an error response from a driver\n * - `%config%`: return the effective configuration object\n */\nextension.on('create.drivers', event => {\n event.createDriver('hello-world', 'no-frills', {\n greet ({ subject }) {\n return `Hello, ${subject ?? 'World'}!`;\n },\n });\n});\n\nextension.on('create.drivers', event => {\n event.createDriver('hello-world', 'slow-hello', {\n greet: span('slow-hello:greet', async ({ subject }) => {\n await new Promise(rslv => setTimeout(rslv, 1000));\n await span.run(async () => {\n await new Promise(rslv => setTimeout(rslv, 1000));\n });\n await new Promise(rslv => setTimeout(rslv, 1000));\n return `Hello, ${subject ?? 'World'}!`;\n }),\n });\n});\n\nextension.on('create.drivers', event => {\n event.createDriver('hello-world', 'extension-examples', {\n greet ({ subject }) {\n if ( subject === 'fail' ) {\n throw new Error('failing on purpose');\n }\n if ( subject === 'config' ) {\n return JSON.stringify(config ?? null);\n }\n\n const STR_KVSET = 'kv-set:';\n if ( subject.startsWith(STR_KVSET) ) {\n return kv.set({\n key: 'extension-examples-test-key',\n value: subject.slice(STR_KVSET.length),\n });\n }\n if ( subject === 'kv-get' ) {\n return kv.get({\n key: 'extension-examples-test-key',\n });\n }\n\n /* eslint-disable */\n const STR_KVSET2 = 'kv-set-2:';\n if ( subject.startsWith(STR_KVSET2) ) {\n return kv.set(\n 'extension-examples-test-key',\n subject.slice(STR_KVSET2.length),\n );\n }\n if ( subject === 'kv-get-2' ) {\n return kv.get(\n 'extension-examples-test-key',\n );\n }\n /* eslint-enable */\n\n return `Hello, ${subject ?? 'World'}!`;\n },\n });\n});\n\n/**\n * Here we specify that both registered and temporary users are allowed\n * to access the `no-frills` implementation of the `hello-world` driver.\n */\nextension.on('create.permissions', event => {\n event.grant_to_everyone('service:no-frills:ii:hello-world');\n event.grant_to_everyone('service:slow-hello:ii:hello-world');\n event.grant_to_everyone('service:extension-examples:ii:hello-world');\n});\n"
},
{
"path": "extensions/hellodriver/package.json",
"content": "{\n \"name\": \"hellodriver\",\n \"main\": \"hellodriver.js\",\n \"type\": \"module\"\n}"
},
{
"path": "extensions/imports_something.js",
"content": "console.log('importing something...');\nconst { testval } = extension.import('exports_something');\nconsole.log(testval);\n\nextension.on('hello', event => {\n console.log(`received \"hello\" from: ${event.from}`);\n});\n"
},
{
"path": "extensions/metering/config.json",
"content": "{\n \"unlimitedUsage\": false,\n \"unlimitedAllowList\": [\n \"admin\"\n ],\n \"allowedGlobalUsageUsers\": [\n \"nj\",\n \"salazareos\"\n ],\n \"priority\": 10\n}"
},
{
"path": "extensions/metering/controllers/UsageController.ts",
"content": "/* global extension */\nimport type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js';\nimport type { MeteringService } from '@heyputer/backend/src/services/MeteringService/MeteringService.js';\nimport type {\n ExtensionRequest,\n ExtensionResponse,\n} from '../../api.d.ts';\n\nconst { Controller, Get, ExtensionController } = extension.import('extensionController');\n\n@Controller('/metering')\nexport class UsageController extends ExtensionController {\n #meteringService: MeteringService;\n #sqlClient: BaseDatabaseAccessService;\n\n constructor (\n meteringService: MeteringService,\n sqlClient: BaseDatabaseAccessService,\n ) {\n super();\n this.#meteringService = meteringService;\n this.#sqlClient = sqlClient;\n }\n\n @Get('usage', { subdomain: 'api' })\n async getUsage (req: ExtensionRequest, res: ExtensionResponse) {\n const actor = req.actor;\n if ( ! actor ) {\n throw Error('actor not found in context');\n }\n const actorUsagePromise = this.#meteringService.getActorCurrentMonthUsageDetails(actor);\n const actorAllowanceInfoPromise = this.#meteringService.getAllowedUsage(actor);\n\n const [actorUsage, allowanceInfo] = await Promise.all([\n actorUsagePromise,\n actorAllowanceInfoPromise,\n ]);\n res.status(200).json({ ...actorUsage, allowanceInfo });\n return;\n }\n\n @Get('usage/:appIdOrName', { subdomain: 'api' })\n async getUsageByApp (req: ExtensionRequest, res: ExtensionResponse) {\n const actor = req.actor;\n if ( ! actor ) {\n throw Error('actor not found in context');\n }\n const appIdOrName = req.params.appIdOrName;\n if ( ! appIdOrName ) {\n res.status(400).json({ error: 'appId parameter is required' });\n return;\n }\n if ( typeof appIdOrName !== 'string' ) {\n res.status(400).json({ error: 'appId parameter must be a string' });\n return;\n }\n\n let appId = appIdOrName;\n if ( !appIdOrName.startsWith('app-') || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(appIdOrName.split('app-')[1]) ) {\n // Check if the part after 'app-' is a valid UUID (v4)\n const appRows = await this.#sqlClient.read(\n 'SELECT `uid` FROM `apps` WHERE `name` = ? LIMIT 1',\n [appIdOrName],\n );\n if ( appRows.length > 0 ) {\n appId = appRows[0].uid;\n } else {\n res.status(404).json({ error: 'App not found' });\n return;\n }\n } else {\n appId = appIdOrName;\n }\n\n const appUsage =\n await this.#meteringService.getActorCurrentMonthAppUsageDetails(\n actor,\n appId,\n );\n\n res.status(200).json(appUsage);\n return;\n }\n\n @Get('globalUsage', { subdomain: 'api' }, extension.config.allowedGlobalUsageUsers || [])\n async getGlobalUsage (req: ExtensionRequest, res: ExtensionResponse) {\n const actor = req.actor;\n if ( ! actor ) {\n throw Error('actor not found in context');\n }\n\n const globalUsage = await this.#meteringService.getGlobalUsage();\n res.status(200).json(globalUsage);\n return;\n }\n}\n"
},
{
"path": "extensions/metering/eventListeners/subscriptionEvents.ts",
"content": "extension.on('metering:overrideDefaultSubscription', async (event) => {\n // bit of a stub implementation for OSS, technically can be always free if you set this config true\n if ( config.unlimitedUsage ) {\n console.warn('WARNING!!! unlimitedUsage is enabled, this is not recommended for production use');\n event.defaultSubscriptionId = 'unlimited';\n }\n});\n\nextension.on('metering:registerAvailablePolicies', async (event) => {\n // bit of a stub implementation for OSS, technically can be always free if you set this config true\n if ( config.unlimitedUsage || config.unlimitedAllowList?.length ) {\n event.availablePolicies.push({\n id: 'unlimited',\n monthUsageAllowance: 5_000_000 * 1_000_000 * 100, // unless you're like, jeff's, mark's, and elon's illegitamate son, you probably won't hit $5m a month\n monthlyStorageAllowance: 100_000 * 1024 * 1024, // 100MiB but ignored in local dev\n });\n }\n});\n\nextension.on('metering:getUserSubscription', async (event) => {\n const userName = event?.actor?.type?.user?.username;\n if ( config.unlimitedAllowList?.includes(userName) ) {\n event.userSubscriptionId;\n }\n else {\n event.userSubscriptionId = event?.actor?.type?.user?.subscription?.active ? event.actor.type.user.subscription?.tier : undefined;\n }\n // default location for user sub, but can techinically be anywhere else or fetched on request\n});\n"
},
{
"path": "extensions/metering/main.ts",
"content": "import { UsageController } from './controllers/UsageController.js';\nimport './eventListeners/subscriptionEvents.js';\n\nconst meteringService = extension.import('service:meteringService');\nconst sqlClient = extension.import('service:database');\n\nconst controller = new UsageController(meteringService, sqlClient);\ncontroller.registerRoutes();\n"
},
{
"path": "extensions/metering/package.json",
"content": "{\n \"name\": \"@heyputer/extension-metering-service\",\n \"main\": \"main.js\",\n \"type\": \"module\",\n \"scripts\": {\n \"postinstall\": \"tsc --noCheck\"\n },\n \"devDependencies\": {\n \"typescript\": \"^5.9.3\"\n }\n}\n"
},
{
"path": "extensions/metering/tsconfig.json",
"content": "{\n \"compilerOptions\": {\n \"target\": \"ES2024\",\n \"module\": \"nodenext\",\n \"moduleResolution\": \"nodenext\",\n \"strict\": true,\n \"forceConsistentCasingInFileNames\": true,\n \"skipLibCheck\": true,\n \"sourceMap\": true,\n \"noEmitOnError\": true,\n \"noImplicitAny\": false,\n \"allowJs\": true,\n \"checkJs\": false, \n },\n \"include\": [\n \"./**/*.ts\",\n \"./**/*.d.ts\"\n ],\n \"exclude\": [\n \"**/*.test.ts\",\n \"**/*.spec.ts\",\n \"**/test/**\",\n \"**/tests/**\",\n \"node_modules\",\n \"dist\",\n \"*.js\"\n ]\n}\n"
},
{
"path": "extensions/metering/types.ts",
"content": "import '../api.js';\n"
},
{
"path": "extensions/puterfs/PuterFSProvider.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n async #getSourceTreeMaxRelativeDepth (node) {\n await node.fetchEntry();\n if ( ! node.entry.is_dir ) return 0;\n const child_uuids = await this.fsEntryController.fast_get_direct_descendants(await node.get('uid'));\n let max = 0;\n for ( const child_uuid of child_uuids ) {\n const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid));\n const child_relative = 1 + await this.#getSourceTreeMaxRelativeDepth(child_node);\n max = Math.max(max, child_relative);\n }\n return max;\n }\n\n /**\n * Throws if destination depth plus source tree depth would exceed MAX_DIRECTORY_DEPTH.\n * @param {number} destinationPathDepth\n * @param {FSNode} sourceNode\n */\n async #assertDepthLimitForTreeOp (destinationPathDepth, sourceNode) {\n const source_relative = await this.#getSourceTreeMaxRelativeDepth(sourceNode);\n const max_depth = destinationPathDepth + source_relative;\n if ( max_depth > MAX_DIRECTORY_DEPTH ) {\n throw APIError.create('directory_depth_limit_exceeded', null, {\n limit: MAX_DIRECTORY_DEPTH,\n would_be: max_depth,\n });\n }\n }\n // #endregion\n\n // TODO: should this be a static member instead?\n get_capabilities () {\n return new Set([\n capabilities.THUMBNAIL,\n capabilities.UPDATE_THUMBNAIL,\n capabilities.UUID,\n capabilities.OPERATION_TRACE,\n capabilities.READDIR_UUID_MODE,\n capabilities.READDIRSTAT_UUID,\n capabilities.PUTER_SHORTCUT,\n\n capabilities.COPY_TREE,\n capabilities.GET_RECURSIVE_SIZE,\n\n capabilities.READ,\n capabilities.WRITE,\n capabilities.CASE_SENSITIVE,\n capabilities.SYMLINK,\n capabilities.TRASH,\n ]);\n }\n\n // #region PuterOnly\n async update_thumbnail ({ context, node, thumbnail }) {\n const {\n actor: inputActor,\n } = context.values;\n const actor = inputActor ?? Context.get('actor');\n\n context = context ?? Context.get();\n const services = context.get('services');\n\n // TODO: this ACL check should not be here, but there's no LL method yet\n // and it's possible we will never implement the thumbnail\n // capability for any other filesystem type\n\n const svc_acl = services.get('acl');\n if ( ! await svc_acl.check(actor, node, 'write') ) {\n throw await svc_acl.get_safe_acl_error(actor, node, 'write');\n }\n\n const uid = await node.get('uid');\n\n const entryOp = await this.fsEntryController.update(uid, {\n thumbnail,\n });\n\n (async () => {\n await entryOp.awaitDone();\n svc_event.emit('fs.write.file', {\n node,\n context,\n });\n })();\n\n return node;\n }\n\n async puter_shortcut ({ parent, name, user, target }) {\n const user_id = user?.id ?? Context.get('actor')?.type?.user?.id;\n await target.fetchEntry({ thumbnail: true });\n\n const ts = Math.round(Date.now() / 1000);\n const uid = uuidv4();\n\n svc_resource.register({\n uid,\n status: RESOURCE_STATUS_PENDING_CREATE,\n });\n\n const raw_fsentry = {\n is_shortcut: 1,\n shortcut_to: target.mysql_id,\n is_dir: target.entry.is_dir,\n thumbnail: target.entry.thumbnail,\n uuid: uid,\n parent_uid: await parent.get('uid'),\n path: path_.join(await parent.get('path'), name),\n user_id: user_id,\n name,\n created: ts,\n updated: ts,\n modified: ts,\n immutable: false,\n };\n\n const entryOp = await this.fsEntryController.insert(raw_fsentry);\n\n (async () => {\n await entryOp.awaitDone();\n svc_resource.free(uid);\n })();\n\n const node = await svc_fs.node(new NodeUIDSelector(uid));\n\n svc_event.emit('fs.create.shortcut', {\n node,\n context: Context.get(),\n });\n\n return node;\n }\n // #endregion\n\n // #region Optimization\n /**\n * The readdirstat_uuid operation is only available for filesystem\n * immplementations with READDIR_UUID_MODE enabled. This implements\n * an optimized readdir operation when the UUID is already known.\n * @param {*} param0\n */\n async readdirstat_uuid ({\n uuid,\n options = {},\n }) {\n const entries = await this.fsEntryController.get_descendants_full(uuid, options);\n const nodes = Promise.all(Array.prototype.map.call(entries, raw_entry => {\n const node = svc_fs.node(new NodeRawEntrySelector(raw_entry, {\n found_thumbnail: options.thumbnail,\n }));\n node.found = true; // TODO: how is it possible for this to be false?\n return node;\n }));\n return nodes;\n };\n // #endregion\n\n // #region Standard FS\n\n /**\n * Check if a given node exists.\n *\n * @param {Object} param\n * @param {NodeSelector} param.selector - The selector used for checking.\n * @returns {Promise} - True if the node exists, false otherwise.\n */\n async quick_check ({\n selector,\n }) {\n // shortcut: has full path\n if ( selector?.path ) {\n const entry = await this.fsEntryController.findByPath(selector.path);\n return Boolean(entry);\n }\n\n // shortcut: has uid\n if ( selector?.uid ) {\n const entry = await this.fsEntryController.findByUID(selector.uid);\n return Boolean(entry);\n }\n\n // shortcut: parent uid + child name\n if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeUIDSelector ) {\n return await this.fsEntryController.nameExistsUnderParent(selector.parent.uid,\n selector.name);\n }\n\n // shortcut: parent id + child name\n if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeInternalIDSelector ) {\n return await this.fsEntryController.nameExistsUnderParentID(selector.parent.id,\n selector.name);\n }\n\n return false;\n }\n\n async unlink ({ context, node, options = {} }) {\n if ( await node.get('type') === TYPE_DIRECTORY ) {\n throw new APIError(409, 'Cannot unlink a directory.');\n }\n\n await this.#rmnode({ context, node, options });\n }\n\n async rmdir ({ context, node, options = {} }) {\n if ( await node.get('type') !== TYPE_DIRECTORY ) {\n throw new APIError(409, 'Cannot rmdir a file.');\n }\n\n if ( await node.get('immutable') ) {\n throw APIError.create('immutable');\n }\n\n const children = await this.fsEntryController.fast_get_direct_descendants(await node.get('uid'));\n\n if ( children.length > 0 && !options.ignore_not_empty ) {\n throw APIError.create('not_empty');\n }\n\n await this.#rmnode({ context, node, options });\n }\n\n /**\n * Create a new directory.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNode} param.parent\n * @param {string} param.name\n * @param {boolean} param.immutable\n * @returns {Promise}\n */\n async mkdir ({ context, parent, name, immutable }) {\n const { actor, thumbnail } = context.values;\n\n const ts = Math.round(Date.now() / 1000);\n const uid = uuidv4();\n\n const existing = await svc_fs.node(new NodeChildSelector(parent.selector, name));\n\n if ( await existing.exists() ) {\n throw APIError.create('item_with_same_name_exists', null, {\n entry_name: name,\n });\n }\n\n if ( ! await parent.exists() ) {\n throw APIError.create('subject_does_not_exist');\n }\n\n const new_path = path_.join(await parent.get('path'), name);\n if ( this.#pathDepth(new_path) > MAX_DIRECTORY_DEPTH ) {\n throw APIError.create('directory_depth_limit_exceeded', null, {\n limit: MAX_DIRECTORY_DEPTH,\n would_be: this.#pathDepth(new_path),\n });\n }\n\n svc_resource.register({\n uid,\n status: RESOURCE_STATUS_PENDING_CREATE,\n });\n\n const raw_fsentry = {\n is_dir: 1,\n uuid: uid,\n parent_uid: await parent.get('uid'),\n path: path_.join(await parent.get('path'), name),\n user_id: actor.type.user.id,\n name,\n created: ts,\n accessed: ts,\n modified: ts,\n immutable: immutable ?? false,\n ...(thumbnail ? {\n thumbnail: thumbnail,\n } : {}),\n };\n\n console.log('raw fsentry', raw_fsentry);\n const entryOp = await this.fsEntryController.insert(raw_fsentry);\n\n await entryOp.awaitDone();\n svc_resource.free(uid);\n\n const node = await svc_fs.node(new NodeUIDSelector(uid));\n\n svc_event.emit('fs.create.directory', {\n node,\n context: Context.get(),\n });\n\n return node;\n }\n\n async read ({ context, node, version_id, range }) {\n const svc_mountpoint = context.get('services').get('mountpoint');\n const storage = svc_mountpoint.get_storage(this.constructor.name);\n const location = await node.get('s3:location') ?? {};\n const stream = (await storage.create_read_stream(await node.get('uid'), {\n // TODO: fs:decouple-s3\n bucket: location.bucket,\n bucket_region: location.bucket_region,\n version_id,\n key: location.key,\n memory_file: node.entry,\n ...(range ? { range } : {}),\n }));\n return stream;\n }\n\n async stat ({\n selector,\n options,\n controls,\n node,\n }) {\n // For Puter FS nodes, we assume we will obtain all properties from\n // fsEntryController, except for 'thumbnail' unless it's\n // explicitly requested.\n\n if ( options.tracer == null ) {\n options.tracer = getTracer();\n }\n\n if ( options.op ) {\n options.trace_options = {\n parent: options.op.span,\n };\n }\n\n let entry;\n\n // stat doesn't work with RawEntrySelector\n if ( selector instanceof NodeRawEntrySelector ) {\n selector = new NodeUIDSelector(node.uid);\n }\n\n await new Promise (rslv => {\n const detachables = new MultiDetachable();\n\n const callback = (_resolver) => {\n detachables.as(TDetachable).detach();\n rslv();\n };\n\n // either the resource is free\n {\n // no detachale because waitForResource returns a\n // Promise that will be resolved when the resource\n // is free no matter what, and then it will be\n // garbage collected.\n svc_resource.waitForResource(selector).then(callback.bind(null, 'resourceService'));\n }\n\n // or pending information about the resource\n // becomes available\n {\n // detachable is needed here because waitForEntry keeps\n // a map of listeners in memory, and this event may\n // never occur. If this never occurs, waitForResource\n // is guaranteed to resolve eventually, and then this\n // detachable will be detached by `callback` so the\n // listener can be garbage collected.\n const det = this.fsEntryController.waitForEntry(node, callback.bind(null, 'fsEntryService'));\n if ( det ) detachables.add(det);\n }\n });\n\n const maybe_uid = node.uid;\n if ( svc_resource.getResourceInfo(maybe_uid) ) {\n entry = await this.fsEntryController.get(maybe_uid, options);\n controls.log.debug('got an entry from the future');\n } else {\n entry = await this.fsEntryController.find(selector, options);\n }\n\n if ( ! entry ) {\n if ( this.log_fsentriesNotFound ) {\n controls.log.warn(`entry not found: ${selector.describe(true)}`);\n }\n }\n\n if ( entry === null || typeof entry !== 'object' ) {\n return null;\n }\n\n if ( entry.id ) {\n controls.provide_selector(new NodeInternalIDSelector('mysql', entry.id, {\n source: 'FSNodeContext optimization',\n }));\n }\n\n return entry;\n }\n\n async copy_tree ({ context, source, parent, target_name }) {\n // Context\n const actor = (context ?? Context).get('actor');\n const user = actor.type.user;\n\n const tracer = getTracer();\n const uuid = uuidv4();\n const timestamp = Math.round(Date.now() / 1000);\n await parent.fetchEntry();\n await source.fetchEntry({ thumbnail: true });\n\n const destination_path = path_.join(await parent.get('path'), target_name);\n await this.#assertDepthLimitForTreeOp(this.#pathDepth(destination_path), source);\n\n // New filesystem entry\n const raw_fsentry = {\n uuid,\n is_dir: source.entry.is_dir,\n ...(source.entry.is_shortcut ? {\n is_shortcut: source.entry.is_shortcut,\n shortcut_to: source.entry.shortcut_to,\n } : {}),\n parent_uid: parent.uid,\n name: target_name,\n created: timestamp,\n modified: timestamp,\n\n path: path_.join(await parent.get('path'), target_name),\n\n // if property exists but the value is undefined,\n // it will still be included in the INSERT, causing\n // an error\n ...(source.entry.thumbnail ?\n { thumbnail: source.entry.thumbnail } : {}),\n\n user_id: user.id,\n };\n\n svc_event.emit('fs.pending.file', {\n fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),\n context: context,\n });\n\n if ( await source.get('has-s3') ) {\n Object.assign(raw_fsentry, {\n size: source.entry.size,\n associated_app_id: source.entry.associated_app_id,\n bucket: source.entry.bucket,\n bucket_region: source.entry.bucket_region,\n });\n\n await tracer.startActiveSpan('fs:cp:storage-copy', async span => {\n let progress_tracker = new UploadProgressTracker();\n\n svc_event.emit('fs.storage.progress.copy', {\n upload_tracker: progress_tracker,\n context,\n meta: {\n item_uid: uuid,\n item_path: raw_fsentry.path,\n },\n });\n\n // const storage = new PuterS3StorageStrategy({ services: svc });\n const storage = context.get('storage');\n const state_copy = storage.create_copy();\n await state_copy.run({\n src_node: source,\n dst_storage: {\n key: uuid,\n bucket: raw_fsentry.bucket,\n bucket_region: raw_fsentry.bucket_region,\n },\n storage_api: { progress_tracker },\n });\n\n span.end();\n });\n }\n\n {\n await svc_size.add_node_size(undefined, source, user);\n }\n\n svc_resource.register({\n uid: uuid,\n status: RESOURCE_STATUS_PENDING_CREATE,\n });\n\n const entryOp = await this.fsEntryController.insert(raw_fsentry);\n\n let node;\n\n const tasks = new ParallelTasks({ tracer, max: 4 });\n await context.arun('fs:cp:parallel-portion', async () => {\n // Add child copy tasks if this is a directory\n if ( source.entry.is_dir ) {\n const children = await this.fsEntryController.fast_get_direct_descendants(source.uid);\n for ( const child_uuid of children ) {\n tasks.add('fs:cp:copy-child', async () => {\n const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid));\n const child_name = await child_node.get('name');\n\n await this.copy_tree({\n context,\n source: await svc_fs.node(new NodeUIDSelector(child_uuid)),\n parent: await svc_fs.node(new NodeUIDSelector(uuid)),\n target_name: child_name,\n });\n });\n }\n }\n\n // Add task to await entry\n tasks.add('fs:cp:entry-op', async () => {\n await entryOp.awaitDone();\n svc_resource.free(uuid);\n const copy_fsNode = await svc_fs.node(new NodeUIDSelector(uuid));\n copy_fsNode.entry = raw_fsentry;\n copy_fsNode.found = true;\n copy_fsNode.path = raw_fsentry.path;\n\n node = copy_fsNode;\n\n svc_event.emit('fs.create.file', {\n node,\n context,\n });\n }, { force: true });\n\n await tasks.awaitAll();\n });\n\n node = node || await svc_fs.node(new NodeUIDSelector(uuid));\n\n // TODO: What event do we emit? How do we know if we're overwriting?\n return node;\n }\n\n async move ({ context, node, new_parent, new_name, metadata }) {\n const old_path = await node.get('path');\n const new_path = path_.join(await new_parent.get('path'), new_name);\n\n await this.#assertDepthLimitForTreeOp(this.#pathDepth(new_path), node);\n\n const op_update = await this.fsEntryController.update(node.uid, {\n ...(\n await node.get('parent_uid') !== await new_parent.get('uid')\n ? { parent_uid: await new_parent.get('uid') }\n : {}\n ),\n path: new_path,\n name: new_name,\n ...(metadata ? { metadata } : {}),\n });\n\n node.entry.name = new_name;\n node.entry.path = new_path;\n\n // NOTE: this is a safeguard passed to update_child_paths to isolate\n // changes to the owner's directory tree, ut this may need to be\n // removed in the future.\n const user_id = await node.get('user_id');\n\n await op_update.awaitDone();\n\n await svc_fs.update_child_paths(old_path, node.entry.path, user_id);\n\n const promises = [];\n promises.push(svc_event.emit('fs.move.file', {\n context,\n moved: node,\n old_path,\n }));\n promises.push(svc_event.emit('fs.rename', {\n uid: await node.get('uid'),\n new_name,\n }));\n\n return node;\n }\n\n async readdir ({ node }) {\n const uuid = await node.get('uid');\n const child_uuids = await this.fsEntryController.fast_get_direct_descendants(uuid);\n return child_uuids;\n }\n\n async directory_has_name ({ parent, name }) {\n const uid = await parent.get('uid');\n\n let check_dupe = await db.read(\n 'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1',\n [uid, name],\n );\n\n return !!check_dupe[0];\n }\n\n /**\n * Write a new file to the filesystem. Throws an error if the destination\n * already exists.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNode} param.parent: The parent directory of the file.\n * @param {string} param.name: The name of the file.\n * @param {File} param.file: The file to write.\n * @returns {Promise}\n */\n async write_new ({ context, parent, name, file }) {\n console.log('calling write new');\n const {\n tmp, fsentry_tmp, message, actor: inputActor, app_id,\n } = context.values;\n const actor = inputActor ?? Context.get('actor');\n\n const uid = uuidv4();\n\n // determine bucket region\n let bucket_region = global_config.s3_region ?? global_config.region;\n let bucket = global_config.s3_bucket;\n\n if ( ! await svc_acl.check(actor, parent, 'write') ) {\n throw await svc_acl.get_safe_acl_error(actor, parent, 'write');\n }\n\n const storage_resp = await this.#storage_upload({\n uuid: uid,\n bucket,\n bucket_region,\n file,\n tmp: {\n ...tmp,\n path: path_.join(await parent.get('path'), name),\n },\n });\n\n fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;\n delete fsentry_tmp.thumbnail_promise;\n\n const timestamp = Math.round(Date.now() / 1000);\n const raw_fsentry = {\n uuid: uid,\n is_dir: 0,\n user_id: actor.type.user.id,\n created: timestamp,\n accessed: timestamp,\n modified: timestamp,\n parent_uid: await parent.get('uid'),\n name,\n size: file.size,\n path: path_.join(await parent.get('path'), name),\n ...fsentry_tmp,\n bucket_region,\n bucket,\n associated_app_id: app_id ?? null,\n };\n\n svc_event.emit('fs.pending.file', {\n fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),\n context,\n });\n\n svc_resource.register({\n uid,\n status: RESOURCE_STATUS_PENDING_CREATE,\n });\n\n const filesize = file.size;\n svc_size.change_usage(actor.type.user.id, filesize);\n\n // Meter ingress\n const ownerId = await parent.get('user_id');\n const ownerActor = new Actor({\n type: new UserActorType({\n user: await get_user({ id: ownerId }),\n }),\n });\n\n svc_metering.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize);\n\n const entryOp = await this.fsEntryController.insert(raw_fsentry);\n\n (async () => {\n await entryOp.awaitDone();\n svc_resource.free(uid);\n\n const new_item_node = await svc_fs.node(new NodeUIDSelector(uid));\n const new_item = await new_item_node.get('entry');\n const store_version_id = storage_resp.VersionId;\n if ( store_version_id ) {\n // insert version into db\n db.write('INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)',\n [\n actor.type.user.id,\n new_item.id,\n new_item.uuid,\n store_version_id,\n message ?? null,\n timestamp,\n ]);\n }\n })();\n\n const node = await svc_fs.node(new NodeUIDSelector(uid));\n\n svc_event.emit('fs.create.file', {\n node,\n context,\n });\n\n return node;\n }\n\n /**\n * Overwrite an existing file. Throws an error if the destination does not\n * exist.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNodeContext} param.node: The node to write to.\n * @param {File} param.file: The file to write.\n * @returns {Promise}\n */\n async write_overwrite ({ context, node, file }) {\n const {\n tmp, fsentry_tmp, message, actor: inputActor,\n } = context.values;\n const actor = inputActor ?? Context.get('actor');\n\n if ( ! await svc_acl.check(actor, node, 'write') ) {\n throw await svc_acl.get_safe_acl_error(actor, node, 'write');\n }\n\n const uid = await node.get('uid');\n\n const bucket_region = node.entry.bucket_region;\n const bucket = node.entry.bucket;\n\n const state_upload = await this.#storage_upload({\n uuid: node.entry.uuid,\n bucket,\n bucket_region,\n file,\n tmp: {\n ...tmp,\n path: await node.get('path'),\n },\n });\n\n if ( fsentry_tmp?.thumbnail_promise ) {\n fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;\n delete fsentry_tmp.thumbnail_promise;\n }\n\n const ts = Math.round(Date.now() / 1000);\n const raw_fsentry_delta = {\n modified: ts,\n accessed: ts,\n size: file.size,\n ...fsentry_tmp,\n };\n\n svc_resource.register({\n uid,\n status: RESOURCE_STATUS_PENDING_CREATE,\n });\n\n const filesize = file.size;\n svc_size.change_usage(actor.type.user.id, filesize);\n\n // Meter ingress\n const ownerId = await node.get('user_id');\n const ownerActor = new Actor({\n type: new UserActorType({\n user: await get_user({ id: ownerId }),\n }),\n });\n svc_metering.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize);\n\n const entryOp = await this.fsEntryController.update(uid, raw_fsentry_delta);\n\n // depends on fsentry, does not depend on S3\n const entryOpPromise = (async () => {\n await entryOp.awaitDone();\n svc_resource.free(uid);\n })();\n\n (async () => {\n await entryOpPromise;\n svc_event.emit('fs.write.file', {\n node,\n context,\n });\n })();\n\n // TODO (xiaochen): determine if this can be removed, post_insert handler need\n // to skip events from other servers (why? 1. current write logic is inside\n // the local server 2. broadcast system conduct \"fire-and-forget\" behavior)\n state_upload.post_insert({\n db, user: actor.type.user, node, uid, message, ts,\n });\n\n return node;\n }\n\n async get_recursive_size ({ node }) {\n const uuid = await node.get('uid');\n const cte_query = `\n WITH RECURSIVE descendant_cte AS (\n SELECT uuid, parent_uid, size\n FROM fsentries\n WHERE parent_uid = ?\n\n UNION ALL\n\n SELECT f.uuid, f.parent_uid, f.size\n FROM fsentries f\n INNER JOIN descendant_cte d\n ON f.parent_uid = d.uuid\n )\n SELECT SUM(size) AS total_size FROM descendant_cte\n `;\n const rows = await db.read(cte_query, [uuid]);\n return rows[0].total_size;\n }\n\n // #endregion\n\n // #region internal\n\n /**\n * @param {Object} param\n * @param {File} param.file: The file to write.\n * @returns\n */\n async #storage_upload ({\n uuid,\n bucket,\n bucket_region,\n file,\n tmp,\n }) {\n const storage = svc_mountpoint.get_storage(this.constructor.name);\n\n bucket ??= global_config.s3_bucket;\n bucket_region ??= global_config.s3_region ?? global_config.region;\n\n let upload_tracker = new UploadProgressTracker();\n\n svc_event.emit('fs.storage.upload-progress', {\n upload_tracker,\n context: Context.get(),\n meta: {\n item_uid: uuid,\n item_path: tmp.path,\n },\n });\n\n if ( ! file.buffer ) {\n let stream = file.stream;\n let alarm_timeout = null;\n stream = stuck_detector_stream(stream, {\n timeout: STUCK_STATUS_TIMEOUT,\n on_stuck: () => {\n console.warn('Upload stream stuck might be stuck', {\n bucket_region,\n bucket,\n uuid,\n });\n alarm_timeout = setTimeout(() => {\n extension.errors.report('fs.write.s3-upload', {\n message: 'Upload stream stuck for too long',\n alarm: true,\n extra: {\n bucket_region,\n bucket,\n uuid,\n },\n });\n }, STUCK_ALARM_TIMEOUT);\n },\n on_unstuck: () => {\n clearTimeout(alarm_timeout);\n },\n });\n file = { ...file, stream };\n }\n\n let hashPromise;\n if ( file.buffer ) {\n const hash = crypto.createHash('sha256');\n hash.update(file.buffer);\n hashPromise = Promise.resolve(hash.digest('hex'));\n } else {\n const hs = hashing_stream(file.stream);\n file.stream = hs.stream;\n hashPromise = hs.hashPromise;\n }\n\n hashPromise.then(hash => {\n svc_event.emit('outer.fs.write-hash', {\n hash, uuid,\n });\n });\n\n const state_upload = storage.create_upload();\n\n try {\n await this.storageController.upload({\n uid: uuid,\n file,\n storage_meta: { bucket, bucket_region },\n storage_api: { progress_tracker: upload_tracker },\n });\n } catch (e) {\n extension.errors.report('fs.write.storage-upload', {\n source: e || new Error('unknown'),\n trace: true,\n alarm: true,\n extra: {\n bucket_region,\n bucket,\n uuid,\n },\n });\n throw APIError.create('upload_failed');\n }\n\n return state_upload;\n }\n\n async #rmnode ({ node, options }) {\n // Services\n if ( !options.override_immutable && await node.get('immutable') ) {\n throw new APIError(403, 'File is immutable.');\n }\n\n const userId = await node.get('user_id');\n const fileSize = await node.get('size');\n svc_size.change_usage(userId,\n -1 * fileSize);\n\n const ownerActor = new Actor({\n type: new UserActorType({\n user: await get_user({ id: userId }),\n }),\n });\n\n svc_metering.incrementUsage(ownerActor, 'filesystem:delete:bytes', fileSize);\n\n const tracer = getTracer();\n const tasks = new ParallelTasks({ tracer, max: 4 });\n\n tasks.add('remove-fsentry', async () => {\n await this.fsEntryController.delete(await node.get('uid'));\n });\n\n if ( await node.get('has-s3') ) {\n tasks.add('remove-from-s3', async () => {\n // const storage = new PuterS3StorageStrategy({ services: svc });\n const storage = Context.get('storage');\n const state_delete = storage.create_delete();\n await state_delete.run({\n node: node,\n });\n });\n }\n\n await tasks.awaitAll();\n }\n // #endregion\n}\n"
},
{
"path": "extensions/puterfs/fsentries/BaseOperation.js",
"content": "import { TeePromise } from 'teepromise';\n\nexport default class BaseOperation {\n static STATUS_PENDING = {};\n static STATUS_RUNNING = {};\n static STATUS_DONE = {};\n\n /** @type {PromiseLike & { resolve: () => void }} */\n #donePromise;\n\n constructor () {\n this.status_ = this.constructor.STATUS_PENDING;\n this.#donePromise = new TeePromise();\n }\n get status () {\n return this.status_;\n }\n set status (status) {\n this.status_ = status;\n if ( status === this.constructor.STATUS_DONE ) {\n this.#donePromise.resolve();\n }\n }\n async awaitDone () {\n await this.#donePromise;\n }\n async onComplete (fn) {\n await this.#donePromise;\n fn();\n }\n}\n"
},
{
"path": "extensions/puterfs/fsentries/Delete.js",
"content": "import BaseOperation from './BaseOperation.js';\n\nexport default class extends BaseOperation {\n constructor (uuid) {\n super();\n this.uuid = uuid;\n }\n\n getStatement () {\n const statement = 'DELETE FROM fsentries WHERE uuid = ? LIMIT 1';\n const values = [this.uuid];\n return { statement, values };\n }\n\n apply (answer) {\n answer.entry = null;\n }\n}\n"
},
{
"path": "extensions/puterfs/fsentries/FSEntryController.js",
"content": "import { TeePromise } from 'teepromise';\nimport BaseOperation from './BaseOperation.js';\nimport Delete from './Delete.js';\nimport Insert from './Insert.js';\nimport Update from './Update.js';\n\nconst { db } = extension.import('data');\nconst svc_params = extension.import('service:params');\n\nconst { PuterPath } = extension.import('fs');\n\nconst {\n RootNodeSelector,\n NodeChildSelector,\n NodeUIDSelector,\n NodePathSelector,\n NodeInternalIDSelector,\n} = extension.import('core').fs.selectors;\n\nexport default class FSEntryController {\n static CONCERN = 'filesystem';\n\n static STATUS_READY = {};\n static STATUS_RUNNING_JOB = {};\n\n constructor () {\n this.status = FSEntryController.STATUS_READY;\n\n this.currentState = {\n queue: [],\n updating_uuids: {},\n };\n this.deferredState = {\n queue: [],\n updating_uuids: {},\n };\n\n this.entryListeners_ = {};\n\n this.mkPromiseForQueueSize_();\n\n // this list of properties is for read operations\n // (originally in FSEntryFetcher)\n this.defaultProperties = [\n 'id',\n 'associated_app_id',\n 'uuid',\n 'public_token',\n 'bucket',\n 'bucket_region',\n 'file_request_token',\n 'user_id',\n 'parent_uid',\n 'is_dir',\n 'is_public',\n 'is_shortcut',\n 'is_symlink',\n 'symlink_path',\n 'shortcut_to',\n 'sort_by',\n 'sort_order',\n 'immutable',\n 'name',\n 'metadata',\n 'modified',\n 'created',\n 'accessed',\n 'size',\n 'layout',\n 'path',\n ];\n\n this.subdomainProperties = [\n 'uuid',\n 'subdomain',\n ];\n }\n\n init () {\n svc_params.createParameters('fsentry-service', [\n {\n id: 'max_queue',\n description: 'Maximum queue size',\n default: 50,\n },\n ], this);\n\n }\n\n mkPromiseForQueueSize_ () {\n this.queueSizePromise = new Promise((resolve, reject) => {\n this.queueSizeResolve = resolve;\n });\n }\n\n // #region write operations\n async insert (entry) {\n const op = new Insert(entry);\n await this.enqueue_(op);\n return op;\n }\n\n async update (uuid, entry) {\n const op = new Update(uuid, entry);\n await this.enqueue_(op);\n return op;\n }\n\n async delete (uuid) {\n const op = new Delete(uuid);\n await this.enqueue_(op);\n return op;\n }\n // #endregion\n\n // #region read operations\n async fast_get_descendants (uuid) {\n return (await db.read(`\n WITH RECURSIVE descendant_cte AS (\n SELECT uuid, parent_uid\n FROM fsentries\n WHERE parent_uid = ?\n\n UNION ALL\n\n SELECT f.uuid, f.parent_uid\n FROM fsentries f\n INNER JOIN descendant_cte d ON f.parent_uid = d.uuid\n )\n SELECT uuid FROM descendant_cte\n `, [uuid])).map(x => x.uuid);\n }\n\n async fast_get_direct_descendants (uuid) {\n return (uuid === PuterPath.NULL_UUID\n ? await db.read('SELECT uuid FROM fsentries WHERE parent_uid IS NULL')\n : await db.read(\n 'SELECT uuid FROM fsentries WHERE parent_uid = ?',\n [uuid],\n )).map(x => x.uuid);\n }\n\n waitForEntry (node, callback) {\n // *** uncomment to debug slow waits ***\n // console.log('ATTEMPT TO WAIT FOR', selector.describe())\n let selector = node.get_selector_of_type(NodeUIDSelector);\n if ( selector === null ) {\n // console.log(new Error('========'));\n return;\n }\n\n const entry_already_enqueued =\n Object.prototype.hasOwnProperty.call(this.currentState.updating_uuids, selector.value) ||\n Object.prototype.hasOwnProperty.call(this.deferredState.updating_uuids, selector.value) ;\n\n if ( entry_already_enqueued ) {\n callback();\n return;\n }\n\n const k = `uid:${selector.value}`;\n if ( ! Object.prototype.hasOwnProperty.call(this.entryListeners_, k) ) {\n this.entryListeners_[k] = [];\n }\n\n const det = {\n detach: () => {\n const i = this.entryListeners_[k].indexOf(callback);\n if ( i === -1 ) return;\n this.entryListeners_[k].splice(i, 1);\n if ( this.entryListeners_[k].length === 0 ) {\n delete this.entryListeners_[k];\n }\n },\n };\n\n this.entryListeners_[k].push(callback);\n\n return det;\n }\n\n async get (uuid, fetch_entry_options) {\n const answer = {};\n for ( const op of this.currentState.queue ) {\n if ( op.uuid != uuid ) continue;\n op.apply(answer);\n }\n for ( const op of this.deferredState.queue ) {\n if ( op.uuid != uuid ) continue;\n op.apply(answer);\n op.apply(answer);\n }\n if ( answer.is_diff ) {\n const base_entry = await this.find(\n new NodeUIDSelector(uuid),\n fetch_entry_options,\n );\n answer.entry = { ...base_entry, ...answer.entry };\n }\n return answer.entry;\n }\n\n /**\n * Returns UUIDs of child fsentries under the specified\n * parent fsentry\n * @param {string} uuid - UUID of parent fsentry\n * @returns fsentry[]\n */\n async get_descendants (uuid) {\n return uuid === PuterPath.NULL_UUID\n ? await db.read(\n 'SELECT uuid FROM fsentries WHERE parent_uid IS NULL',\n [uuid],\n )\n : await db.read(\n 'SELECT uuid FROM fsentries WHERE parent_uid = ?',\n [uuid],\n )\n ;\n }\n\n /**\n * Returns full fsentry nodes for entries under the specified\n * parent fsentry\n * @param {string} uuid - UUID of parent fsentry\n * @returns fsentry[]\n */\n async get_descendants_full (uuid, fetch_entry_options) {\n const { thumbnail } = fetch_entry_options;\n const columns = `${\n [\n ...this.defaultProperties.map(v => `f.${v}`),\n ...this.subdomainProperties\n .map(v => `s.${v} AS subdomain_${v}`),\n ].join(', ')\n }${thumbnail ? ', thumbnail' : ''}`;\n const results_with_dupes = uuid === PuterPath.NULL_UUID\n ? await db.read(\n `SELECT ${columns} FROM fsentries WHERE parent_uid IS NULL`,\n [uuid],\n )\n : await db.read(\n `SELECT ${columns} FROM fsentries AS f ` +\n 'LEFT JOIN subdomains AS s ON f.id=s.root_dir_id ' +\n 'WHERE parent_uid = ? ORDER BY f.id',\n [uuid],\n )\n ;\n\n const byId = new Map();\n for ( const row of results_with_dupes ) {\n const id = row.id;\n let entry = byId.get(id);\n if ( ! entry ) {\n entry = { ...row };\n if ( thumbnail ) entry.thumbnail = row.thumbnail;\n entry.subdomains = [];\n byId.set(id, entry);\n }\n if ( row.subdomain_uuid != null ) {\n entry.subdomains.push({\n uuid: row.subdomain_uuid,\n subdomain: row.subdomain_subdomain,\n });\n }\n }\n return Array.from(byId.values());\n }\n\n async get_recursive_size (uuid) {\n const cte_query = `\n WITH RECURSIVE descendant_cte AS (\n SELECT uuid, parent_uid, size\n FROM fsentries\n WHERE parent_uid = ?\n\n UNION ALL\n\n SELECT f.uuid, f.parent_uid, f.size\n FROM fsentries f\n INNER JOIN descendant_cte d\n ON f.parent_uid = d.uuid\n )\n SELECT SUM(size) AS total_size FROM descendant_cte\n `;\n const rows = await db.read(cte_query, [uuid]);\n return rows[0].total_size;\n }\n\n /**\n * Finds a filesystem entry using the provided selector.\n * @param {Object} selector - The selector object specifying how to find the entry\n * @param {Object} fetch_entry_options - Options for fetching the entry\n * @returns {Promise} The filesystem entry or null if not found\n */\n async find (selector, fetch_entry_options) {\n if ( selector instanceof RootNodeSelector ) {\n return selector.entry;\n }\n if ( selector instanceof NodePathSelector ) {\n return await this.findByPath(selector.value, fetch_entry_options);\n }\n if ( selector instanceof NodeUIDSelector ) {\n return await this.findByUID(selector.value, fetch_entry_options);\n }\n if ( selector instanceof NodeInternalIDSelector ) {\n return await this.findByID(selector.id, fetch_entry_options);\n }\n if ( selector instanceof NodeChildSelector ) {\n let id;\n\n if ( selector.parent instanceof RootNodeSelector ) {\n id = await this.findNameInRoot(selector.name);\n } else {\n const parentEntry = await this.find(selector.parent);\n if ( ! parentEntry ) return null;\n id = await this.findNameInParent(parentEntry.uuid, selector.name);\n }\n\n if ( id === undefined ) return null;\n if ( typeof id !== 'number' ) {\n throw new Error(\n 'unexpected type for id value',\n typeof id,\n id,\n );\n }\n return this.find(new NodeInternalIDSelector('mysql', id));\n }\n }\n\n /**\n * Finds a filesystem entry by its UUID.\n * @param {string} uuid - The UUID of the entry to find\n * @param {Object} fetch_entry_options - Options including thumbnail flag\n * @returns {Promise} The filesystem entry or undefined if not found\n */\n async findByUID (uuid, fetch_entry_options = {}) {\n const { thumbnail } = fetch_entry_options;\n\n let fsentry = await db.tryHardRead(\n `SELECT ${\n this.defaultProperties.join(', ')\n }${thumbnail ? ', thumbnail' : ''\n } FROM fsentries WHERE uuid = ? LIMIT 1`,\n [uuid],\n );\n\n return fsentry[0];\n }\n\n /**\n * Finds a filesystem entry by its internal database ID.\n * @param {number} id - The internal ID of the entry to find\n * @param {Object} fetch_entry_options - Options including thumbnail flag\n * @returns {Promise} The filesystem entry or undefined if not found\n */\n async findByID (id, fetch_entry_options = {}) {\n const { thumbnail } = fetch_entry_options;\n\n let fsentry = await db.tryHardRead(\n `SELECT ${\n this.defaultProperties.join(', ')\n }${thumbnail ? ', thumbnail' : ''\n } FROM fsentries WHERE id = ? LIMIT 1`,\n [id],\n );\n\n return fsentry[0];\n }\n\n /**\n * Finds a filesystem entry by its full path.\n * @param {string} path - The full path of the entry to find\n * @param {Object} fetch_entry_options - Options including thumbnail flag and tracer\n * @returns {Promise} The filesystem entry or false if not found\n */\n async findByPath (path, fetch_entry_options = {}) {\n const { thumbnail } = fetch_entry_options;\n\n if ( path === '/' ) {\n return this.find(new RootNodeSelector());\n }\n\n const parts = path.split('/').filter(path => path !== '');\n if ( parts.length === 0 ) {\n // TODO: invalid path; this should be an error\n return false;\n }\n\n // TODO: use a closure table for more efficient path resolving\n let parent_uid = null;\n let result;\n\n const resultColsSql = this.defaultProperties.join(', ') +\n (thumbnail ? ', thumbnail' : '');\n\n result = await db.read(\n `SELECT ${ resultColsSql\n } FROM fsentries WHERE path=? LIMIT 1`,\n [path],\n );\n\n // using knex instead\n\n if ( result[0] ) return result[0];\n\n const loop = async () => {\n for ( let i = 0 ; i < parts.length ; i++ ) {\n const part = parts[i];\n const isLast = i == parts.length - 1;\n const colsSql = isLast ? resultColsSql : 'uuid';\n if ( parent_uid === null ) {\n result = await db.read(\n `SELECT ${ colsSql\n } FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1`,\n [part],\n );\n } else {\n result = await db.read(\n `SELECT ${ colsSql\n } FROM fsentries WHERE parent_uid=? AND name=? LIMIT 1`,\n [parent_uid, part],\n );\n }\n\n if ( ! result[0] ) return false;\n parent_uid = result[0].uuid;\n }\n };\n\n if ( fetch_entry_options.tracer ) {\n const tracer = fetch_entry_options.tracer;\n const options = fetch_entry_options.trace_options;\n await tracer.startActiveSpan(\n 'fs:sql:findByPath',\n ...(options ? [options] : []),\n async span => {\n await loop();\n span.end();\n },\n );\n } else {\n await loop();\n }\n\n return result[0];\n }\n\n /**\n * Finds the ID of a child entry with the given name in the root directory.\n * @param {string} name - The name of the child entry to find\n * @returns {Promise} The ID of the child entry or undefined if not found\n */\n async findNameInRoot (name) {\n let child_id = await db.read(\n 'SELECT `id` FROM `fsentries` WHERE `parent_uid` IS NULL AND name = ? LIMIT 1',\n [name],\n );\n return child_id[0]?.id;\n }\n\n /**\n * Finds the ID of a child entry with the given name under a specific parent.\n * @param {string} parent_uid - The UUID of the parent directory\n * @param {string} name - The name of the child entry to find\n * @returns {Promise} The ID of the child entry or undefined if not found\n */\n async findNameInParent (parent_uid, name) {\n let child_id = await db.read(\n 'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1',\n [parent_uid, name],\n );\n return child_id[0]?.id;\n }\n\n /**\n * Checks if an entry with the given name exists under a specific parent.\n * @param {string} parent_uid - The UUID of the parent directory\n * @param {string} name - The name to check for\n * @returns {Promise} True if the name exists under the parent, false otherwise\n */\n async nameExistsUnderParent (parent_uid, name) {\n let check_dupe = await db.read(\n 'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1',\n [parent_uid, name],\n );\n return !!check_dupe[0];\n }\n\n /**\n * Checks if an entry with the given name exists under a parent specified by ID.\n * @param {number} parent_id - The internal ID of the parent directory\n * @param {string} name - The name to check for\n * @returns {Promise} True if the name exists under the parent, false otherwise\n */\n async nameExistsUnderParentID (parent_id, name) {\n const parent = await this.findByID(parent_id);\n if ( ! parent ) {\n return false;\n }\n return this.nameExistsUnderParent(parent.uuid, name);\n }\n // #endregion\n\n // #region queue logic\n async enqueue_ (op) {\n const tp = new TeePromise();\n while (\n this.currentState.queue.length > this.max_queue ||\n this.deferredState.queue.length > this.max_queue\n ) {\n await this.queueSizePromise;\n }\n\n if ( ! (op instanceof BaseOperation) ) {\n throw new Error('Invalid operation');\n }\n\n const state = this.status === FSEntryController.STATUS_READY ?\n this.currentState : this.deferredState;\n\n if ( ! Object.prototype.hasOwnProperty.call(state.updating_uuids, op.uuid) ) {\n state.updating_uuids[op.uuid] = [];\n }\n state.updating_uuids[op.uuid].push(state.queue.length);\n\n state.queue.push(op);\n\n // DRY: same pattern as FSOperationContext:provideValue\n // DRY: same pattern as FSOperationContext:rejectValue\n if ( Object.prototype.hasOwnProperty.call(this.entryListeners_, op.uuid) ) {\n const listeners = this.entryListeners_[op.uuid];\n\n delete this.entryListeners_[op.uuid];\n\n for ( const lis of listeners ) lis();\n }\n\n this.checkShouldExec_();\n\n await op.awaitDone();\n }\n\n checkShouldExec_ () {\n if ( this.status !== FSEntryController.STATUS_READY ) return;\n if ( this.currentState.queue.length === 0 ) return;\n this.exec_();\n }\n\n async exec_ () {\n if ( this.status !== FSEntryController.STATUS_READY ) {\n throw new Error('Duplicate exec_ call');\n }\n\n const queue = this.currentState.queue;\n\n this.status = FSEntryController.STATUS_RUNNING_JOB;\n\n // const conn = await db_primary.promise().getConnection();\n // await conn.beginTransaction();\n\n for ( const op of queue ) {\n op.status = op.constructor.STATUS_RUNNING;\n // await conn.execute(stmt, values);\n }\n\n // await conn.commit();\n // conn.release();\n\n // const stmtAndVals = queue.map(op => op.getStatementAndValues());\n // const stmts = stmtAndVals.map(x => x.stmt).join('; ');\n // const vals = stmtAndVals.reduce((acc, x) => acc.concat(x.values), []);\n\n // *** uncomment to debug batch queries ***\n // this.log.debug({ stmts, vals });\n // console.log('<<========================');\n // console.log({ stmts, vals });\n // console.log('>>========================');\n\n // this.log.debug('array?', Array.isArray(vals))\n\n await db.batch_write(queue.map(op => op.getStatement()));\n\n for ( const op of queue ) {\n op.status = op.constructor.STATUS_DONE;\n }\n\n this.flipState_();\n this.status = FSEntryController.STATUS_READY;\n\n for ( const op of queue ) {\n op.status = op.constructor.STATUS_DONE;\n }\n\n this.checkShouldExec_();\n }\n\n flipState_ () {\n this.currentState = this.deferredState;\n this.deferredState = {\n queue: [],\n updating_uuids: {},\n };\n const queueSizeResolve = this.queueSizeResolve;\n this.mkPromiseForQueueSize_();\n queueSizeResolve();\n }\n // #endregion\n}\n"
},
{
"path": "extensions/puterfs/fsentries/Insert.js",
"content": "import { safeHasOwnProperty } from '../lib/objectfn.js';\nimport BaseOperation from './BaseOperation.js';\n\nexport default class extends BaseOperation {\n static requiredForCreate = [\n 'uuid',\n 'parent_uid',\n ];\n\n static allowedForCreate = [\n ...this.requiredForCreate,\n 'name',\n 'user_id',\n 'is_dir',\n 'created',\n 'modified',\n 'immutable',\n 'shortcut_to',\n 'is_shortcut',\n 'metadata',\n 'bucket',\n 'bucket_region',\n 'thumbnail',\n 'accessed',\n 'size',\n 'symlink_path',\n 'is_symlink',\n 'associated_app_id',\n 'path',\n ];\n\n constructor (entry) {\n super();\n const requiredForCreate = this.constructor.requiredForCreate;\n const allowedForCreate = this.constructor.allowedForCreate;\n\n {\n const sanitized_entry = {};\n for ( const k of allowedForCreate ) {\n if ( safeHasOwnProperty(entry, k) ) {\n sanitized_entry[k] = entry[k];\n }\n }\n entry = sanitized_entry;\n }\n\n for ( const k of requiredForCreate ) {\n if ( ! safeHasOwnProperty(entry, k) ) {\n throw new Error(`Missing required property: ${k}`);\n }\n }\n\n this.entry = entry;\n }\n\n getStatement () {\n const fields = Object.keys(this.entry);\n const statement = 'INSERT INTO fsentries ' +\n `(${fields.join(', ')}) ` +\n `VALUES (${fields.map(() => '?').join(', ')})`;\n const values = fields.map(k => this.entry[k]);\n return { statement, values };\n }\n\n apply (answer) {\n answer.entry = { ...this.entry };\n }\n\n get uuid () {\n return this.entry.uuid;\n }\n};\n"
},
{
"path": "extensions/puterfs/fsentries/Update.js",
"content": "import { safeHasOwnProperty } from '../lib/objectfn.js';\nimport BaseOperation from './BaseOperation.js';\n\nexport default class extends BaseOperation {\n static allowedForUpdate = [\n 'name',\n 'parent_uid',\n 'user_id',\n 'modified',\n 'shortcut_to',\n 'metadata',\n 'thumbnail',\n 'size',\n 'path',\n ];\n\n constructor (uuid, entry) {\n super();\n const allowedForUpdate = this.constructor.allowedForUpdate;\n\n {\n const sanitized_entry = {};\n for ( const k of allowedForUpdate ) {\n if ( safeHasOwnProperty(entry, k) ) {\n sanitized_entry[k] = entry[k];\n }\n }\n entry = sanitized_entry;\n }\n\n this.uuid = uuid;\n this.entry = entry;\n }\n\n getStatement () {\n const fields = Object.keys(this.entry);\n const statement = 'UPDATE fsentries SET ' +\n `${fields.map(k => `${k} = ?`).join(', ')} ` +\n 'WHERE uuid = ? LIMIT 1';\n const values = fields.map(k => this.entry[k]);\n values.push(this.uuid);\n return { statement, values };\n }\n\n apply (answer) {\n if ( ! answer.entry ) {\n answer.is_diff = true;\n answer.entry = {};\n }\n Object.assign(answer.entry, this.entry);\n }\n};\n"
},
{
"path": "extensions/puterfs/lib/objectfn.js",
"content": "/**\n * Instead of `myObject.hasOwnProperty(k)`, always write:\n * `safeHasOwnProperty(myObject, k)`.\n *\n * This is a less verbose way to call `Object.prototype.hasOwnProperty.call`.\n * This prevents unexpected behavior when `hasOwnProperty` is overridden,\n * which is especially possible for objects parsed from user-sent JSON.\n *\n * explanation: https://eslint.org/docs/latest/rules/no-prototype-builtins\n * @param {*} o\n * @param {...any} a\n * @returns\n */\nexport const safeHasOwnProperty = (o, ...a) => {\n return Object.prototype.hasOwnProperty.call(o, ...a);\n};\n"
},
{
"path": "extensions/puterfs/main.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see Puter Worker Sandbox Playground \n \n\n\n \n Puter Worker Sandbox Playground \n Use this page to interact with the puter APIs in the same sandbox as your worker.
\n \n Run \n Clear Logs \n
\n \n \n Code \n \n \n \n Logs \n \n \n \n \n \n\n\n`;\n\nextension.get('/', { noauth: true, subdomain: 'worker-sandbox' }, (req, res) => {\n res.type('html').send(page);\n});\n"
},
{
"path": "install.md",
"content": "\n# INSTALL.md\n\n## Node.js & npm Installation Guide\n\n\n## 1. Arch Linux / Manjaro\n\n```bash\n# Update package database\nsudo pacman -Syu\n\n# Install Node.js and npm\nsudo pacman -S nodejs npm\n\n# Verify installation\nnode -v\nnpm -v\n````\n\n---\n\n## 2. Debian / Ubuntu\n\n```bash\n# Update package database and install curl\nsudo apt update\nsudo apt install -y curl\n\n# Install nvm (Node Version Manager)\ncurl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash\n\n# Load nvm\nexport NVM_DIR=\"$HOME/.nvm\"\n[ -s \"$NVM_DIR/nvm.sh\" ] && \\. \"$NVM_DIR/nvm.sh\"\n\n# Reload shell\nsource ~/.bashrc # Or source ~/.zshrc if using Zsh\n\n# Install latest Node.js and npm\nnvm install node\n\n# Verify installation\nnode -v\nnpm -v\n```\n\n---\n\n## 3. CentOS / RHEL\n\n```bash\n# Install curl if missing\nsudo yum install -y curl\n\n# Install nvm\ncurl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash\n\n\n# Load nvm\nexport NVM_DIR=\"$HOME/.nvm\"\n[ -s \"$NVM_DIR/nvm.sh\" ] && \\. \"$NVM_DIR/nvm.sh\"\n\n# Reload shell\nsource ~/.bashrc # Or source ~/.zshrc if using Zsh\n\n# Install latest Node.js and npm\nnvm install node\n\n# Verify installation\nnode -v\nnpm -v\n```\n\n---\n\n## 4. Fedora\n\n```bash\n# Update system\nsudo dnf update -y\n\n# Install Node.js and npm from modules\nsudo dnf module list nodejs # Check available versions\nsudo dnf module enable nodejs:18 # Example: enable Node 18 LTS\nsudo dnf install -y nodejs npm\n\n# Verify installation\nnode -v\nnpm -v\n```\n\n---\n\n## 5. openSUSE\n\n```bash\n# Refresh repositories\nsudo zypper refresh\n\n# Install Node.js and npm\nsudo zypper install -y nodejs npm\n\n# Verify installation\nnode -v\nnpm -v\n```\n\n---\n\n## 6. Using nvm (Optional, Recommended)\n\n`nvm` allows installing multiple Node.js versions and switching between them easily:\n\n```bash\n# Install nvm\ncurl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash\n\n# Load nvm\nexport NVM_DIR=\"$HOME/.nvm\"\n[ -s \"$NVM_DIR/nvm.sh\" ] && \\. \"$NVM_DIR/nvm.sh\"\n\n# Reload shell\nsource ~/.bashrc # Or source ~/.zshrc if using Zsh\n\n# Install latest Node.js and npm\nnvm install node\n\n# Or install latest LTS version\nnvm install --lts\n\n# Switch Node versions\nnvm use node\n\n# Verify installation\nnode -v\nnpm -v\n```\n\n"
},
{
"path": "mod_packages/testex/package.json",
"content": "{}\n"
},
{
"path": "mods/README.md",
"content": "# Puter Mods\n\nA list of Puter mods which may be expanded in the future.\n\n**Contributions of new mods are welcome.**\n\n## kdmod\n\n- **location:** [./kdmod](./kdmod)\n- **description:**\n > \"kernel dev mod\"; specifically for the devex needs of\n > GitHub user KernelDeimos and provided in case anyone else\n > finds it of any use.\n"
},
{
"path": "mods/mods_available/dev-socket/main.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see Test actions \n \n\n\n Test actions \n
\n \n\n\n`;\n\nextension.get('/test-actions', (req, res) => {\n res.setHeader('Content-Type', 'text/html; charset=utf-8');\n res.send(PAGE_HTML(JSON.stringify(ACTIONS)));\n});\n\nextension.post('/test-actions/invoke/:actionId', async (req, res) => {\n const actionId = req.params.actionId;\n const handler = INVOKE_HANDLERS[actionId];\n if ( ! handler ) {\n return res.status(404).json({ ok: false, error: 'Unknown action' });\n }\n return handler(req, res, req.body || {});\n});\n\nextension.on('ai.prompt.validate', async event => {\n console.log('ai.prompt.validate');\n const messages = event.parameters?.messages ?? [];\n console.log(`ai prompt validate: ${messages.length} messages`);\n\n console.log('is user suspended?', event.actor.type.user.suspended);\n});\n"
},
{
"path": "mods/mods_available/test-actions/package.json",
"content": "{\n \"name\": \"@heyputer/test-actions\",\n \"version\": \"1.0.0\",\n \"description\": \"Actions for test purposes\",\n \"main\": \"main.js\",\n \"type\": \"module\",\n \"private\": true\n}"
},
{
"path": "mods/mods_available/testex.js",
"content": "// Test extension for event listeners\n\nextension.on('ai.prompt.complete', event => {\n console.log('GOT AI.PROMPT.COMPLETE EVENT', event);\n});\n\nextension.on('ai.prompt.validate', event => {\n console.log('GOT AI.PROMPT.VALIDATE EVENT', event);\n});\n\nextension.on('app.new-icon', event => {\n console.log('GOT APP.NEW-ICON EVENT', event);\n});\n\nextension.on('app.rename', event => {\n console.log('GOT APP.RENAME EVENT', event);\n});\n\nextension.on('apps.invalidate', event => {\n console.log('GOT APPS.INVALIDATE EVENT', event);\n});\n\nextension.on('email.validate', event => {\n console.log('GOT EMAIL.VALIDATE EVENT', event);\n});\n\nextension.on('fs.create.directory', event => {\n console.log('GOT FS.CREATE.DIRECTORY EVENT', event);\n});\n\nextension.on('fs.create.file', event => {\n console.log('GOT FS.CREATE.FILE EVENT', event);\n});\n\nextension.on('fs.create.shortcut', event => {\n console.log('GOT FS.CREATE.SHORTCUT EVENT', event);\n});\n\nextension.on('fs.create.symlink', event => {\n console.log('GOT FS.CREATE.SYMLINK EVENT', event);\n});\n\nextension.on('fs.move.file', event => {\n console.log('GOT FS.MOVE.FILE EVENT', event);\n});\n\nextension.on('fs.pending.file', event => {\n console.log('GOT FS.PENDING.FILE EVENT', event);\n});\n\nextension.on('fs.storage.progress.copy', event => {\n console.log('GOT FS.STORAGE.PROGRESS.COPY EVENT', event);\n});\n\nextension.on('fs.storage.upload-progress', event => {\n console.log('GOT FS.STORAGE.UPLOAD-PROGRESS EVENT', event);\n});\n\nextension.on('fs.write.file', event => {\n console.log('GOT FS.WRITE.FILE EVENT', event);\n});\n\nextension.on('ip.validate', event => {\n console.log('GOT IP.VALIDATE EVENT', event);\n});\n\nextension.on('outer.fs.write-hash', event => {\n console.log('GOT OUTER.FS.WRITE-HASH EVENT', event);\n});\n\nextension.on('outer.gui.item.added', event => {\n console.log('GOT OUTER.GUI.ITEM.ADDED EVENT', event);\n});\n\nextension.on('outer.gui.item.moved', event => {\n console.log('GOT OUTER.GUI.ITEM.MOVED EVENT', event);\n});\n\nextension.on('outer.gui.item.pending', event => {\n console.log('GOT OUTER.GUI.ITEM.PENDING EVENT', event);\n});\n\nextension.on('outer.gui.item.updated', event => {\n console.log('GOT OUTER.GUI.ITEM.UPDATED EVENT', event);\n});\n\nextension.on('outer.gui.notif.ack', event => {\n console.log('GOT OUTER.GUI.NOTIF.ACK EVENT', event);\n});\n\nextension.on('outer.gui.notif.message', event => {\n console.log('GOT OUTER.GUI.NOTIF.MESSAGE EVENT', event);\n});\n\nextension.on('outer.gui.notif.persisted', event => {\n console.log('GOT OUTER.GUI.NOTIF.PERSISTED EVENT', event);\n});\n\nextension.on('outer.gui.notif.unreads', event => {\n console.log('GOT OUTER.GUI.NOTIF.UNREADS EVENT', event);\n});\n\nextension.on('outer.gui.submission.done', event => {\n console.log('GOT OUTER.GUI.SUBMISSION.DONE EVENT', event);\n});\n\nextension.on('puter-exec.submission.done', event => {\n console.log('GOT PUTER-EXEC.SUBMISSION.DONE EVENT', event);\n});\n\nextension.on('request.measured', event => {\n console.log('GOT REQUEST.MEASURED EVENT', event);\n});\n\nextension.on('sns', event => {\n console.log('GOT SNS EVENT', event);\n});\n\nextension.on('template-service.hello', event => {\n console.log('GOT TEMPLATE-SERVICE.HELLO EVENT', event);\n});\n\nextension.on('usages.query', event => {\n console.log('GOT USAGES.QUERY EVENT', event);\n});\n\nextension.on('user.email-changed', event => {\n console.log('GOT USER.EMAIL-CHANGED EVENT', event);\n});\n\nextension.on('user.email-confirmed', event => {\n console.log('GOT USER.EMAIL-CONFIRMED EVENT', event);\n});\n\nextension.on('user.save_account', event => {\n console.log('GOT USER.SAVE_ACCOUNT EVENT', event);\n});\n\nextension.on('web.socket.connected', event => {\n console.log('GOT WEB.SOCKET.CONNECTED EVENT', event);\n});\n\nextension.on('web.socket.user-connected', event => {\n console.log('GOT WEB.SOCKET.USER-CONNECTED EVENT', event);\n});\n\nextension.on('wisp.get-policy', event => {\n console.log('GOT WISP.GET-POLICY EVENT', event);\n});\n"
},
{
"path": "mods/mods_enabled/.gitignore",
"content": "*\n!.gitignore\n"
},
{
"path": "package.json",
"content": "{\n \"name\": \"puter.com\",\n \"version\": \"2.5.1\",\n \"author\": \"Puter Technologies Inc.\",\n \"license\": \"AGPL-3.0-only\",\n \"description\": \"Desktop environment in the browser!\",\n \"homepage\": \"https://puter.com\",\n \"type\": \"module\",\n \"main\": \"exports.js\",\n \"directories\": {\n \"lib\": \"lib\"\n },\n \"devDependencies\": {\n \"@eslint/js\": \"^9.35.0\",\n \"@playwright/test\": \"^1.56.1\",\n \"@stylistic/eslint-plugin\": \"^5.3.1\",\n \"@types/mime-types\": \"^3.0.1\",\n \"@types/uuid\": \"^10.0.0\",\n \"@typescript-eslint/eslint-plugin\": \"^8.46.1\",\n \"@typescript-eslint/parser\": \"^8.46.1\",\n \"@vitest/coverage-v8\": \"^4.1.0\",\n \"@vitest/ui\": \"^4.1.0\",\n \"chalk\": \"^4.1.0\",\n \"clean-css\": \"^5.3.2\",\n \"dotenv\": \"^16.4.5\",\n \"eslint\": \"^9.35.0\",\n \"eslint-rule-composer\": \"^0.3.0\",\n \"express\": \"^4.18.2\",\n \"globals\": \"^15.15.0\",\n \"html-entities\": \"^2.3.3\",\n \"html-webpack-plugin\": \"^5.6.0\",\n \"husky\": \"^9.1.7\",\n \"license-check-and-add\": \"^4.0.5\",\n \"mocha\": \"^7.2.0\",\n \"nodemon\": \"^3.1.0\",\n \"simple-git\": \"^3.32.3\",\n \"ts-proto\": \"^2.8.0\",\n \"typescript\": \"^5.4.5\",\n \"uglify-js\": \"^3.17.4\",\n \"vite-plugin-static-copy\": \"^3.3.0\",\n \"vitest\": \"^4.1.0\",\n \"webpack\": \"^5.88.2\",\n \"webpack-cli\": \"^5.1.1\",\n \"yaml\": \"^2.8.1\"\n },\n \"scripts\": {\n \"test\": \"npx vitest run --config=src/backend/vitest.config.ts && node src/backend/tools/test.mjs\",\n \"test:puterjs-api\": \"vitest run tests/puterJsApiTests\",\n \"test:backend\": \"npm run build:ts; vitest run --config=src/backend/vitest.config.ts\",\n \"test:backend-coverage\": \"npm run build:ts; vitest run --config=src/backend/vitest.config.ts\",\n \"start=gui\": \"nodemon --exec \\\"node dev-server.js\\\" \",\n \"start\": \"node ./tools/run-selfhosted.js\",\n \"prestart\": \"npm run build:ts\",\n \"dev\": \"npm run build:ts && DEVCONSOLE=1 node ./tools/run-selfhosted.js\",\n \"build\": \"npx eslint --quiet -c eslint/mandatory.eslint.config.js src/backend/src extensions && npm run build:ts && cd src/gui && node ./build.js\",\n \"check-translations\": \"node tools/check-translations.js\",\n \"prepare\": \"husky\",\n \"build:ts\": \"tsc -p tsconfig.build.json\",\n \"gen\": \"./scripts/gen.sh\"\n },\n \"workspaces\": [\n \"src/*\",\n \"tools/*\",\n \"experiments/js-parse-and-output\"\n ],\n \"nodemonConfig\": {\n \"ext\": \"js, json, mjs, jsx, svg, css\",\n \"ignore\": [\n \"./dist/\",\n \"./node_modules/\"\n ]\n },\n \"dependencies\": {\n \"@ai-sdk/openai\": \"^3.0.25\",\n \"@anthropic-ai/sdk\": \"^0.68.0\",\n \"@aws-sdk/client-dynamodb\": \"^3.490.0\",\n \"@aws-sdk/client-secrets-manager\": \"^3.879.0\",\n \"@aws-sdk/client-sns\": \"^3.907.0\",\n \"@aws-sdk/lib-dynamodb\": \"^3.490.0\",\n \"@google/genai\": \"^1.19.0\",\n \"@heyputer/putility\": \"^1.0.2\",\n \"@paralleldrive/cuid2\": \"^2.2.2\",\n \"@stylistic/eslint-plugin-js\": \"^4.4.1\",\n \"ai\": \"^6.0.73\",\n \"dedent\": \"^1.5.3\",\n \"dynalite\": \"^4.0.0\",\n \"express-xml-bodyparser\": \"^0.4.1\",\n \"file-type\": \"21.3.3\",\n \"javascript-time-ago\": \"^2.5.11\",\n \"json-colorizer\": \"^3.0.1\",\n \"music-metadata\": \"11.12.3\",\n \"open\": \"^10.1.0\",\n \"parse-domain\": \"^8.2.2\",\n \"string-template\": \"^1.0.0\",\n \"uuid\": \"^9.0.1\"\n },\n \"optionalDependencies\": {\n \"sharp\": \"^0.34.4\",\n \"sharp-bmp\": \"^0.1.5\",\n \"sharp-ico\": \"^0.1.5\"\n },\n \"engines\": {\n \"node\": \">=24.0.0\"\n }\n}\n"
},
{
"path": "rust-toolchain.toml",
"content": "[toolchain]\nchannel = \"nightly\"\ncomponents = [ \"rustc\", \"rust-std\" ]\ntargets = [ \"wasm32-unknown-unknown\", \"i686-unknown-linux-gnu\" ]\nprofile = \"minimal\"\n"
},
{
"path": "scripts/gen.sh",
"content": "#!/usr/bin/env bash\n\nset -euo pipefail\n\n\nprotoc \\\n -I=src/backend/src/filesystem/definitions/proto \\\n --plugin=protoc-gen-ts_proto=$(npm root)/.bin/protoc-gen-ts_proto \\\n --ts_proto_out=src/backend/src/filesystem/definitions/ts \\\n --ts_proto_opt=esModuleInterop=true,outputServices=none,outputJsonMethods=true,useExactTypes=false,snakeToCamel=false \\\n src/backend/src/filesystem/definitions/proto/fsentry.proto"
},
{
"path": "src/backend/.gitignore",
"content": "# MAC OS hidden directory settings file\n.DS_Store\n\n# Created by https://www.toptal.com/developers/gitignore/api/node\n# Edit at https://www.toptal.com/developers/gitignore?templates=node\n\n### Node ###\n# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\nlerna-debug.log*\n.pnpm-debug.log*\n\n# Diagnostic reports (https://nodejs.org/api/report.html)\nreport.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json\n\n# Runtime data\npids\n*.pid\n*.seed\n*.pid.lock\n\n# Directory for instrumented libs generated by jscoverage/JSCover\nlib-cov\n\n# Coverage directory used by tools like istanbul\ncoverage\n*.lcov\n\n# nyc test coverage\n.nyc_output\n\n# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)\n.grunt\n\n# Bower dependency directory (https://bower.io/)\nbower_components\n\n# node-waf configuration\n.lock-wscript\n\n# Compiled binary addons (https://nodejs.org/api/addons.html)\nbuild/Release\n\n# Dependency directories\nnode_modules/\njspm_packages/\n\n# Snowpack dependency directory (https://snowpack.dev/)\nweb_modules/\n\n# TypeScript cache\n*.tsbuildinfo\n\n# Optional npm cache directory\n.npm\n\n# Optional eslint cache\n.eslintcache\n\n# Microbundle cache\n.rpt2_cache/\n.rts2_cache_cjs/\n.rts2_cache_es/\n.rts2_cache_umd/\n\n# Optional REPL history\n.node_repl_history\n\n# Output of 'npm pack'\n*.tgz\n\n# Yarn Integrity file\n.yarn-integrity\n\n# dotenv environment variables file\n.env\n.env.test\n.env.production\n\n# parcel-bundler cache (https://parceljs.org/)\n.cache\n.parcel-cache\n\n# Next.js build output\n.next\nout\n\n# Nuxt.js build / generate output\n.nuxt\ndist\n\n# Gatsby files\n.cache/\n# Comment in the public line in if your project uses Gatsby and not Next.js\n# https://nextjs.org/blog/next-9-1#public-directory-support\n# public\n\n# vuepress build output\n.vuepress/dist\n\n# Serverless directories\n.serverless/\n\n# FuseBox cache\n.fusebox/\n\n# DynamoDB Local files\n.dynamodb/\n\n# TernJS port file\n.tern-port\n\n# Stores VSCode versions used for testing VSCode extensions\n.vscode-test\n\n# yarn v2\n.yarn/cache\n.yarn/unplugged\n.yarn/build-state.yml\n.yarn/install-state.gz\n.pnp.*\n\n# End of https://www.toptal.com/developers/gitignore/api/node\npublic/.DS_Store\n*.zip\n*.pem\npublic/.DS_Store\npublic/.DS_Store\npublic/.DS_Store\n./build\nbuild\n\n# config file\nvolatile/\nssl\nssl/\nkeys\n*.code-workspace\n\n# credentials\ncreds*\n\n# test-webhook persisted key\ntools/.test-webhook-config.json\n\n# thumbnai-service\nthumbnail-service\n\n# init sql generated from ./run.sh\ninit.sql\n"
},
{
"path": "src/backend/CONTRIBUTING.md",
"content": "# Contributing to Puter's Backend\n\n## File Structure\n\n\n\n## Architecture\n\n- [boot sequence](./doc/contributors/boot-sequence.md)\n- [modules and services](./doc/contributors/modules.md)\n\n## Features\n\n- [protected apps](./doc/features/protected-apps.md)\n- [service scripts](./doc/features/service-scripts.md)\n\n## Lists of Things\n\n- [list of permissions](./doc/lists-of-things/list-of-permissions.md)\n\n## Code-First Approach\n\nIf you prefer to understand a system by looking at the\nfirst files which are invoked and starting from there,\nhere's a handy list!\n\n- [Kernel](./src/Kernel.js), despite its intimidating name, is a\n relatively simple (< 200 LOC) class which loads the modules\n (modules register services), and then starts all the services.\n- [RuntimeEnvironment](./src/boot/RuntimeEnvironment.js)\n sets the configuration and runtime directories. It's invoked by Kernel.\n- The default setup for running a self-hosted Puter loads these modules:\n - [CoreModule](./src/CoreModule.js)\n - [DatabaseModule](./src/DatabaseModule.js)\n - [LocalDiskStorageModule](./src/LocalDiskStorageModule.js)\n- HTTP endpoints are registered with\n [WebServerService](./src/services/WebServerService.js)\n by these services:\n - [ServeGUIService](./src/services/ServeGUIService.js)\n - [PuterAPIService](./src/services/PuterAPIService.js)\n - [FilesystemAPIService](./src/services/FilesystemAPIService.js)\n\n## Development Philosophies\n\n### The copy-paste rule\n\nIf you're copying and pasting code, you need to ask this question:\n- am I copying as a reference (i.e. how this function is used),\n- or am I copying an implementation of actual behavior?\n\nIf your answer is the first, you should find more than one piece of\ncode that's doing the same thing you want to do and see if any of them\nare doing it differently. One of the ways of doing this thing is going\nto be more recent and/or (yes, potentially \"or\") more correct.\nMore correct approaches are ones which reduce\n[coupling](https://en.wikipedia.org/wiki/Coupling_(computer_programming)),\nmove from legacy implementations to more recent ones, and are actually\nmore convenient for you to use. Whenever ever any of these three things\nare in contention it's very important to communicate this to the\nappropriate maintainers and contributors.\n\nIf your answer is the second, you should find a way to\n[DRY that code](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself).\n\n### Architecture Mistakes? You will make them and it will suck.\n\nIn my experience, the harder I think about the correct way to implement\nsomething, the bigger a mistake I'm going to make; ***unless*** a big part\nof the reason I'm thinking so hard is because I want to find a solution\nthat reduces complexity and has the right maintenance trade-off.\nThere's no easy solution for this so just keep it in mind; there are some\nthings we might write 2 times, 3 times, even more times over before we\nreally get it right and *that's okay*; sometimes part of doing useful work is\ndoing the useless work that reveals what the useful work is.\n\n## Underlying Constructs\n\n- [putility's README.md](../putility/README.md)\n - Whenever you see `AdvancedBase`, that's from here\n - Many things in backend extend this. Anything that doesn't only doesn't\n because it was written before `AdvancedBase` existed.\n - Allows adding \"traits\" to classes\n - Have you ever wanted to wrap every method of a class with\n common behavior? This can do that!\n"
},
{
"path": "src/backend/README.md",
"content": "# Puter Backend\n\n_Part of a High-Level Distributed Operating System_\n\nWhether or not you call Puter an operating system\n(we call it a \"high-level distributed operating system\"),\n**operating systems for devices** \nare a useful reference point to describe the architecture of Puter.\nIf Puter's \"hardware\" is services, and Puter's \"userspace\" is the\nclient side of the API, then Puter's \"kernel\" is the backend.\n\nPuter's backend is composed of:\n- The **Kernel** class, which is responsible for initialization\n- A number of **Modules** which are registered in **Kernel** for a customized\n Puter instance.\n- Many **Services** which are contained inside modules.\n\n## Documentation\n\n- [Backend File Structure](./doc/contributors/structure.md)\n- [Boot Sequence](./doc/contributors/boot-sequence.md)\n- [Kernel](./doc/Kernel.md)\n- [Modules](./doc/contributors/modules.md)\n\n## Can I use Puter's Backend Alone?\n\nPuter's backend is not dependent on Puter's frontned. In fact, you could\nprevent Puter's GUI from ever showing up by disabling PuterHomepageModule.\nSimilarly, you can run Puter's backend with no modules loaded for a completely\nblank slate, or only include CoreModule and WebModule to quickly build your\nown backend that's compatible with any of Puter's services.\n\n## What can it do?\n\nPuter's Kernel only initializes modules, nothing more. The modules bring a lot\nof capabilities to the table, however. Within this directory you'll find modules that:\n- coerce all the well-known AI services to a common interface\n- manage authentication with Wisp servers (this brings TCP to the browser!)\n- manage apps on Puter\n- allow a user to host websites from Puter\n- provide persistent key-value storage to Puter's desktop and apps\n- provide a fast filesystem implementation\n- communicate with other instances of Puter's backend,\n secured with elliptic curve cryptography\n- provide more services like converting files and compiling low-level code.\n\n\n"
},
{
"path": "src/backend/doc/A-and-A/auth.md",
"content": "# Authentication Documentation\n\n## Concepts\n\n### Actor\n\nAn \"Actor\" is an entity that can be authenticated. The following types of\nactors are currently supported by Puter:\n- **UserActorType** - represents a user and is identified by a user's UUID\n- **AppUnderUserActorType** - represents an app running in an iframe from a\n `puter.site` domain or another origin and is identified by a user's UUID\n and an app's UUID together.\n- **AccessTokenActorType** - not widely currently, but Puter supports\n a concept called \"access tokens\". Any user can create an access token and\n then grant any permissions they want to that access token. The access\n token will have those permissions granted provided that the user who\n created the access token does as well (via permission cascade)\n- **SiteActorType** - represents a `puter.site` website accessing Puter's API.\n- **SystemActorType** - internal representation of the actor during a privileged\n backend operation. This actor cannot be authenticated in a request.\n This actor does not represent the `system` user.\n\n### Token\n\n- **Legacy** - legacy tokens result in an error response\n- **Session** - this token is a JWT with a claim for the UUID of an entry in\n server memory or the database that we call a \"session\". This entry associates\n the token to a user and some metadata for security auditing purposes.\n Revoking the session entry disables the token.\n This type of token resolves to an actor with **UserActorType**.\n- **AppUnderUser** - this token is a JWT with a claim for an app UUID and a\n claim for a session UUID.\n Revoking the session entry disables the token.\n This type of token resolves to an actor with **AppUnderUserActorType**.\n- **AccessToken** - this token is a JWT with three claims:\n - A session UUID\n - An optional App UUID\n - A UUID representing the access token for permission associations\n The session or session+app creates a **UserActorType** or\n **AppUnderUserActorType** actor respectively. This actor is called\n the \"authorizor\". This actor is aggregated by an **AccessTokenActorType**\n actor which becomes the effective actor for a request.\n- **ActorSite** - this token is a JWT with a claim for a site UID.\n The site UID is associated with an origin, generally a `puter.site`\n subdomain.\n\n## Components\n\n### Auth Middleware\n\nThere have so far been three iterations of the authentication middleware:\n- `src/backend/src/middleware/auth.js`\n- `src/backend/src/middleware/auth2.js`\n- `src/backend/src/middleware/configurable_auth.js`\n\nThe newest implementation is `configurable_auth` and eventually the other\ntwo will be removed. There is no legacy behavior involved:\n- `auth` was rewritten to use `auth2`\n- `auth2` was rewritten to use `configurable_auth`\n\nThe `configurable_auth` middleware accepts a parameter that can be specified\nif an endpoint is optionally authenticated. In this case, the request's\n`actor` will be `undefined` if there was no information for authentication.\n"
},
{
"path": "src/backend/doc/A-and-A/permission.md",
"content": "# Permission Documentation\n\n## Concepts\n\n### Permission\n\nA permission is a string composed of colon-delimited components which identifies\na resource or functionality to which access can be controlled.\n\nFor example, `fs:e8ac2973-287b-4121-a75d-7e0619eb8e87:read` is a permission which\nrepresents reading the file or directory with UUID `e8ac2973-287b-4121-a75d-7e0619eb8e87`.\n\n### Group\n\nA group has an owner and several member users. An owner decides what users are in the\ngroup and what users are not. Any user can grant permissions to the group.\n\n### Granting & Revoking\n\nGranting is the act of creating a permission association to a user or group from\nthe current user. A permission association also holds an object called `extra`\nwhich holds additional claims associated with the permission association.\nThese are arbitrary and can be used in any way by the subsystem or extension that\nis checking the permission. `extra` is usually just an empty object.\n\nRevoking is the act of removing a permission association.\n\n### Permission Options\n\nPermission options are an association between a permission and an actor that can not\nbe revoked by another actor. For example, the user `ed` always has access to files\nunder `/ed`. The user `system` always has all permissions granted. These can also be\nconsidered \"terminals\" because they will always be at\nthe end of a pathway through granted permissions between users.\nThis are also called \"implied\" permissions because they are implied by the system.\n\n### Permission Pathways\n\nA permission pathway is the path between users or groups that leads to a permission.\n\nFor example, `ed` can grant the permission `a:b` to `fred`, then `fred` can grant\nthat permission to the group `cool_group`, and then `alice` may be in the group\n`cool_group`. Assuming `ed` holds the implied permission `a:b`, a permission path\nexists between `alice` and `ed` via `cool_group` and `fred`:\n\n```\nalice <--<> cool_group <-- fred <-- ed (a:b)\n```\n\nIf any link in this chain breaks the permission is effectively revoked from `alice`\nunless there is another pathway leading to a valid permission option for `a:b`.\n\n### Reading - AKA Permission Scan Result\n\nA permission reading is a JSON-serializable object which contains all the pathways\na specified actor has to permissions options matching the specified permission strings.\n\nThe following is an example reading for the user `ed3` on the permission\n`fs:24729b88-a4c5-4990-ad4e-272b87895732:read`. This file is owned by the\nuser `admin` who shared it with `ed3`.\n\n```\n[\n {\n \"$\": \"explode\",\n \"from\": \"fs:24729b88-a4c5-4990-ad4e-272b87895732:read\",\n \"to\": [\n \"fs:24729b88-a4c5-4990-ad4e-272b87895732:read\",\n \"fs:24729b88-a4c5-4990-ad4e-272b87895732:write\",\n \"fs:24729b88-a4c5-4990-ad4e-272b87895732\",\n \"fs\"\n ]\n },\n {\n \"$\": \"path\",\n \"via\": \"user\",\n \"has_terminal\": true,\n \"permission\": \"fs:24729b88-a4c5-4990-ad4e-272b87895732:read\",\n \"data\": {},\n \"holder_username\": \"ed3\",\n \"issuer_username\": \"admin\",\n \"reading\": [\n {\n \"$\": \"explode\",\n \"from\": \"fs:24729b88-a4c5-4990-ad4e-272b87895732:read\",\n \"to\": [\n \"fs:24729b88-a4c5-4990-ad4e-272b87895732:read\",\n \"fs:24729b88-a4c5-4990-ad4e-272b87895732:write\",\n \"fs:24729b88-a4c5-4990-ad4e-272b87895732\",\n \"fs\"\n ]\n },\n {\n \"$\": \"option\",\n \"permission\": \"fs:24729b88-a4c5-4990-ad4e-272b87895732:read\",\n \"source\": \"implied\",\n \"by\": \"is-owner\",\n \"data\": {}\n },\n {\n \"$\": \"option\",\n \"permission\": \"fs:24729b88-a4c5-4990-ad4e-272b87895732:write\",\n \"source\": \"implied\",\n \"by\": \"is-owner\",\n \"data\": {}\n },\n {\n \"$\": \"option\",\n \"permission\": \"fs:24729b88-a4c5-4990-ad4e-272b87895732\",\n \"source\": \"implied\",\n \"by\": \"is-owner\",\n \"data\": {}\n },\n {\n \"$\": \"time\",\n \"value\": 19\n }\n ]\n },\n {\n \"$\": \"time\",\n \"value\": 20\n }\n]\n```\n\nEach object in the reading has a property named `$` which is the type for the object.\nThe most fundamental types for permission readings are `path` and `option`. A path\nalways contains another reading, which contains more paths or options. An option\nspecifies the permission string, the name of the rule that granted the permission,\nand a data object which may hold additional claims.\n\nReadings begin with an `explode` if there are multiple strings that may grant the\npermission.\n\nReadings end with a `time` that repots how long the reading took to help manage\nthe potential performance impact of complex permission graphs.\n\n## Permission Service\n\n### check(actor, permissions)\n\nReturns true if the current actor has a path to any permission options matching\nany of the permission strings specified by `permissions`. This is done by invoking\n`scan()` and returning `true` if there are more than 0 permission options.\n\n### scan(actor, permissions)\n\nReturns a \"reading\". A permission reading is a JSON-serializable structure.\nReadings are described above.\n\n## Permission Scan Sequence\n\nThe `scan()` method of **PermissionService** invokes the permission scan sequence.\nThe permission scan sequence is a [Sequence](https://github.com/HeyPuter/puter/blob/0e0bfd6d7c92eed5080518a099c9a66a2f2dc9ec/src/backend/src/codex/Sequence.js)\nthat is defined in [scan-permission.js](src/backend/src/structured/sequence/scan-permission.js).\nIt invokes many \"permission scanners\" which are defined in\n[permission-scanners.js](src/backend/src/unstructured/permission-scanners.js)\n\nThe Permission Scan Sequence is as follows:\n- `grant_if_system` - if system user, push an option to the reading and stop\n- `rewrite_permission` - process the permission through any permission string\n rewriters that were registered with PermissionService by other services.\n For example, since path-based file permissions aren't currently supported\n the FilesystemService regsiters a rewriter that converts any `fs:/`\n permission into a corresponding UUID permission.\n- `explode_permission` - break the permission into multiple permissions\n than are sufficient to grant the permission being scanned. For example if\n there are multiple components, like `a.b.c`, having either permission `a.b` or\n `a` granted implis having `a.b.c` granted. Other services can also register\n \"permission exploders\" which handle non-hierarchical cases such as\n `fs:AAAA:write` implying `fs:AAAA:read`.\n- `run_scanners` - run the permission scanners.\n\nEach permission scanner has a name, documentation text, and a scan function.\nThe scan function has access to the scan sequence's context and can push\nobjects onto the permission reading.\n\nFor information on individual scanners, refer to permission-scanners.js.\n"
},
{
"path": "src/backend/doc/Kernel.md",
"content": "# Puter Kernel Documentation\n\n## Overview\n\nThe **Puter Kernel** is the core runtime component of the Puter system. It provides the foundational infrastructure for:\n\n- Initializing the runtime environment\n- Managing internal and external modules (extensions)\n- Setting up and booting core services\n- Configuring logging and debugging utilities\n- Integrating with third-party modules and performing dependency installs at runtime\n\nThis kernel is responsible for orchestrating the startup sequence and ensuring that all necessary services, modules, and environmental configurations are properly loaded before the application enters its operational state.\n\n---\n\n## Features\n\n1. **Modular Architecture**: \n The Kernel supports both internal and external modules:\n - **Internal Modules**: Provided to Kernel by an initializing script, such\n as `tools/run-selfhosted.js`, via the `add_module()` method.\n - **External Modules**: Discovered in configured module directories and installed\n dynamically. This includes resolving and executing `package.json` entries and\n running `npm install` as needed.\n\n2. **Service Container & Registry**: \n The Kernel initializes a service container that manages a wide range of services. Services can:\n - Register modules\n - Initialize dependencies\n - Emit lifecycle events (`boot.consolidation`, `boot.activation`, `boot.ready`) to\n orchestrate a stable and consistent environment.\n\n3. **Runtime Environment Setup**: \n The Kernel sets up a `RuntimeEnvironment` to determine configuration paths and environment parameters. It also provides global helpers like `kv` for key-value storage and `cl` for simplified console logging.\n\n4. **Logging and Debugging**: \n Uses a temporary `BootLogger` for the initialization phase until LogService is\n initialized, at which point it will replace the boot logger. Debugging features\n (`ll`, `xtra_log`) are enabled in development environments for convenience.\n\n## Initialization & Boot Process\n\n1. **Constructor**: \n When a Kernel instance is created, it sets up basic parameters, initializes an empty\n module list, and prepares `useapi()` integration.\n\n2. **Booting**: \n The `boot()` method:\n - Parses CLI arguments using `yargs`.\n - Calls `_runtime_init()` to set up the `RuntimeEnvironment` and boot logger.\n - Initializes global debugging/logging utilities.\n - Sets up the service container (usually called `services`c instance of **Container**).\n - Invokes module installation and service bootstrapping processes.\n\n3. **Module Installation**: \n Internal modules are registered and installed first. \n External modules are discovered, packaged, installed, and their code is executed. \n External modules are given a special context with access to `useapi()`, a dynamic\n import mechanism for Puter modules and extensions.\n\n4. **Service Bootstrapping**: \n After modules and extensions are installed, services are initialized and activated.\n For more information about how this works, see [boot-sequence.md](./contributors/boot-sequence.md).\n\n"
},
{
"path": "src/backend/doc/README.md",
"content": "## Backend - Contributor Documentation\n\n### Where to Start\n\nStart with [Backend File Structure](./contributors/structure.md).\n\nThere also also some videos. In one of the videos Eric does a\nSteve Ballmer impression so it's definitely worth it.\n- [Services and Modules in Puter](https://www.youtube.com/watch?v=TOeS67QXMVU)\n- [Puter's Boot Sequence](https://www.youtube.com/watch?v=a8bOLNnW1Uo)\n- [Building a Driver on Puter](https://www.youtube.com/watch?v=8znQmrKgNxA)\n\n### Index\n\n- [Backend File Structure](./contributors/structure.md)\n- [Boot Sequence](./contributors/boot-sequence.md)\n- [Kernel](./Kernel.md)\n- [Modules](./contributors/modules.md)\n- [Configuring Logs](./log_config.md)\n"
},
{
"path": "src/backend/doc/contributors/boot-sequence.md",
"content": "# Puter Backend Boot Sequence\n\nThis document describes the boot sequence of Puter's backend.\n\n**Runtime Environment**\n - Configuration directory is determined\n - Runtime directory is determined\n - Mod directory is determined\n - Services are instantiated\n\n**Construction**\n - Data structures are created\n\n**Initialization**\n - Registries are populated\n - Services prepare for next phase\n\n**Consolidation**\n - Service event bus receives first event (`boot.consolidation`)\n - Services perform coordinated setup behaviors\n - Services prepare for next phase\n\n**Activation**\n - Blocking listeners of `boot.consolidation` have resolved\n - HTTP servers start listening\n\n**Ready**\n - Services are informed that Puter is providing service\n\n## Boot Phases\n\n### Construction\n\nServices implement a method called `construct` which initializes members\nof an instance. Services do not override the class constructor of\n**BaseService**. This makes it possible to use the `new` operator without\ninvoking a service's constructor behavior during debugging.\n\nThe first phase of the boot sequence, \"construction\", is simply a loop to\ncall `construct` on all registered services.\n\nThe `_construct` override should not:\n- call other services\n- emit events\n\n### Initialization\n\nAt initialization, the `init()` method is called on all services.\nThe `_init` override can be used to:\n- register information with other services, when services don't\n need to register this information in a specific sequence.\n An example of this is registering commands with CommandService.\n- perform setup that is required before the consolidation phase starts.\n\n### Consolidation\n\nConsolidation is a phase where services should emit events that\nare related to bringing up the system. For example, WebServerService\n('web-server') emits an event telling services to install middlewares,\nand later emits an event telling services to install routes.\n\nConsolidation starts when Kernel emits `boot.consolidation` to the\nservices event bus, which happens after `init()` resolves for all\nservices.\n\n### Activation\n\nActivation is a phase where services begin listening on external\ninterfaces. For example, this is when the web server starts listening.\n\nActivation starts when Kernel emits `boot.activation`.\n\n### Ready\n\nReady is a phase where services are informed that everything is up.\n\nReady starts when Kernel emits `boot.ready`.\n\n## Events and Asynchronous Execution\n\nThe services event bus is implemented so you can `await` a call to `.emit()`.\nEvent listeners can choose to have blocking behavior by returning a promise.\n\nDuring emission of a particular event, listeners of this event will not\nblock each other, but all listeners must resolve before the call to\n`.emit()` is resolved. (i.e. `emit` uses `Promise.all`)\n\n## Legacy Services\n\nSome services were implemented before the `BaseService` class - which\nimplements the `init` method - was created. These services are called\n\"legacy services\" and they are instantiated _after_ initialization but\n_before_ consolidation.\n"
},
{
"path": "src/backend/doc/contributors/coding-style.md",
"content": "# Backend Style\n\n## File Structure\n\n### Copyright Notice\n\nAll files should begin with the standard copyright notice:\n\n```javascript\n/*\n * Copyright (C) 2025-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } The user object\n * @throws {Error} If user not found\n */\nasync function getUserById(id) {\n // ...\n}\n```\n\n### Inline Comments\n\n- Use inline comments to explain complex logic\n- Prefix comments with tags like `track:` to indicate specific purposes\n\n```javascript\n// track: slice a prefix\nconst uid = uid_part.slice('uid#'.length);\n```\n"
},
{
"path": "src/backend/doc/contributors/modules.md",
"content": "# Puter Kernel Moduels and Services\n\n## Modules\n\nA Puter kernel module is simply a collection of services that run when\nthe module is installed. You can find an example of this in the\n`run-selfhosted.js` script at the root of the Puter monorepo.\n\nHere is the relevant excerpt in `run-selfhosted.js` at the time of\nwriting this documentation:\n\n```javascript\nconst {\n Kernel,\n CoreModule,\n DatabaseModule,\n LocalDiskStorageModule,\n SelfHostedModule\n} = (await import('@heyputer/backend')).default;\n\nconst k = new Kernel();\nk.add_module(new CoreModule());\nk.add_module(new DatabaseModule());\nk.add_module(new LocalDiskStorageModule());\nk.add_module(new SelfHostedModule());\nk.boot();\n```\n\nA few modules are added to Puter before booting. If you want to install\nyour own modules into Puter you can edit this file for self-hosted runs\nor create your own script that boots Puter. This makes it possible to\nhave deployments of Puter with custom functionality.\n\nTo function properly, Puter needs **CoreModule**, a database module,\nand a storage module.\n\nA module extends\n[AdvancedBase](../../../putility/README.md)\nand implements\nan `install` method. The install method has one parameter, a\n[Context](../../src/util/context.js)\nobject containing all the values kernel modules have access to. This\nincludes the `services`\n[Container](../../src/services/Container.js`).\n\nA module adds services to Puter.eA typical module may look something\nlike this:\n\n```javascript\nclass MyPuterModule extends AdvancedBase {\n async install (context) {\n const services = context.get('services');\n\n const MyService = require('./path/to/MyService.js');\n services.registerService('my-service', MyService, {\n some_options: 'for-my-service',\n });\n }\n}\n```\n\n## Services\n\nServices extend\n[BaseService](../../src/services/BaseService.js)\nand provide additional functionality for Puter. They can add HTTP\nendpoints and register objects with other services.\n\nWhen implementing a service it is important to understand\nPuter's [boot sequence](./boot-sequence.md)\n\nA typical service may look like this:\n\n```javascript\nclass MyService extends BaseService {\n static MODULES = {\n // Use node's `require` function to populate this object;\n // this makes these available to `this.require` and offers\n // dependency-injection for unit testing.\n ['some-module']: require('some-module')\n }\n\n // Do not override the constructor of BaseService - use this instead!\n async _construct () {\n this.my_list = [];\n }\n\n // This method is called after _construct has been called on all\n // other services.\n async _init () {\n const services = this.services;\n\n // We can get the instances of other services here\n const svc_otherService = services.get('other-service');\n }\n\n // The service container can listen on the \"service event bus\"\n async ['__on_boot.consolidation'] () {}\n async ['__on_boot.activation'] () {}\n async ['__on_start.webserver'] () {}\n async ['__on_install.routes'] () {}\n}\n```\n"
},
{
"path": "src/backend/doc/contributors/structure.md",
"content": "# Puter Backend - Directory Structure\n\n## MFU - Most Frequently Used\n\nThese locations under `/src/backend/src` are the most important\nto know about. Whether you're contributing a feature or fixing a bug,\nyou might only need to look at code in these locations.\n\n### `modules` directory\n\nThe `modules` directory contains Puter backend kernel modules only.\nEverything in here has a `Module.js` file and one or more\n`Service.js` files.\n\n> **Note:** A \"backend kernel module\" is simply a class understood by\n [`src/backend/src/Kernel.js`](../../src/Kernel.js)\n that registers a number of \"Service\" classes.\n You can look at [Puter's init file](../../../../tools/run-selfhosted.js)\n to see how modules are added to Puter.\n\nThe `README.md` file inside any module directory is generated with\nthe `module-docgen` script in the Puter repo's `/tools` directory.\nThe actual documentation for the module exists in jsdoc comments\nin the source files.\n\nEach module might contain these directories:\n- `doc/` - additional module documentation, like sample requests\n- `lib/` - utility code that isn't a Module or Service class.\n This utility code may be exposed by a service in the module\n to Puter's runtime import mechanism for extension support.\n\n### `services` directory\n\nThis directory existed before the `modules` directory. Most of\nthe services here go on a module called **CoreModule**\n(CoreModule.js is directly in `/src/backend/src`), but this\ndirectory can be thought of as \"services that are not yet\norganized in a distinct module\".\n\n### `routers` directory\n\nWhile routes are typically registered by Services, the implementation\nof a route might be placed under `src/backend/src/routers` to keep the\nservice's code tidy or for legacy reasons.\n\nThese are some services that reference files under `src/backend/src/routers`:\n- [PermissionAPIService](../../src/services/PermissionAPIService.js) - \n This service registers routes that allow a user to configure permissions they\n grant to apps and groups. This is a relatively recent case of using files under the\n `routers` directory to clean up the service.\n- [UserProtectedEndpointsService](../../src/services/web/UserProtectedEndpointsService.js) -\n This service follows a slightly different approach where files under\n `routers/user-protected` contain an \"endpoint specification\" instead of an express\n handler function. This might be good inspiration for future routes.\n- [PuterAPIService](../../src/services/PuterAPIService.js) -\n This service is a catch-all for routes that existed before separation of concerns\n into backend kernel modules.\n \n### `filesystem` directory\n\nThe filesystem is likely the most complex portion of Puter's source code. This code\nis in its own directory as a matter of circumstance more than intention. Ideally the\nfilesystem's concerns will be split across a few modules as we prepare to add\nsupport for mounting different file systems and improved cache behavior.\nFor example, Puter's native filesystem implementation should be mostly moved to\n`src/backend/src/modules/puterfs` as we continue this development.\n\nSince this directory is in flux, don't trust this documentation completely.\nIf you're contributing to filesystem,\n[tag @KernelDeimos on the community Discord](https://discord.gg/PQcx7Teh8u)\nif you have questions.\n\nThese are the key locations in the `filesystem` directory:\n- `FSNodeContext.js` - When you have a reference to a file or directory in backend code,\n it is an instance of the FSNodeContext class.\n- `ll_operations` - Runnables that implement the behavior of a filesystem operation.\n These used to include the behavior of Puter's filesystem, but they now delegate\n the actual behavior to the implementation in the `.provider` member of a\n FSNodeContext (filesystem node / a file or directory) so that we can eventually\n support \"mountpoints\" (multiple filesystem implementations).\n- `hl_operations` - Runnables that implement the behavior of higher-level versions\n of filesystem operations. For example, the high-level mkdir operation might create\n multiple directories in chain; the high-level write might change the name of the\n file to avoid conflicts if you specify the `dedupe_name` flag.\n"
},
{
"path": "src/backend/doc/dev_socket.md",
"content": "## Backend - dev socket\n\nThe \"dev socket\" allows you to interact with Puter's backend by running commands.\nIt's a UNIX socket that lets you run commands registered with\n[CommandService](../../src/services/CommandService.js) (e.g. `help`, `logs:indent`, `params:get`, etc.).\n\n### Enabling the dev socket\n\nThe dev socket is provided by the **dev-console extension** and is **opt-in**. To enable it:\n\n1. Set the environment variable `DEVCONSOLE=1` when starting Puter (e.g. `npm run dev` already does this).\n2. The extension lives in `extensions/dev-console/` and registers a `dev-socket` service when `DEVCONSOLE=1`.\n\n### Socket location\n\nThe socket is created in a directory chosen as follows (in order):\n\n- `PUTER_DEV_SOCKET_DIR` if set\n- `./volatile/runtime` if it exists (typical local dev)\n- otherwise the process current working directory\n\nThe socket file is named `dev.sock`.\n\n### Connecting\n\nWhen in that directory, connect with your tool of choice. For example, using `nc` and `rlwrap` for readline history:\n\n```\nrlwrap nc -U ./dev.sock\n```\n\nIf it is successful you will see a message with instructions. Enter a command (e.g. `help`) and press Enter.\n"
},
{
"path": "src/backend/doc/extensions/README.md",
"content": "# Puter Backend Extensions\n\n## What Are Extensions\n\nExtensions can extend the functionality of Puter's backend by handling specific\nevents or importing/exporting runtime libraries.\n\n## Creating an Extension\n\nThe easiest way to create an extension is to place a new file or directory under\nthe `extensions/` directory immediately under the root directory of the Puter\nrepository. If your extension is a single `.js` file called `my-extension.js` it\nwill be implicitly converted into a CJS module with the following structure:\n\n```\nextensions/\n |\n |- my-extension/\n |\n |- package.json\n |- main.js\n```\n\nThe location of the extensions directory can be changed in\n[the config file](../../../../doc/self-hosters/config.md)\nby setting `mod_directories` to an array of valid locations.\nThe `mod_directories` parameter has the following default value:\n```json\n[\"{repo}/mods/mods_enabled\", \"{repo}/extensions\"]\n```\n\n### Events\n\nThe primary mechanism of communication between extensions and Puter,\nand between different extensions, is through events. The `extension`\npseudo-global provides `.on(fn)` to add event listemers and\n`.emit('name', { arbitrary: 'data' })` to emit events.\n\nTo try working with events, you could make a simple extension that\nemits an event after adding a listener for its own event:\n\n```javascript\n// Listen to a test event called 'test-event'\nextension.on('test-event', event => {\n console.log(`We got the test event from ${sender}`);\n});\n\n// Listen to init; a good time to emit events\nextension.on('init', event => {\n extension.emit('test-event', { sender: 'Quinn' });\n});\n```\n\n### Puter Extension Imports\n\nYour extensions may need to invoke specific actions in Puter's backend\nin response to an event. Puter provides libraries at runtime which you\ncan access via `extension.imports`:\n\n```javascript\nconst { kv } = extension.imports('data');\nkv.set('some-key', 'some value');\n```\n\n#### The `data` import\n\nThe data import makes it possible to access Puter's database, persistent\nkey-value store, and in-memory cache.\n- [Read more about the 'data' import](./builtins/data.md)\n\n\n### Adding Features to Puter\n- [Implementing Drivers](./pages/drivers.md)\n\n### Bundled extensions\n\n- **dev-console** – When `DEVCONSOLE=1` is set (e.g. `npm run dev`), the dev-console extension registers a UNIX socket (`dev.sock`) so you can run backend commands (see [CommandService](../../src/services/CommandService.js)) from a terminal. See [Backend – dev socket](../dev_socket.md).\n\n## Extensions - Planned Features\n\nExtensions are under refactor currently. This is the checklist:\n- [x] Add RuntimeModule construct for imports and exports\n- [x] Add support to implement drivers in extensions\n- [ ] Add the ability to target specific extensions when\n emitting events\n- [ ] Add event name aliasing and configurable import mapping\n- [ ] Extract extension loading from the core\n- [ ] List exports in console\n"
},
{
"path": "src/backend/doc/extensions/builtins/data.md",
"content": "## Extensions - the `data` extension\n\nThe `data` extension can be imported in custom extensions for access\nto the database and key-value store.\n\nYou can import these from `'data'`:\n- `db` - Puter's main SQL database\n- `kv` - A persistent key-value store\n- `cache` - In-memory [kv.js](https://github.com/HeyPuter/kv.js/) store\n\n```javascript\nconst { db, kv, cache } = extension.import('data');\n```\n\n### Database (`db`)\n\nDon't forget to import it first!\n```javascript\nconst { db } = extension.import('data');\n```\n\n#### `db.read`\n\nUsage:\n\n```javascript\nconst rows = await db.read('SELECT * FROM apps WHERE `name` = ?', [\n 'editor'\n]);\n```\n#### `db.write`\n\nUsage:\n\n```javascript\nconst {\n insertId, // internal ID of new row (if this is an INSERT)\n anyRowsAffected, // true if 1 or more rows were affected\n} = await db.write(\n // A query like INSERT, UPDATE, DELETE, etc...\n 'INSERT INTO example_table (a, b, c) VALUES (?, ?, ?)',\n // Parameters (all user input should go here)\n [\n \"Value for column a\",\n \"Value for column b\",\n \"Value for column c\",\n ]\n);\n```\n\n### Persistent KV Store (`kv`)\n\nDon't forget to import it first!\n```javascript\nconst { kv } = extension.import('data');\n```\n\n#### `kv.get({ key })`\n\n```javascript\n// Short-Form (like kv.js)\nconst someValue = kv.get('some-key');\n\n// Long-Form (the `puter-kvstore` driver interface)\nconst someValue = kv.get({ key: 'some-key' });\n```\n\n#### `kv.set({ key, value })`\n\n```javascript\nawait kv.set('some-key', 'some value');\n\n// or...\n\nawait kv.set({\n key: 'some-key',\n value: 'some value',\n});\n```\n\n#### `kv.expire({ key, ttl })`\n\nThis key will persist for 20 minutes, even if the server restarts.\n\n```javascript\nkv.expire({\n key: 'some-key',\n ttl: 1000 * 60 * 20, // 20 minutes\n});\n```\n\n### `kv.expireAt({ key, timestamp })`\n\nThe following example expires a key 1 second before\n[\"the apocalypse\"](https://en.wikipedia.org/wiki/Year_2038_problem).\n(don't worry, KV won't break in 2038)\n\n```javascript\nkv.expireAt(\n key: 'some-key',\n // Expires Jan 19 2038 3:14:07 GMT\n timestamp: 2147483647,\n);\n```\n\n### In-Memory Cache (`cache`)\n\nDon't forget to import it first!\n```javascript\nconst { cache } = extension.import('data');\n```\n\nThe in-memory cache is provided by [kv.js](https://github.com/HeyPuter/kv.js).\nBelow is a simple example.\nFor comprehensive documentation, see the [kv.js repository's readme](https://github.com/HeyPuter/kv.js/blob/main/README.md).\n\n```javascript\nconst { cache } = extension.require('data');\n\ncache.set('some-key', 'some value');\nconst value = cache.get('some-key'); // some value\n\n// This value only exists for 5 minutes\ncache.set('temporary', 'abcdefg', { EX: 5 * 60 });\n\ncache.incr('qwerty'); // cache.get('qwerty') is now: 1\ncache.incr('qwerty'); // cache.get('qwerty') is now: 2\n```\n"
},
{
"path": "src/backend/doc/extensions/pages/core-devs.md",
"content": "## Extensions - Technical Context for Core Devs\n\nThis document provides technical context for extensions from the perspective of\ncore backend modules and services, including the backend kernel.\n\n### Lifecycle\n\nFor extensions, the concept of an \"init\" event handler is different from core.\nThis is because a developer of an extension expects `init` to occur after core\nmodules and services have been initialized. For this reason, extensions receive\n`init` when backend services receive `boot.consolidation`.\n\nIt is still possible to handle core's `init` event in an extension. This is done\nusing the `preinit` event.\n\n```\nBackend Core Lifecycle\n Modules -> Construction -> Initialization -> Consolidation -> Activation -> Ready\nExtension Lifecycle\n index.js executed -> (no event) -> 'preinit' -> 'init' -> (no event) -> 'ready'\n```\n\nExtensions have an implicit Service instance that needs to listen for events on\nthe **Service Event Bus** such as `install.routes` (emitted by WebServerService).\nSince extensions need to affect the behavior of the service when these events\noccur (for example using `extension.post()` to add a POST handler) it is necessary\nfor their entry files to be loaded during a module installation phase, when\nservices are being registered and `_construct()` has not yet been called on any\nservice.\n\nKernel.js loads all core modules/services before any extensions. This allows\ncore modules and services to create [runtime modules](./runtime-modules.md)\nwhich can be imported by services.\n\n### How Extensions are Loaded\n\nBefore extensions are loaded, all of Puter's core modules have their `.install()`\nmethods called. The core modules are the ones added with `kernel.add_module`,\nfor example in [run-selfhosted.js](../../../../../tools/run-selfhosted.js).\n\nThen, `Kernel.install_extern_mods_` is called. This is where a `readdir` is\nperformed on each directory listed in the `\"mod_directories\"` configuration\nparameter, which has a default value of `[\"{repo}/extensions\"]` (the\nplaceholder `{repo}` is automatically replaced with the path to the Puter\nrepository).\n\nFor each item in each mod directory, except for ignored items like `.git`\ndirectories, a mod is installed. First a directory is created in Puter's\nruntime directory (`volatile/runtime` locally, `/var/puter` on a server).\nIf the item is a file then a `package.json` will be created for it after\n`//@extension` directives are processed. If the item is a directory then\nit is copied as is and `//@extension` directives are not supported\n(`puter.json` is used instead). Source files for the mod are copied to\nthe mod directory under the runtime directory.\n\nIt is at this point the pseudo-globals are added be prepending `cost`\ndeclarations at the top of `.js` files in the extension. This is not\na great way to do this, but there is a severe lack of options here.\nSee the heading below - \"Extension Pseudo-Globals\" - for details.\n\nBefore the entry file for the extension is `require()`'d a couple of\nobjects are created: an `ExtensionModule` and an `Extension`.\nThe `ExtensionModule` is a Puter module just like any of the Puter core\nmodules, so it has an `.install()` method that installs services before\nPuter's kernel starts the initialization sequence. In this case it will\ninstall the implied service that an extension creates if it registers\nroutes or performs any other action that's typically done inside services\nin core modules.\n\nA RuntimeModule is also created. This could be thought of as analygous\nto node's own `Module` class, but instead of being for imports/exports\nbetween npm modules it's for imports/exports between Puter extensions\nloaded at runtime. (see [runtime modules](./runtime-modules.md))\n\n### Extension Pseudo-Globals\n\nThe `extension` global is a different object per extension, which will\nmake it possible to develop \"remapping\" for imports/exports when\nextension names collide among other functions that need context about\nwhich extension is calling them. Implementing this per-extension global\nwas very tricky and many solutions were considered, including using the\n`node:vm` builtin module to run the extension in a different instance.\nUnfortunately `node:vm` support for EMCAScript Modules is lacking;\n`vm.Module` has a drastically different API from `vm.Script`, requires\nan experimental feature flag to be passed to node, and does not provide\nany alternative to `createRequire` to make a valid linker for the\ndependencies of a package being run in `node:vm`.\n\nThe current solution - which sucks - is as follows: prepend `const`\ndefinitions to the top of every `.js` file in the extension's installation\ndirectory unless it's under a directory called `node_modules` or `gui`.\nThis type of \"pseudo-global\" has a quirk when compared to real globals,\nwhich is that they can't be shadowed at the root scope without an error\nbeing thrown. The naive solution of wrapping the rest of the file's\ncontents in a scope limiter (`{ ... }`) would break ES Module support\nbecause `import` directives must be in the top-level scope, and the naive\nsolution to that problem of moving imports to the top of the file after\nadding the scope limiter requires invoking a javascript parser do\ndetermine the difference between a line starting with `import` because\nit's actually an import and this unholy abomination of a situation:\n```\nconsole.log(`\nimport { me, and, everything, breaks } from 'lackOfLexicalAnalysis';\n`);\n```\n\nExposing the same instance for `extension` to all extensions with a\nreal global and using AsyncLocalStorage to get the necessary information\nabout the calling extension on each of `extension`'s methods was another\nidea. This would cause surprising behavior for extension developers when\ncalling methods on `extension` in callbacks that lose the async context\nfail because of missing extension information.\n\nEventually a better compromise will be to have commonjs extensions\nrun using `vm.Script` and ESM extensions continue to run using this hack.\n\n### Event Listener Sub-Context\n\nIn extensions, event handlers are registered using `extension.on`. These\nhandlers, when called, are supplemented with identifying information for\nthe extension through AsyncLocalStorage. This means any methods called\non the object passed from the event (usually just called `event`) will\nbe able to access the extension's name.\n\nThis is used by CommandService's `create.commands` event. For example\nthe following extension code will register the command `utils:say-hello`\nif it is invoked form an extension named `utils`:\n\n```javascript\nextension.on('create.commands', event => {\n event.createCommand('say-hello', async (args, console) => {\n console.log('Hello,', ...args);\n });\n});\n```\n"
},
{
"path": "src/backend/doc/extensions/pages/drivers.md",
"content": "## Extensions - Implementing Drivers\n\nPuter's concept of drivers has existed long before the extension system\nwas refined, and to keep things moving forward it has become easier to\ndevelop Puter drivers in extensions than anywhere else in Puter's source.\nIf you want to build a driver, an extension is the recommended way to do it.\n\n### What are Puter drivers?\n\nPuter drivers are all called through the `/drivers/call` endpoint, so they\ncan be thought of as being \"above\" the HTTP layer. When a method on a driver\nthrows an error you will still receive a `200` HTTP status response because\nthe the invocation - from the HTTP layer - was successful.\n\nA driver response follows this structure:\n```json\n{\n \"success\": true,\n \"service\": {\n \"name\": \"implementation-name\"\n },\n \"result\": \"any type of value goes here\",\n \"metadata\": {}\n}\n```\n\nThere exists an example driver called `hello-world`. This driver implements\na method called `greet` with the optional parameter `subject` which returns\na string greeting either `World` (default) or the specified subject.\n\n```javascript\nawait puter.call('hello-world', 'no-frills', 'greet', { subject: 'Dave' });\n```\n\nLet's break it down:\n\n#### `'hello-world'`\n\n`'hello-world'` is the name of an \"interface\". An interface can be thought of\na contract of what inputs are allowed and what outputs are expected. For\nexample the `hello-world` interface specifies that there must be a method\ncalled `greet` and it should return a string representing a greeting.\n\nTo add another example, an interface called `weather` specify a method called\n`forcast5day` that always returns a list of 5 objects with a particular\nstructure.\n\n#### `no-frills`\n\n`'no-frills'` is a simple - \"no frills\" (nothing extra) - implementation of\nthe `hello-world` interface. All it does is return the string:\n```javascript\n`Hello, ${subject ?? 'World'}!`\n```\n\n\n#### `'greet'`\n\n`greet` is the method being called. It's the only method on the `hello-world`\ninterface.\n\n#### `{ subject: 'Dave' }`\n\nThese are the arguments to the `greet` method. The arguments specify that we\nwant to say \"Hello\" to Dave. Hopefully he doesn't ask us to open the pod bay\ndoors, or if he does we hopefully have extensions to add a driver interface\nand driver implementation for the pod bay doors so that we can interact with\nthem.\n\n### Drivers in Extensions\n\nThe `hellodriver` extension adds the `hello-world` interface like this:\n```javascript\nextension.on('create.interfaces', event => {\n // createInterface is the only method on this `event`\n event.createInterface('hello-world', {\n description: 'Provides methods for generating greetings',\n methods: {\n greet: {\n description: 'Returns a greeting',\n parameters: {\n subject: {\n type: 'string',\n optional: true\n },\n locale: {\n type: 'string',\n optional: true\n },\n }\n }\n }\n })\n});\n```\n\nThe `hellodriver` extension adds the `no-frills` implementation for\n`hello-world` like this:\n```javascript\nextension.on('create.drivers', event => {\n event.createDriver('hello-world', 'no-frills', {\n greet ({ subject }) {\n return `Hello, ${subject ?? 'World'}!`;\n }\n });\n});`\n```\n\nYou can pass an instance of a class for a driver implementation as well:\n```javascript\nclass Greeter {\n greet ({ subject }) {\n return `Hello, ${subject ?? 'World'}!`;\n }\n}\n\nextension.on('create.drivers', event => {\n event.createDriver('hello-world', 'no-frills', new Greeter());\n});`\n```\n\nInstances of classes being supported\nmay seem to be implied by the example before this\none, but that is not the case. What's shown here is that function members\nof the object passed to `createDriver` will not be \"bound\" (have their\n`.bind()` method called with a different object as the instance variable).\n\n### Permission Denied\n\nWhen you try to access a driver as any user other than the default\n`admin` user, it will not work unless permission has been granted.\n\nThe `hellodriver` extension grants permission to all clients using\nthe following snippet:\n```javascript\nextension.on('create.permissions', event => {\n event.grant_to_everyone('service:no-frills:ii:hello-world');\n});\n```\n\nThe `create.permissions` event's `event` object has a few methods\nyou can use depending on the desired granularity:\n- `grant_to_everyone` - grants permission to all users\n- `grant_to_users` - grants permission to only registered users\n (i.e. not to temporary/guest users)\n"
},
{
"path": "src/backend/doc/extensions/pages/import-and-export.md",
"content": "## Extensions - Importing & Exporting\n\nHere are two extensions. One extension has an \"extension export\" (an export to\nother extensions) and an \"extension import\" (an import from another extension).\nThis is different from regular `import` or `require()` because it resolves to\na Puter extension loaded at runtime rather than an `npm` module.\n\nTo import and export in Puter extensions, we use `extension.import()` and `extension.exports`.\n\n`exports-something.js`\n```javascript\n//@puter priority -1\n// ^ setting load priority to \"-1\" allows other extensions to import\n// this extension's exports before the initialization event occurs\n\n// Just like \"module.exports\", but for extensions!\nextension.exports = {\n test_value: 'Hello, extensions!',\n};\n```\n\n`imports-something.js`\n```javascript\nconst { test_value } = extension.import('exports-something');\n\nconsole.log(test_value); // 'Hello, extensions!'\n```\n\n"
},
{
"path": "src/backend/doc/extensions/pages/runtime-modules.md",
"content": "## Extensions - Runtime Modules\n\nRuntime modules are modules that extensions can import with tihs syntax:\n\n```javascript\nconst somelib = extension.import('somelib');\n```\n\nThese modules are registered in the [runtime module registry](../../../src/extension/RuntimeModuleRegistry.js)\nwhich is instantiated by [Kernel.js](../../../src/Kernel.js).\n\nAll extensions implicitly have a Runtime Module. The runtime module shares the name\nof the extension that it corresponds to. Extensions can export to their module by\nusing `extension.exports`:\n\n```javascript\nextension.exports = { /* ... */ };\n```\n\nThe [Extension](../../../src/Extension.js) object proxies this call to the\nruntime module (called `this.runtime` in the snippet):\n\n```javascript\nclass Extension extends AdvancedBase {\n // ...\n set exports (value) {\n this.runtime.exports = value;\n }\n // ...\n}\n```\n\nYou may be wondering why RuntimeModule is a separate class from Extension,\nrather than just registering extensions into this registry.\n\nSeparating RuntimeModule allows core code that has not yet been migrated\nto extensions to export values as if they came from extensions.\nSince core modules are loaded before extensions, this allows any legacy\n`useapi` definitions be be exported where modules are installed.\n\nFor example, in [CoreModule.js](../../../src/CoreModule.js) this snippet\nof code is used to add a runtime module called `core`:\n\n```javascript\n// Extension compatibility\nconst runtimeModule = new RuntimeModule({ name: 'core' });\ncontext.get('runtime-modules').register(runtimeModule);\nruntimeModule.exports = useapi.use('core');\n```\n"
},
{
"path": "src/backend/doc/features/batch-and-symlinks.md",
"content": "# Batch and Symlinks\n\n2024-10-08\n\n### Batch and Symlinks\n\nAll filesystem operations will eventually be available through batch requests.\nSince batch requests can also handle the cases for single files, it seems silly\nto support those endpoints too, so eventually most calls will be done through\n`/batch`. Puter's legacy filesystem endpoints will always be supported, but a\nfuture `api.___/fs/v2.0` urlspace for the filesystem API might not include them.\n\nThis is batch:\n\n```javascript\nawait (async () => {\n const endpoint = 'http://api.puter.localhost:4100/batch';\n\n const ops = [ \n {\n op: 'mkdir',\n path: '/default_user/Desktop/some-dir',\n },\n {\n op: 'write',\n path: '/default_user/Desktop/some-file.txt',\n }\n ];\n\n const blob = new Blob([\"12345678\"], { type: 'text/plain' });\n const formData = new FormData();\n for ( const op of ops ) {\n formData.append('operation', JSON.stringify(op));\n }\n formData.append('fileinfo', JSON.stringify({\n name: 'file.txt',\n size: 8,\n mime: 'text/plain',\n }));\n formData.append('file', blob, 'hello.txt');\n\n const response = await fetch(endpoint, {\n method: 'POST',\n headers: { 'Authorization': `Bearer ${puter.authToken}` },\n body: formData\n });\n return await response.json();\n})();\n```\nSymlinks are also created via `/batch`\n\n```javascript\nawait (async () => {\n const endpoint = 'http://api.puter.localhost:4100/batch';\n\n const ops = [ \n {\n op: 'symlink',\n path: '~/Desktop',\n name: 'link',\n target: '/bb/Desktop/some'\n },\n ];\n\n const formData = new FormData();\n for ( const op of ops ) {\n formData.append('operation', JSON.stringify(op));\n }\n\n const response = await fetch(endpoint, {\n method: 'POST',\n headers: { 'Authorization': `Bearer ${puter.authToken}` },\n body: formData\n });\n return await response.json();\n})();\n```\n"
},
{
"path": "src/backend/doc/features/protected-apps.md",
"content": "# Protected Apps and Subdomains\n\n## Protected Sites\n\nIf a site is not protected, anyone can access the site.\nWhen a site is protected, the following changes:\n\n- The site can only be accessed inside a Puter app iframe\n- Only users with explicit permission will be able to load\n the page associated with the site.\n\n## Protected Apps\n\nIf an app is not protected, anyone with the name of the\napp or its UUID will be able to access the app.\nIf the app is **approved for listing** (todo: doc this)\nall users can access the app.\nIf an app is protected, the following changes:\n\n- The app can only be \"seen\" (listed) by users\n with explicit permission.\n- App metadata can only be accessed by users\n with explicit permission.\n\nNote that an app being protected does not imply that the\nsite is protected. If a user action results in an app\nbeing protected it should also result in the site (subdomain)\nbeing protected **if they own it**. If the site will not\nbe protected the user should have some indication.\n"
},
{
"path": "src/backend/doc/features/service-scripts.md",
"content": "> **NOTICE:** This documentation is new and might contain errors.\n> Feel free to open a Github issue if you run into any problems.\n\n# Service Scripts\n\n## What is a Service Script?\n\nService scripts allow backend services to provide client-side code that\nruns in Puter's GUI. This is useful if you want to make a mod or plugin\nfor Puter that has backend functionality. For example, you might want\nto add a tab to the settings panel to make use of or configure the service.\n\nService scripts are made possible by the `puter-homepage` service, which\nallows you to register URLs for additional javascript files Puter's\nGUI should load.\n\n## ES Modules - A Problem of Ordering\n\nIn browsers, script tags with `type=module` implicitly behave according\nto those with the `defer` attribute. This means after the DOM is loaded\nthe scripts will run in the order in which they appear in the document.\n\nRelying on this execution order however does not work. This is because\n`import` is implicitly asynchronous. Effectively, this means these\nscripts will execute in arbitrary order if they all have imports.\n\nIn a situation where all the client-side code is bundled with rollup\nor webpack this is not an issue as you typically only have one\nentry script. To facilitate loading service scripts, which are not\nbundled with the GUI, we require that service scripts call the global\n`service_script` function to access the API for service scripts.\n\n## Providing a Service Script\n\nFor a service to provide a service script, it simply needs to serve\nstatic files (the \"service script\") on some URL, and register that\nURL with the `puter-homepage` service.\n\nIn this example below we use builtin functionality of express to serve\nstatic files.\n\n```javascript\nclass MyService extends BaseService {\n async _init () {\n // First we tell `puter-homepage` that we're going to be serving\n // a javascript file which we want to be included when the GUI\n // loads.\n const svc_puterHomepage = this.services.get('puter-homepage');\n svc_puterHomepage.register_script('/my-service-script/main.js');\n }\n\n async ['__on_install.routes'] (_, { app }) {\n // Here we ask express to serve our script. This is made possible\n // by WebServerService which provides the `app` object when it\n // emits the 'install.routes` event.\n app.use('/my-service-script',\n express.static(\n PathBuilder.add(__dirname).add('gui').build()\n )\n );\n }\n}\n```\n\n## A Simple Service Script\n\n\n\n```javascript\nimport SomeModule from \"./SomeModule.js\";\n\nservice_script(api => {\n api.on_ready(() => {\n // This callback is invoked when the GUI is ready\n\n // We can use api.get() to import anything exposed to\n // service scripts by Puter's GUI; for example:\n const Button = api.use('ui.components.Button');\n // ^ Here we get Puter's Button component, which is made\n // available to service scripts.\n });\n});\n```\n\n## Adding a Settings Tab\n\nStarting with the following example: \n\n```javascript\nimport MySettingsTab from \"./MySettingsTab.js\";\n\nglobalThis.service_script(api => {\n api.on_ready(() => {\n const svc_settings = globalThis.services.get('settings');\n svc_settings.register_tab(MySettingsTab(api));\n });\n});\n```\n\nThe module **MySettingsTab** exports a function for scoping the `api`\nobject, and that function returns a settings tab. The settings tab is\nan object with a specific format that Puter's settings window understands.\n\nHere are the contents of `MySettingsTab.js`:\n\n```javascript\nimport MyWindow from \"./MyWindow.js\";\n\nexport default api => ({\n id: 'my-settings-tab',\n title_i18n_key: 'My Settings Tab',\n icon: 'shield.svg',\n factory: () => {\n const NotifCard = api.use('ui.component.NotifCard');\n const ActionCard = api.use('ui.component.ActionCard');\n const JustHTML = api.use('ui.component.JustHTML');\n const Flexer = api.use('ui.component.Flexer');\n const UIAlert = api.use('ui.window.UIAlert');\n\n // The root component for our settings tab will be a \"flexer\",\n // which by default displays its child components in a vertical\n // layout.\n const component = new Flexer({\n children: [\n // We can insert raw HTML as a component\n new JustHTML({\n no_shadow: true, // use CSS for settings window\n html: 'Some Heading ',\n }),\n new NotifCard({\n text: 'I am a card with some text',\n style: 'settings-card-success',\n }),\n new ActionCard({\n title: 'Open an Alert',\n button_text: 'Click Me',\n on_click: async () => {\n // Here we open an example window\n await UIAlert({\n message: 'Hello, Puter!',\n });\n }\n })\n ]\n });\n\n return component;\n }\n});\n```\n"
},
{
"path": "src/backend/doc/howto_make_driver.md",
"content": "# How to Make a Puter Driver\n\n## What is a Driver?\n\nA driver can be one of two things depending on what you're\ntalking about:\n- a **driver interface** describes a general type of service\n and what its parameters and result look like.\n For example, `puter-chat-completion` is a driver interface\n for AI Chat services, and it specifies that any service\n on Puter for AI Chat needs a method called `complete` that\n accepts a JSON parameter called `messages`.\n- a **driver implementation** exists when a **Service** on\n Puter implements a **trait** with the same name as a\n driver interface.\n\n## Part 1: Choose or Create a Driver Interface\n\nAvailable driver interfaces exist at this location in the repo:\n[/src/backend/src/services/drivers/interfaces.js](../src/services/drivers/interfaces.js).\n\nWhen creating a new Puter driver implementation, you should check\nthis file to see if there's an appropriate interface. We're going\nto make a driver that returns greeting strings, so we can use the\nexisting `hello-world` interface. If there wasn't an existing\ninterface, it would need to be created. Let's break down this\ninterface:\n\n```javascript\n'hello-world': {\n description: 'A simple driver that returns a greeting.',\n methods: {\n greet: {\n description: 'Returns a greeting.',\n parameters: {\n subject: {\n type: 'string',\n optional: true,\n },\n },\n result: { type: 'string' },\n }\n }\n},\n```\n\nThe **description** describes what the interface is for. This\nshould be provided that both driver developers and users can\nquickly identify what types of services should use it.\n\nThe **methods** object should have at least one entry, but it\nmay have more. The key of each entry is the name of a method;\nin here we see `greet`. Each method also has a description,\na **parameters** object, and a **result** object.\n\nThe **parameters** object has an entry for each parameter that\nmay be passed to the method. Each entry is an object with a\n`type` property specifying what values are allowed, and possibly\nan `optional: true` entry.\n\nAll methods for Puter drivers use _named parameters_. There are no\npositional parameters in Puter driver methods.\n\nThe **result** object specifies the type of the result. A service\ncalled DriverService will use this to determine the response format\nand headers of the response.\n\n## Part 2: Create a Service\n\nCreating a service is very easy, provided the service doesn't do\nanything. Simply add a class to `src/backend/src/services` or into\nthe module of your choice (`src/backend/src/modules/`)\nthat looks like this:\n\n```javascript\nconst BaseService = require('./BaseService')\n// NOTE: the path specified ^ HERE might be different depending\n// on the location of your file.\n\nclass PrankGreetService extends BaseService {\n}\n```\n\nNotice I called the service \"PrankGreet\". This is a good service\nname because you already know what the service is likely to\nimplement: this service generates a greeting, but it is a greeting\nthat intends to play a prank on whoever is beeing greeted.\n\nThen, register the service into a module. If you put the service\nunder `src/backend/src/services`, then it goes in\n[CoreModule](..//src/CoreModule.js) somewhere near the end of\nthe `install()` method. Otherwise, it will go in the `*Module.js`\nfile in the module where you placed your service.\n\nThe code to register the service is two lines of code that will\nlook something like this:\n\n```javascript\nconst { PrankGreetServie } = require('./path/to/PrankGreetServie.js');\nservices.registerService('prank-greet', PrankGreetServie);\n```\n\n## Part 3: Verify that the Service is Registered\n\nIt's always a good idea to verify that the service is loaded\nwhen starting Puter. Otherwise, you might spend time trying to\ndetermine why your code doesn't work, when in fact it's not\nrunning at all to begin with.\n\nTo do this, we'll add an `_init` handler to the service that\nlogs a message after a few seconds. We wait a few seconds so that\nany log noise from boot won't bury our message.\n\n```javascript\nclass PrankGreetService extends BaseService {\n async _init () {\n // Wait for 5 seconds\n await new Promise(rslv => setTimeout(rslv), 5000);\n\n // Display a log message\n console.debug('Hello from PrankGreetService!');\n }\n}\n```\n\n## Part 4: Implement the Driver Interface in your Service\n\nNow that it has been verified that the service is loaded, we can\nstart implementing the driver interface we chose eralier.\n\n```javascript\nclass PrankGreetService extends BaseService {\n async _init () {\n // ... same as before\n }\n\n // Now we add this:\n static IMPLEMENTS = {\n ['hello-world']: {\n async greet ({ subject }) {\n if ( subject ) {\n return `Hello ${subject}, tell me about updog!`;\n }\n return `Hello, tell me about updog!`;\n }\n }\n }\n}\n```\n\n## Part 5: Test the Driver Implementation\n\nWe have now created the `prank-greet` implementation of `hello-world`.\nLet's make a request in the browser to check it out. The example below\nis a `fetch` call using `http://api.puter.localhost:4100` as the API\norigin, which is the default when you're running Puter's backend locally.\n\nAlso, in this request I refer to `puter.authToken`. If you run this\nsnippet in the Dev Tools window of your browser from a tab with Puter\nopen (your local Puter, to be precise), this should contain the current\nvalue for your auth token.\n\n```javascript\nawait (await fetch(\"http://api.puter.localhost:4100/drivers/call\", {\n \"headers\": {\n \"Content-Type\": \"application/json\",\n \"Authorization\": `Bearer ${puter.authToken}`,\n },\n \"body\": JSON.stringify({\n interface: 'hello-world',\n service: 'prank-greet',\n method: 'greet',\n args: {\n subject: 'World',\n },\n }),\n \"method\": \"POST\",\n})).json();\n```\n\n**You might see a permissions error!** Don't worry, this is expected;\nin the next step we'll add the required permissions.\n\n## Part 6: Permissions\n\nIn the previous step, you will only have gotten a successful response\nif you're logged in as the `admin` user. If you're logged in as another\nuser you won't have access to the service's driver implementations be\ndefault.\n\nTo grant permission for all users, update\n[hardcoded-permissions.js](../src/data/hardcoded-permissions.js).\n\nFirst, look for the constant `hardcoded_user_group_permissions`.\nWhereever you see an entry for `service:hello-world:ii:hello-world`, add\nthe corresponding entry for your service, which will be called\n```\nservice:prank-greet:ii:hello-world\n```\n\nTo help you remember the permission string, its helpful to know that\n`ii` in the string stands for \"invoke interface\". i.e. the scope of the\npermission is under `service:prank-greet` (the `prank-greet` service)\nand we want permission to invoke the interface `hello-world` on that\nservice.\n\nYou'll notice each entry in `hardcoded_user_group_permissions` has a value\ndetermined by a call to the utility function `policy_perm(...)`. The policy\ncalled `user.es` is a permissive policy for storage drivers, and we can\nre-purpose it for our greeting implementor.\n\nThe policy of a permission determines behavior like rate limiting. This is\nan advanced topic that is not covered in this guide.\n\nIf you want apps to be able to access the driver implementation without\nexplicit permission from a user, you will need to also register it in the\n`default_implicit_user_app_permissions` constant. Additionally, you can\nuse the `implicit_user_app_permissions` constant to grant implicit\npermission to the builtin Puter apps only.\n\nPermissions to implementations on services can also be granted at runtime\nto a user or group of users using the permissions API. This is beyond the\nscope of this guide.\n\n## Part 7: Verify Successful Response\n\nIf all went well, you should see the response in your console when you\ntry the request from Part 5. Try logging into a user other than `admin`\nto verify permisison is granted.\n\n```json\n\"Hello World, tell me about updog!\"\n```\n\n## Part 8: Next Steps\n\n- [Access Configuration](./services/config.md)\n- [Output Logs](./services/log.md)\n- [Add HTTP Routes](./services/http.md)\n"
},
{
"path": "src/backend/doc/license_header.txt",
"content": "Copyright (C) 2024 Puter Technologies Inc.\n\nThis file is part of Puter.\n\nPuter is free software: you can redistribute it and/or modify\nit under the terms of the GNU Affero General Public License as published\nby the Free Software Foundation, either version 3 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\nGNU Affero General Public License for more details.\n\nYou should have received a copy of the GNU Affero General Public License\nalong with this program. If not, see :`\n\n- `` specifies the file that this permission\n is associated with.\n The ACL service\n (which checks filesystem permissions)\n knows if the value is a path or UUID based on the presence\n of a leading slash; if it starts with `\"/\"` it's a path.\n- `` specifies one of:\n `write`, `read`, `list`, `see`; where each item in that\n list implies all the access levels which follow.\n- A permission that grants access to a directory,\n such as `/user/shared`, implies access\n of the same **access level** to all child file or directory\n nodes under that location, **recursively**;\n `fs:/user/shared:read` implies `fs:/user/shared/nested/file.txt:read`\n- The \"real\" permission is `fs::`;\n whenever path is specified the permission is rewritten.\n **note:** future support for other filesystems\n could make this rewrite rule conditional.\n\n## App and Subdomain permissions\n\n### `site::access`\n- `` specifies the subdomain that this\n permission is associated with.\n Here, \"subdomain\" means the **\"name of the subdomain\"**,\n which means a site accessed via `my-name.example.site`\n will be specified here with `my-name`.\n- This permission is always rewritten as the permission\n described below (backend does this automatically).\n\n### `site:uid#:access`\n- If the subdomain is **not** [protected](../features/protected-apps.md),\n this permission is ignored by the system.\n- If the subdomain **is** protected, this permission will\n allow access to the site via a Puter app iframe with\n a token for the entity to which permission was granted\n\n### `app::access`\n\n- `` specifies the app that this\n permission is associated with.\n- This permission is always rewritten as the permission\n described below (backend does this automatically).\n \n### `app:uid#:access`\n- If the app is **not** [protected](../features/protected-apps.md),\n this permission is ignored by the system.\n- If the app **is** protected, this permission will\n allow reading the app's metadata and seeing that the app exists.\n"
},
{
"path": "src/backend/doc/lists-of-things/list-of-tto-types.md",
"content": "# Types for Type-Tagged Objects\n\n## Internal Use\n\n### `{ $: 'share-intent' }`\n\n- Used in the `/share` endpoint\n- Permissions get applied to existing users\n- For email shares, is trasnformed into a `token:share`\n which is stored in the `share` database table.\n\n- **variants:**\n - `share-intent:file`\n - `share-intent:app`\n- **properties:**\n - `permissions` - a list of permissions to grant\n \n### `{ $: 'internal:share' }`\n- Stored in the `share` database table\n- **properties:**\n - `permissions` - a list of permissions to grant\n\n### `{ $: 'token:share }`\n\n- Stored in a JWT called the \"share token\"\n- Contains only the share UUID\n\n- **properties:**\n - `uid` - UUID of a share\n"
},
{
"path": "src/backend/doc/log_config.md",
"content": "## Backend - Configuring Logs\n\n### Log visibility specified by configuration file\n\nThe configuration file can define an array parameter called `logging`.\nThis configures the visibility of specific logs in core areas based on\nwhich string flags are present.\n\nFor example, the following configuration enables HTTP request logs:\n```json\n{\n \"logging\": ['http']\n}\n```\n\nSometimes \"enabling\" a log means moving its log level from `debug` to `info`.\n\n#### Available logging flags:\n- `http`: http requests\n- `fsentries-not-found`: information about files that were stat'd but weren't there\n\n#### Other log options\n\n- Setting `log_upcoming_alarms` to `true` will log alarms before they are created.\n This would be useful if AlarmService itself is failing.\n- Setting `trace_logs` to `true` will display a stack trace below every log message.\n This can be useful if you don't know where a particular log is coming from and\n want to track it down.\n\n#### Service-level log configuration\n\nServices can be configured to change their logging behavior. Services will have one of\ntwo behaviors:\n\n1. **info logging** - `log.info` can be used to create an `[INFO]` log message\n2. **debug logging only** - `log.info` is redirected to `log.debug`\n\nServices will have **info logging** enabled by default, unless the class definition\nhas the static member `static LOG_DEBUG = true` (in which case **debug logging only**\nis the default).\n\nIn a service's configuration block the desired behavior can be specified by setting\neither `\"log_debug\": true` or `\"log_info\": true`\n"
},
{
"path": "src/backend/doc/modules/filesystem/API_SPEC.md",
"content": "# Filesystem API\n\nFilesystem endpoints allow operations on files and directories in the Puter filesystem.\n\n## POST `/mkdir` (auth required)\n\n### Description\n\nCreates a new directory in the filesystem. Currently support 2 formats:\n\n- Full path: `{\"path\": \"/foo/bar\", args ...}` — this API is used by apitest (`./tools/api-tester/apitest.js`) and aligns more closely with the POSIX spec (https://linux.die.net/man/3/mkdir)\n- Parent + path: `{\"parent\": \"/foo\", \"path\": \"bar\", args ...}` — this API is used by `puter-js` via `puter.fs.mkdir`\n\nA future work would be use a unified format for all filesystem operations.\n\n### Parameters\n\n- **path** _- required_\n - **accepts:** `string`\n - **description:** The path where the directory should be created\n - **notes:** Cannot be empty, null, or undefined\n\n- **parent** _- optional_\n - **accepts:** `string | UUID`\n - **description:** The parent directory path or UUID\n - **notes:** If not provided, path is treated as full path\n\n- **overwrite** _- optional_\n - **accepts:** `boolean`\n - **default:** `false`\n - **description:** Whether to overwrite existing files/directories\n\n- **dedupe_name** _- optional_\n - **accepts:** `boolean`\n - **default:** `false`\n - **description:** Whether to automatically rename if name exists\n\n- **create_missing_parents** _- optional_\n - **accepts:** `boolean`\n - **default:** `false`\n - **description:** Whether to create parent directories if they don't exist\n - **aliases:** `create_missing_ancestors`\n\n- **shortcut_to** _- optional_\n - **accepts:** `string | UUID`\n - **description:** Creates a shortcut/symlink to the specified target\n\n### Example\n\n```json\n{\n \"path\": \"/user/Desktop/new-directory\"\n}\n```\n\n```json\n{\n \"parent\": \"/user\",\n \"path\": \"Desktop/new-directory\"\n}\n```\n\n### Response\n\nReturns the created directory's metadata including name, path, uid, and any parent directories created.\n\n## Other Filesystem Endpoints\n\n[Additional endpoints would be documented here...] "
},
{
"path": "src/backend/doc/modules/puterai/README.md",
"content": "# PuterAI Module\n\nThe PuterAI module provides AI capabilities to Puter through various services including:\n\n- Text generation and chat completion\n- Text-to-speech synthesis\n- Image generation\n- Document analysis\n\n## Metered Services\n\nAll AI services in this module are metered using Puter's MeteringService. This allows us to charge per `unit` usage, where a `unit` is defined by the specific service:\nfor example, most LLMs will charge per token, AWS Polly charges per character, and AWS Textract charges per page. the metering service tracks usage units, and relies on its centralized cost maps to determine if a user has enough credits to perform an operation, and to record usage after the operation is complete.\n\nsee [MeteringService](../../../src/services/MeteringService/MeteringService.ts) for more details on how metering works."
},
{
"path": "src/backend/doc/notes/2024-10-03_email_in_use_checks.md",
"content": "## 2024-10-03\n\n### Plan (constantly changing as per what's below)\n\n- `signup.js` only says \"email already used\" if the one that's\n already been used is confirmed.\n- \"change email\" needs to follow the same logic; show an error when\n an email already exists on an account with a confirmed email.\n Then, upon confirming the update, Ensure that in the meanwhile no\n new account came up with that email set.\n- ensure `clean_email` is updated whenever the email is updated\n\n### Email duplicate check on confirmation\n\n- signup.js:149 -> this is where email dupe is currently checked\n- signup.js:290 -> This is where we send the confirmation email.\n There is also a branch that sends a \"confirm token\".\n I don't recall what this is for.\n\n### Investigating the \"confirm token\"\n\n- email template is `email_verification_code`\n instead of `email_verification_link`\n- This happens when either:\n - user.requires_email_confirmation is TRUE\n - send_confirmation_code is TRUE in REQUEST\n\n### Figuring out when `requires_email_confirmation` is TRUE\n\nI'm mostly curious about this state on a user.\nIt's strange that `signup.js` would do anything on EXISTING users.\n\n1. `pseudo_user` may be populated if `req.body.email` exists\n AND a user with no password exists with that email\n2. `uuid_user` may be populated if a user exists with the specified\n UUID, but it has no usefulness unless `uuid_user` has the same\n id as `pseudo_user`.\n \n`uuid_user` is only used to set `email_confirmation_required` to 0\n IFF `pseudo_user` has same id as `uuid_user`\n AND `psuedo_user` has an email\n\nWhen does `pseudo_user` have an email?\n\n### Figuring out when a pseudo user can have an email\n- asking NJ, I'm at a loss on this one for the moment\n\n### Figuring out if account takeover is possible on signup.js with a uuid\n- Nope, looks like `uuid_user` is only used to set\n `email_confirmation_required = 0`\n\n### Figuring out when `send_confirmation_code` is TRUE in REQUEST\n- IFF `require_email_verification_to_publish_website` is TRUE\n - it's not currently, but we need this to be possible to enable\n- ^ That seems to be the ONLY place when this matters\n\n### Current Thoughts\n\n- `email_verification_code` will be difficult to test because there is\n nothing currently in the system that's using it. However, I could try\n enabling `require_email_verification_to_publish_website` locally and\n see if this behavior begins to work as expected.\n\n- `email_verification_link` where we can confirm an email. If another email\n was already confirmed since the time the link was sent, we need to display\n an error message to the user.\n\n### Find places where (on backend) email change process is triggered\n\nRight now there are two handlers:\n- `/user-protected/change-email` (UserProtectedEndpointsService)\n - Invokes the process (sends confirmation email)\n- `/change_email/confirm` (PuterAPIService)\n - Endpoint that the email link points to\n"
},
{
"path": "src/backend/doc/services/config.md",
"content": "# Service Configuration\n\nTo locate your configuration file, see [Configuring Puter](https://github.com/HeyPuter/puter/wiki/self_hosters-config).\n\n### Accessing Service Configuration\n\nService configuration appears under the `\"services\"` property in the\nconfiguration file for Puter. If Puter's configuration had no other\nvalues except for a service config with one key, it might look like\nthis:\n\n```json\n{\n \"services\": {\n \"my-service\": {\n \"somekey\": \"some value\"\n }\n }\n}\n```\n\nServices have their configuration object assigned to `this.config`.\n\n```javascript\nclass MyService extends BaseService {\n async _init () {\n // You can access configuration for a service like this\n this.log.info('value of my key is: ' + this.config.somekey);\n }\n}\n```\n\n### Accessing Global Configuration\n\nServices can access global configuration. This can be useful for knowing how\nPuter itself is configured, but using this global config object for service\nconfiguration is discouraged as it could create conflicts between services.\n\n```javascript\nclass MyService extends BaseService {\n async _init () {\n // You can access configuration for a service like this\n this.log.info('Puter is hosted on: ' + this.global_config.domain);\n }\n}\n```\n"
},
{
"path": "src/backend/doc/services/event_buses.md",
"content": "# Event Buses\n\nPuter's backend has two event buses:\n- Service Event Bus\n- Application Event Bus\n\n## Service Event Bus\n\nThis is a simple event bus that lives in the [Container](../../src/services/Container.js)\nclass. There is only one instance of **Container** and it is called the \"services container\".\nWhen Puter boots, all the services registered by modules are registered into the services\ncontainer.\n\nServices handle events from the Service Event Bus by implementing methods which are named\nwith the prefix `__on_`. This prefix looks a little strange at first so it's worth\nbreaking it down:\n- `__` (two underscores) prevents collision with common method names, and also\n common conventions like beginning a method name with a single underscore\n to indicate a method that should be overridden.\n- `on` is the meaningful name.\n- `_`, the last underscore, is for readability, as the event name conventionally\n begins with a lowercase letter.\n \nNote that you will need to use the \n\nExample:\n```javascript\nclass MyService extends BaseService {\n ['__on_boot.ready'] () {\n //\n }\n}\n```\n"
},
{
"path": "src/backend/doc/services/http.md",
"content": "# Adding HTTP Routes to Services\n\nServices can serve HTTP routes when the [WebModule](../../src/modules/web/WebModule.js)\nis enabled by listening for the `install.routes` event on the [Service Event Bus](./)"
},
{
"path": "src/backend/doc/services/log.md",
"content": "# Logging in Services\n\n# NOTE: You can, and maybe should, just use console log methods, as they are overriden to log through our logger\n\nServices all have a logger available at `this.log`.\n\n```javascript\nclass MyService extends BaseService {\n async init () {\n this.log.info('Hello, Logger!');\n }\n}\n```\n\nThere are multiple \"log levels\", similar to `logrus` or other common logging\nlibraries.\n\n```javascript\nclass MyService extends BaseService {\n async init () {\n this.log.info('I\\'m just a regular log.');\n this.log.debug('I\\'m only for developers.');\n this.log.warn('It is statistically unlikely I will be awknowledged.');\n this.log.error('Something is broken! Pay attention!');\n this.log.noticeme('This will be noticed, unlike warnings. Use sparingly.');\n this.log.system('I am a system event, like shutdown.');\n this.log.tick('A periodic behavior like cache pruning is occurring.');\n }\n}\n```\n\nLog methods can take a second parameter, an object specifying fields.\n\n```javascript\n\nclass MyService extends BaseService {\n async init () {\n this.log.info('I have fields!', {\n why: \"why not\",\n random_number: 1, // chosen by coin toss, guarenteed to be random\n });\n }\n}\n```\n"
},
{
"path": "src/backend/exports.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } Resolves when legacy services are installed\n */\n async install_legacy (context) {\n const services = context.get('services');\n await install_legacy({ services });\n }\n}\n\nmodule.exports = CoreModule;\n"
},
{
"path": "src/backend/src/DatabaseModule.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see >();\n\nconst getDynalitePathKey = (path?: string) => {\n if ( path === ':memory:' ) return LOCAL_DYNAMO_PATH_KEY;\n return path || './puter-ddb';\n};\n\nconst getOrCreateLocalDynaliteEndpoint = async (pathKey: string) => {\n let endpointPromise = localDynaliteEndpointPromises.get(pathKey);\n if ( endpointPromise ) return endpointPromise;\n\n endpointPromise = (async () => {\n const dynaliteOptions = pathKey === LOCAL_DYNAMO_PATH_KEY\n ? { createTableMs: 0 }\n : { createTableMs: 0, path: pathKey };\n\n const dynaliteInstance = dynalite(dynaliteOptions);\n const dynaliteServer = dynaliteInstance.listen(0, '127.0.0.1');\n // Don't keep test workers alive just because dynalite is still open.\n dynaliteServer.unref?.();\n await once(dynaliteServer, 'listening');\n\n const address = dynaliteServer.address();\n const port = (typeof address === 'object' && address ? address.port : undefined) || 4567;\n return `http://127.0.0.1:${port}`;\n })();\n\n localDynaliteEndpointPromises.set(pathKey, endpointPromise);\n endpointPromise.catch(() => {\n if ( localDynaliteEndpointPromises.get(pathKey) === endpointPromise ) {\n localDynaliteEndpointPromises.delete(pathKey);\n }\n });\n return endpointPromise;\n};\n\nexport class DDBClient {\n ddbClientPromise: Promise;\n #documentClient!: DynamoDBDocumentClient;\n config?: DBClientConfig;\n\n constructor (config?: DBClientConfig) {\n this.config = config;\n this.ddbClientPromise = this.#getClient();\n this.ddbClientPromise.then(client => {\n this.#documentClient = DynamoDBDocumentClient.from(client, {\n marshallOptions: {\n removeUndefinedValues: true,\n } });\n });\n }\n\n async recreateClient () {\n this.ddbClientPromise = this.#getClient();\n this.#documentClient = DynamoDBDocumentClient.from(await this.ddbClientPromise, {\n marshallOptions: {\n removeUndefinedValues: true,\n } });\n }\n\n async #getClient () {\n if ( ! this.config?.aws ) {\n console.warn('No config for DynamoDB, will fall back on local dynalite');\n const pathKey = getDynalitePathKey(this.config?.path);\n const dynamoEndpoint = await getOrCreateLocalDynaliteEndpoint(pathKey);\n\n const client = new DynamoDBClient({\n credentials: {\n accessKeyId: 'fake',\n secretAccessKey: 'fake',\n },\n maxAttempts: 3,\n requestHandler: new NodeHttpHandler({\n connectionTimeout: 5000,\n requestTimeout: 5000,\n httpsAgent: new httpsAgent({ keepAlive: true }),\n }),\n endpoint: dynamoEndpoint,\n region: 'us-west-2',\n });\n console.log(`Dynalite client created within instance for region: ${await client.config.region()}`);\n return client;\n }\n\n const client = new DynamoDBClient({\n credentials: {\n accessKeyId: this.config.aws.access_key,\n secretAccessKey: this.config.aws.secret_key,\n },\n maxAttempts: 3,\n requestHandler: new NodeHttpHandler({\n connectionTimeout: 5000,\n requestTimeout: 5000,\n httpsAgent: new httpsAgent({ keepAlive: true }),\n }),\n ...(this.config.endpoint ? { endpoint: this.config.endpoint } : {}),\n region: this.config.aws.region || 'us-west-2',\n });\n console.log(`DynamoDB client created with region ${await client.config.region()}`);\n return client;\n }\n\n async get >(table: string, key: T, consistentRead = false) {\n const command = new GetCommand({\n TableName: table,\n Key: key,\n ConsistentRead: consistentRead,\n ReturnConsumedCapacity: 'TOTAL',\n });\n\n const response = await this.#documentClient.send(command);\n\n return response;\n }\n\n async put >(table: string, item: T) {\n const command = new PutCommand({\n TableName: table,\n Item: item,\n ReturnConsumedCapacity: 'TOTAL',\n });\n\n const response = await this.#documentClient.send(command);\n return response;\n }\n\n async batchGet (params: { table: string, items: Record }[], consistentRead = false) {\n // TODO DS: implement chunking for more than 100 items or more than allowed req size\n const allRequestItemsPerTable = params.reduce((acc, curr) => {\n if ( ! acc[curr.table] ) acc[curr.table] = [];\n acc[curr.table].push(curr.items);\n return acc;\n }, {} as Record[]>);\n\n const RequestItems: BatchGetCommandInput['RequestItems'] = Object.entries(allRequestItemsPerTable).reduce(\n (acc, [table, keyList]) => {\n const Keys = keyList;\n acc[table] = {\n Keys,\n ConsistentRead: consistentRead,\n };\n return acc;\n },\n {} as NonNullable,\n );\n\n const command = new BatchGetCommand({\n RequestItems,\n ReturnConsumedCapacity: 'TOTAL',\n });\n\n return this.#documentClient.send(command);\n }\n\n async del> (table: string, key: T) {\n const command = new DeleteCommand({\n TableName: table,\n Key: key,\n ReturnConsumedCapacity: 'TOTAL',\n });\n\n return this.#documentClient.send(command);\n }\n\n async query> (\n table: string,\n keys: T,\n limit = 0,\n pageKey?: Record,\n index = '',\n consistentRead = false,\n options?: { beginsWith?: { key: string; value: string } },\n ) {\n\n const keyExpressionParts = Object.keys(keys).map(key => `#${key} = :${key}`);\n const expressionAttributeValues = Object.entries(keys).reduce((acc, [key, value]) => {\n acc[`:${key}`] = value;\n return acc;\n }, {});\n const expressionAttributeNames = Object.keys(keys).reduce((acc, key) => {\n acc[`#${key}`] = key;\n return acc;\n }, {});\n\n if ( options?.beginsWith?.key && typeof options.beginsWith.value === 'string' && options.beginsWith.value !== '' ) {\n const beginsKey = options.beginsWith.key;\n const beginsValueToken = `:${beginsKey}_begins_with`;\n keyExpressionParts.push(`begins_with(#${beginsKey}, ${beginsValueToken})`);\n expressionAttributeValues[beginsValueToken] = options.beginsWith.value;\n expressionAttributeNames[`#${beginsKey}`] = beginsKey;\n }\n\n const keyExpression = keyExpressionParts.join(' AND ');\n\n const command = new QueryCommand({\n TableName: table,\n ...(!index ? {} : { IndexName: index }),\n KeyConditionExpression: keyExpression,\n ExpressionAttributeValues: expressionAttributeValues,\n ExpressionAttributeNames: expressionAttributeNames,\n ConsistentRead: consistentRead,\n ...(!pageKey ? {} : { ExclusiveStartKey: pageKey }),\n ...(!limit ? {} : { Limit: limit }),\n ReturnConsumedCapacity: 'TOTAL',\n });\n\n return await this.#documentClient.send(command);\n }\n\n async update> (\n table: string,\n key: T,\n expression: string,\n expressionValues?: Record,\n expressionNames?: Record,\n ) {\n const hasValues = !!expressionValues && Object.keys(expressionValues).length > 0;\n const hasNames = !!expressionNames && Object.keys(expressionNames).length > 0;\n const command = new UpdateCommand({\n TableName: table,\n Key: key,\n UpdateExpression: expression,\n ...(hasValues ? { ExpressionAttributeValues: expressionValues } : {}),\n ...(hasNames ? { ExpressionAttributeNames: expressionNames } : {}),\n ReturnValues: 'ALL_NEW',\n ReturnConsumedCapacity: 'TOTAL',\n });\n try {\n return await this.#documentClient.send(command);\n } catch ( e ) {\n console.error('DDB Update Error', e);\n throw e;\n }\n }\n\n async createTableIfNotExists (params: CreateTableCommandInput, ttlAttribute?: string) {\n if ( this.config?.aws ) {\n console.warn('Creating DynamoDB tables in AWS is disabled by default, but if you need to enable it, modify the DDBClient class');\n return;\n }\n try {\n await this.#documentClient.send(new CreateTableCommand(params));\n } catch ( e ) {\n if ( (e as Error)?.name !== 'ResourceInUseException' ) {\n throw e;\n }\n setTimeout(async () => {\n if ( ttlAttribute ) {\n // ensure TTL is set\n await this.#documentClient.send(new UpdateTimeToLiveCommand({\n TableName: params.TableName!,\n TimeToLiveSpecification: {\n AttributeName: ttlAttribute,\n Enabled: true,\n },\n }));\n }\n }, 5000); // wait 5 seconds to ensure table is active\n\n }\n }\n}\n"
},
{
"path": "src/backend/src/clients/dynamodb/DDBClientWrapper.ts",
"content": "import { BaseService } from '@heyputer/backend/src/services/BaseService.js';\nimport { DDBClient } from './DDBClient.js';\n\n/** Wrapping actual implementation to be usable through our core structure */\nclass DDBClientServiceWrapper extends BaseService {\n ddbClient!: DDBClient;\n async _construct () {\n this.ddbClient = new DDBClient(this.config as unknown as ConstructorParameters[0]);\n\n await this.ddbClient.ddbClientPromise; // ensure client is ready\n\n Object.getOwnPropertyNames(DDBClient.prototype).forEach(fn => {\n if ( fn === 'constructor' ) return;\n this[fn] = (...args: unknown[]) => this.ddbClient[fn](...args);\n });\n }\n}\n\nexport const DDBClientWrapper = DDBClientServiceWrapper as unknown as DDBClient;\n"
},
{
"path": "src/backend/src/clients/redis/.gitignore",
"content": "*.js\n*.js.map"
},
{
"path": "src/backend/src/clients/redis/cacheUpdate.ts",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see => {\n const flattened: Array = [];\n for ( const input of inputs ) {\n if ( Array.isArray(input) ) {\n flattened.push(...flattenCacheKeys(input));\n continue;\n }\n flattened.push(input);\n }\n return flattened;\n};\n\nexport const normalizeCacheKeys = (cacheKey: CacheKeyInput | CacheKeyInput[]): string[] => {\n const arr = Array.isArray(cacheKey) ? cacheKey : [cacheKey];\n return [...new Set(flattenCacheKeys(arr)\n .map(key => key === null || key === undefined ? '' : String(key))\n .filter(Boolean))];\n};\n\nconst getEventService = (eventService?: CacheUpdateOptions['eventService']) => {\n if ( eventService?.emit ) return eventService;\n\n const contextServices = Context.get('services', { allow_fallback: true });\n if ( contextServices?.get ) {\n try {\n return contextServices.get('event');\n } catch (e) {\n // no-op\n }\n }\n\n const globalServices = (globalThis)[SERVICES_KEY]?.services as typeof contextServices;\n if ( globalServices?.get ) {\n try {\n return globalServices.get('event');\n } catch (e) {\n // no-op\n }\n }\n\n return null;\n};\n\nexport const emitOuterCacheUpdate = (\n {\n cacheKey,\n data,\n ttlSeconds,\n }: {\n cacheKey: CacheKeyInput | CacheKeyInput[],\n data?: unknown,\n ttlSeconds?: number,\n },\n {\n eventService,\n emitEvent = true,\n }: CacheUpdateOptions = {},\n) => {\n if ( ! emitEvent ) return;\n const keys = normalizeCacheKeys(cacheKey);\n if ( ! keys.length ) return;\n\n const svc_event = getEventService(eventService);\n if ( ! svc_event ) return;\n\n const payload: Record = { cacheKey: keys };\n if ( data !== undefined ) payload.data = data;\n if ( ttlSeconds !== undefined && ttlSeconds !== null ) {\n payload.ttlSeconds = ttlSeconds;\n }\n\n svc_event.emit('outer.cacheUpdate', payload);\n};\n\nexport const setRedisCacheValue = async (\n key: string,\n value: string | number,\n {\n ttlSeconds,\n }: {\n ttlSeconds?: number,\n eventData?: unknown,\n eventService?: CacheUpdateOptions['eventService'],\n emitEvent?: boolean,\n } = {},\n) => {\n if ( ttlSeconds ) {\n await redisClient.set(key, value, 'EX', ttlSeconds);\n } else {\n await redisClient.set(key, value);\n }\n};\n"
},
{
"path": "src/backend/src/clients/redis/deleteRedisKeys.ts",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see => {\n const flattened: Array = [];\n\n for ( const input of inputs ) {\n if ( Array.isArray(input) ) {\n flattened.push(...flattenInputs(input));\n continue;\n }\n flattened.push(input);\n }\n\n return flattened;\n};\n\nexport const deleteRedisKeys = async (...inputs: (DeleteRedisKeysInput | DeleteRedisKeysOptions)[]) => {\n const keysInput = [...inputs];\n if ( isDeleteOptions(keysInput[keysInput.length - 1]) ) {\n keysInput.pop() as DeleteRedisKeysOptions;\n }\n\n const keys = flattenInputs(keysInput as DeleteRedisKeysInput[])\n .map(key => key === null || key === undefined ? '' : String(key))\n .filter(Boolean);\n\n if ( keys.length === 0 ) {\n return 0;\n }\n\n const uniqueKeys = [...new Set(keys)];\n\n const deleteResults = await Promise.allSettled(uniqueKeys.map(key => redisClient.del(key)));\n const deleted = deleteResults.reduce((sum, promiseCount) => sum + (promiseCount.status === 'fulfilled' ? promiseCount.value : 0), 0);\n\n return deleted;\n};\n"
},
{
"path": "src/backend/src/clients/redis/redisSingleton.test.ts",
"content": "import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';\n\nconst redisMocks = vi.hoisted(() => {\n const redisClusterInstances: Array<{\n on: ReturnType;\n once: ReturnType;\n }> = [];\n\n return {\n redisClusterInstances,\n redisClusterConstructorMock: vi.fn(),\n mockRedisClusterConstructorMock: vi.fn(),\n };\n});\n\nvi.mock('ioredis', () => {\n class RedisClusterMock {\n on = vi.fn().mockReturnThis();\n once = vi.fn().mockReturnThis();\n\n constructor (...args: unknown[]) {\n redisMocks.redisClusterConstructorMock(...args);\n redisMocks.redisClusterInstances.push(this);\n }\n }\n\n return {\n default: {\n Cluster: RedisClusterMock,\n },\n };\n});\n\nvi.mock('ioredis-mock', () => {\n class MockRedisClusterMock {\n constructor (...args: unknown[]) {\n redisMocks.mockRedisClusterConstructorMock(...args);\n }\n }\n\n return {\n default: {\n Cluster: MockRedisClusterMock,\n },\n };\n});\n\ndescribe('redisSingleton', () => {\n const initialRedisConfig = process.env.REDIS_CONFIG;\n\n beforeEach(() => {\n vi.resetModules();\n redisMocks.redisClusterInstances.length = 0;\n redisMocks.redisClusterConstructorMock.mockReset();\n redisMocks.mockRedisClusterConstructorMock.mockReset();\n process.env.REDIS_CONFIG = JSON.stringify([{ host: '127.0.0.1', port: 6379 }]);\n vi.spyOn(console, 'log').mockImplementation(() => undefined);\n vi.spyOn(console, 'warn').mockImplementation(() => undefined);\n vi.spyOn(console, 'error').mockImplementation(() => undefined);\n });\n\n afterEach(() => {\n if ( initialRedisConfig === undefined ) {\n delete process.env.REDIS_CONFIG;\n } else {\n process.env.REDIS_CONFIG = initialRedisConfig;\n }\n vi.restoreAllMocks();\n });\n\n it('uses resilient cluster options and registers startup-safe listeners', async () => {\n const singletonModule = await import('./redisSingleton.ts');\n\n expect(redisMocks.redisClusterConstructorMock).toHaveBeenCalledTimes(1);\n const [startupNodes, clusterOptions] = redisMocks.redisClusterConstructorMock.mock.calls[0];\n\n expect(startupNodes).toEqual([{ host: '127.0.0.1', port: 6379 }]);\n expect(clusterOptions).toEqual(expect.objectContaining({\n enableOfflineQueue: true,\n retryDelayOnFailover: 500,\n retryDelayOnClusterDown: 1000,\n retryDelayOnTryAgain: 300,\n slotsRefreshTimeout: 5000,\n clusterRetryStrategy: expect.any(Function),\n dnsLookup: expect.any(Function),\n redisOptions: expect.objectContaining({\n connectTimeout: 10000,\n maxRetriesPerRequest: null,\n tls: {},\n }),\n }));\n expect(clusterOptions.clusterRetryStrategy(1)).toBe(200);\n expect(clusterOptions.clusterRetryStrategy(100)).toBe(2000);\n\n const clusterInstance = redisMocks.redisClusterInstances[0];\n expect(singletonModule.redisClient).toBe(clusterInstance);\n expect(clusterInstance.once).toHaveBeenCalledWith('connect', expect.any(Function));\n expect(clusterInstance.once).toHaveBeenCalledWith('ready', expect.any(Function));\n expect(clusterInstance.on).toHaveBeenCalledWith('error', expect.any(Function));\n expect(clusterInstance.on).toHaveBeenCalledWith('node error', expect.any(Function));\n });\n});\n"
},
{
"path": "src/backend/src/clients/redis/redisSingleton.ts",
"content": "import Redis, { Cluster } from 'ioredis';\nimport MockRedis from 'ioredis-mock';\n\nconst redisStartupRetryMaxDelayMs = 2000;\nconst redisSlotsRefreshTimeoutMs = 5000;\nconst redisConnectTimeoutMs = 10000;\nconst redisBootRetryRegex = /Cluster(All)?FailedError|None of startup nodes is available/i;\n\nconst formatRedisError = (error: unknown): string => {\n if ( error instanceof Error ) {\n return `${error.name}: ${error.message}`;\n }\n return String(error);\n};\n\nconst attachClusterEventHandlers = (clusterClient: Cluster): void => {\n clusterClient.once('connect', () => {\n console.log('[redis] cluster transport connected');\n });\n\n clusterClient.once('ready', () => {\n console.log('[redis] cluster ready');\n });\n\n clusterClient.on('error', (error: unknown) => {\n const errorText = formatRedisError(error);\n if ( redisBootRetryRegex.test(errorText) ) {\n console.warn(`[redis] startup issue while connecting to cluster; retrying automatically (${errorText})`);\n return;\n }\n console.error('[redis] cluster error', error);\n });\n\n clusterClient.on('node error', (error: unknown, nodeKey: string) => {\n const errorText = formatRedisError(error);\n if ( redisBootRetryRegex.test(errorText) ) {\n console.warn(`[redis] startup issue for cluster node ${nodeKey}; retrying automatically (${errorText})`);\n return;\n }\n console.error(`[redis] cluster node error (${nodeKey})`, error);\n });\n};\n\nlet redisOpt: Cluster;\n\nif ( process.env.REDIS_CONFIG ) {\n const redisConfig = JSON.parse(process.env.REDIS_CONFIG);\n redisOpt = new Redis.Cluster(redisConfig, {\n dnsLookup: (address, callback) => callback(null, address),\n clusterRetryStrategy: (attempts) => Math.min(100 + (attempts * 100), redisStartupRetryMaxDelayMs),\n retryDelayOnFailover: 500,\n retryDelayOnClusterDown: 1000,\n retryDelayOnTryAgain: 300,\n slotsRefreshTimeout: redisSlotsRefreshTimeoutMs,\n enableOfflineQueue: true,\n redisOptions: {\n tls: {},\n connectTimeout: redisConnectTimeoutMs,\n maxRetriesPerRequest: null,\n },\n });\n attachClusterEventHandlers(redisOpt);\n console.log('connecting to redis from config');\n} else {\n redisOpt = new MockRedis.Cluster(['redis://localhost:7001']);\n console.log('connected to local redis mock');\n}\n\nexport const redisClient = redisOpt;\n"
},
{
"path": "src/backend/src/codex/CodeUtil.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } SequenceCallable\n * A callable function returned by the Sequence constructor.\n * @param {Object|Sequence.SequenceState} [opt_values] - Initial values for the sequence scope, or a SequenceState.\n * @returns {Promise} The return value of the last step in the sequence.\n */\n/**\n * Sequence is a callable object that executes a series of functions in order.\n * The functions are expected to be asynchronous; if they're not it might still\n * work, but it's neither tested nor supported.\n *\n * Note: arrow functions are supported, but they are not recommended;\n * using keyword functions allows each step to be named.\n *\n * Example usage:\n *\n * const seq = new Sequence([\n * async function set_foo (a) {\n * a.set('foo', 'bar')\n * },\n * async function print_foo (a) {\n * console.log(a.get('foo'));\n * },\n * async function third_step (a) {\n * // do something\n * },\n * ]);\n *\n * await seq();\n *\n * Example with controlled conditional branches:\n *\n * const seq = new Sequence([\n * async function first_step (a) {\n * // do something\n * },\n * {\n * condition: async a => a.get('foo') === 'bar',\n * fn: async function second_step (a) {\n * // do something\n * }\n * },\n * async function third_step (a) {\n * // do something\n * },\n * ]);\n *\n * If it is called with an argument, it must be an object containing values\n * which will populate the \"sequence scope\".\n *\n * If it is called on an instance with a member called `values`\n * (i.e. if `this.values` is defined), then these values will populate the\n * sequence scope. This is to maintain compatibility for Sequence to be used\n * as an implementation of a runnable class. (See CodeUtil.mrwrap or BaseOperation)\n *\n * The object returned by the constructor is a function, which is used to\n * make the object callable. The callable object will execute the sequence\n * when called. The return value of the sequence is the return value of the\n * last function in the sequence.\n *\n * Each function in the sequence is passed a SequenceState object\n * as its first argument. Conventionally, this argument is called `a`,\n * which is short for either \"API\", \"access\", or \"the `a` variable\"\n * depending on which you prefer. Sequence provides methods for accessing\n * the sequence scope.\n *\n * By accessing the sequence scope through the `a` variable, changes to the\n * sequence scope can be monitored and recorded. (TODO: implement observe methods)\n */\n/**\n * Sequence is a callable object that executes a series of asynchronous functions in order.\n * Each function receives a SequenceState instance for accessing and mutating the sequence scope.\n * Supports conditional steps, deferred steps, and can be used as a runnable implementation for classes.\n * @class @extends Function\n */\nclass Sequence {\n /**\n * SequenceState represents the state of a Sequence execution.\n * Provides access to the sequence scope, step control, and utility methods for step functions.\n */\n static SequenceState = class SequenceState {\n /**\n * Create a new SequenceState.\n * @param {Sequence|function} sequence - The Sequence instance or its callable function.\n * @param {Object} [thisArg] - The instance to bind as `this` for step functions.\n */\n constructor (sequence, thisArg) {\n if ( typeof sequence === 'function' ) {\n sequence = sequence.sequence;\n }\n\n this.sequence_ = sequence;\n this.thisArg = thisArg;\n this.steps_ = null;\n this.value_history_ = [];\n this.scope_ = {};\n this.last_return_ = undefined;\n this.i = 0;\n this.stopped_ = false;\n\n this.defer_ptr_ = undefined;\n this.defer = this.constructor.defer_0;\n }\n\n /**\n * Get the current steps array for this sequence execution.\n * @returns {Array} The steps to execute.\n */\n get steps () {\n return this.steps_ ?? this.sequence_?.steps_;\n }\n\n /**\n * Run the sequence from the current step index.\n * @param {Object} [values] - Initial values for the sequence scope.\n * @returns {Promise}\n */\n async run (values) {\n // Initialize scope\n values = values || this.thisArg?.values || {};\n Object.setPrototypeOf(this.scope_, values);\n\n // Run sequence\n for ( ; this.i < this.steps.length ; this.i++ ) {\n let step = this.steps[this.i];\n if ( typeof step !== 'object' ) {\n step = {\n name: step.name,\n fn: step,\n };\n }\n\n if ( step.condition && !await step.condition(this) ) {\n continue;\n }\n\n const parent_scope = this.scope_;\n this.scope_ = {};\n // We could do Object.assign(this.scope_, parent_scope), but\n // setting the prototype should be faster (in theory)\n Object.setPrototypeOf(this.scope_, parent_scope);\n\n if ( this.sequence_.options_.record_history ) {\n this.value_history_.push(this.scope_);\n }\n\n if ( this.sequence_.options_.before_each ) {\n await this.sequence_.options_.before_each(this, step);\n }\n\n this.last_return_ = await step.fn.call(this.thisArg, this);\n\n if ( this.last_return_ instanceof Sequence.SequenceState ) {\n this.scope_ = this.last_return_.scope_;\n }\n\n if ( this.sequence_.options_.after_each ) {\n await this.sequence_.options_.after_each(this, step);\n }\n\n if ( this.stopped_ ) {\n break;\n }\n }\n }\n\n // Why check a condition every time code is called,\n // when we can check it once and then replace the code?\n\n /**\n * The first time defer is called, clones the steps and sets up for deferred insertion.\n * @param {function(Sequence.SequenceState): Promise} fn - The function to defer.\n */\n static defer_0 = function (fn) {\n this.steps_ = [...this.sequence_.steps_];\n this.defer = this.constructor.defer_1;\n this.defer_ptr_ = this.steps_.length;\n this.defer(fn);\n };\n /**\n * Subsequent calls to defer insert the function before the deferred pointer.\n * @param {function(Sequence.SequenceState): Promise} fn - The function to defer.\n */\n static defer_1 = function (fn) {\n // Deferred functions don't affect the return value\n const real_fn = fn;\n fn = async () => {\n await real_fn(this);\n return this.last_return_;\n };\n\n // Insert deferred step before the pointer\n this.steps_.splice(this.defer_ptr_, 0, fn);\n };\n\n /**\n * Get a value from the sequence scope.\n * @param {string} k - The key to retrieve.\n * @returns {any} The value associated with the key.\n */\n get (k) {\n // TODO: record read1\n return this.scope_[k];\n }\n\n /**\n * Set a value in the sequence scope.\n * @param {string} k - The key to set.\n * @param {any} v - The value to assign.\n */\n set (k, v) {\n // TODO: record mutation\n this.scope_[k] = v;\n }\n\n /**\n * Get or set multiple values in the sequence scope.\n * @param {Object} [opt_itemsToSet] - Optional object of key-value pairs to set.\n * @returns {Object} Proxy to the current scope for value access.\n */\n values (opt_itemsToSet) {\n if ( opt_itemsToSet ) {\n for ( const k in opt_itemsToSet ) {\n this.set(k, opt_itemsToSet[k]);\n }\n }\n\n return new Proxy(this.scope_, {\n get: (target, property) => {\n if ( property in target ) {\n // TODO: record read\n return target[property];\n }\n return undefined;\n },\n });\n }\n\n /**\n * Get a value from the instance (`thisArg`).\n * @param {string} [k] - The property name to retrieve. If omitted, returns the instance.\n * @returns {any} The value from the instance or the instance itself.\n */\n iget (k) {\n if ( k === undefined ) return this.thisArg;\n return this.thisArg?.[k];\n }\n\n // Instance call: call a method on the instance\n /**\n * Call a method on the instance (`thisArg`).\n * @param {string} k - The method name.\n * @param {...any} args - Arguments to pass to the method.\n * @returns {any} The result of the method call.\n */\n icall (k, ...args) {\n return this.thisArg?.[k]?.call(this.thisArg, ...args);\n }\n\n // Instance dynamic call: call a method on the instance,\n // passing the sequence state as the first argument\n /**\n * Call a method on the instance, passing the sequence state as the first argument.\n * @param {string} k - The method name.\n * @param {...any} args - Arguments to pass after the sequence state.\n * @returns {any} The result of the method call.\n */\n idcall (k, ...args) {\n return this.thisArg?.[k]?.call(this.thisArg, this, ...args);\n }\n\n /**\n * Get the logger from the instance, if available.\n * @returns {Object|undefined} The logger object.\n */\n get log () {\n return this.iget('log');\n }\n\n /**\n * Stop the sequence early and optionally return a value.\n * @param {any} [return_value] - Value to return from the sequence.\n * @returns {any} The provided return value.\n */\n stop (return_value) {\n this.stopped_ = true;\n return return_value;\n }\n };\n\n /**\n *\n * @param {Array | {condition: (a: A) => boolean | Promise, fn: function(A): Promise}> | function(A): Promise | Object} args\n * @returns {Sequence}\n */\n /**\n * Create a new Sequence.\n * @param {...(Array|Object>|function(Sequence.SequenceState): Promise|Object)} args\n * - Arrays of step functions or step objects, individual step functions, or options objects.\n * - Step objects may have a `condition` property (function) and a `fn` property (function).\n * - Options object may include `name`, `record_history`, `before_each`, `after_each`.\n * @returns {SequenceCallable} A callable function that runs the sequence.\n */\n constructor (...args) {\n const sequence = this;\n\n const steps = [];\n const options = {};\n\n for ( const arg of args ) {\n if ( Array.isArray(arg) ) {\n steps.push(...arg);\n } else if ( typeof arg === 'object' ) {\n Object.assign(options, arg);\n } else if ( typeof arg === 'function' ) {\n steps.push(arg);\n } else {\n throw new TypeError(`Invalid argument to Sequence constructor: ${arg}`);\n }\n }\n\n /**\n * Callable function to execute the sequence.\n * @param {Object|Sequence.SequenceState} [opt_values] - Initial values or a SequenceState.\n * @returns {Promise} The return value of the last step.\n */\n const fn = async function (opt_values) {\n if ( opt_values && opt_values instanceof Sequence.SequenceState ) {\n opt_values = opt_values.scope_;\n }\n // eslint-disable-next-line no-invalid-this\n const state = new Sequence.SequenceState(sequence, this); // TODO: fix this odd structure, what is this even bound to ?\n await state.run(opt_values ?? undefined);\n return state.last_return_;\n };\n\n this.steps_ = steps;\n this.options_ = options || {};\n\n Object.defineProperty(fn, 'name', {\n value: options.name || 'Sequence',\n });\n Object.defineProperty(fn, 'sequence', { value: this });\n\n return fn;\n }\n}\n\nmodule.exports = {\n Sequence,\n};\n"
},
{
"path": "src/backend/src/config/ConfigLoader.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see ;\n\nexport interface IConfig extends ConfigRecord {\n load_config: (o: ConfigRecord) => void;\n __set_config_object__: (\n object: ConfigRecord,\n options?: { replacePrototype?: boolean; useInitialPrototype?: boolean }\n ) => void;\n}\n\ndeclare const config: IConfig;\n\nexport = config;\n"
},
{
"path": "src/backend/src/config.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see = {\n encode(message: FSEntry, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {\n if (message.uuid !== \"\") {\n writer.uint32(10).string(message.uuid);\n }\n if (message.uid !== \"\") {\n writer.uint32(18).string(message.uid);\n }\n if (message.name !== \"\") {\n writer.uint32(26).string(message.name);\n }\n if (message.path !== \"\") {\n writer.uint32(34).string(message.path);\n }\n if (message.parent_uuid !== \"\") {\n writer.uint32(42).string(message.parent_uuid);\n }\n if (message.parent_uid !== \"\") {\n writer.uint32(50).string(message.parent_uid);\n }\n if (message.parent_id !== \"\") {\n writer.uint32(58).string(message.parent_id);\n }\n if (message.is_dir !== false) {\n writer.uint32(64).bool(message.is_dir);\n }\n if (message.created !== 0) {\n writer.uint32(72).int64(message.created);\n }\n if (message.modified !== 0) {\n writer.uint32(80).int64(message.modified);\n }\n if (message.accessed !== 0) {\n writer.uint32(88).int64(message.accessed);\n }\n if (message.size !== 0) {\n writer.uint32(96).int64(message.size);\n }\n return writer;\n },\n\n decode(input: BinaryReader | Uint8Array, length?: number): FSEntry {\n const reader = input instanceof BinaryReader ? input : new BinaryReader(input);\n const end = length === undefined ? reader.len : reader.pos + length;\n const message = createBaseFSEntry();\n while (reader.pos < end) {\n const tag = reader.uint32();\n switch (tag >>> 3) {\n case 1: {\n if (tag !== 10) {\n break;\n }\n\n message.uuid = reader.string();\n continue;\n }\n case 2: {\n if (tag !== 18) {\n break;\n }\n\n message.uid = reader.string();\n continue;\n }\n case 3: {\n if (tag !== 26) {\n break;\n }\n\n message.name = reader.string();\n continue;\n }\n case 4: {\n if (tag !== 34) {\n break;\n }\n\n message.path = reader.string();\n continue;\n }\n case 5: {\n if (tag !== 42) {\n break;\n }\n\n message.parent_uuid = reader.string();\n continue;\n }\n case 6: {\n if (tag !== 50) {\n break;\n }\n\n message.parent_uid = reader.string();\n continue;\n }\n case 7: {\n if (tag !== 58) {\n break;\n }\n\n message.parent_id = reader.string();\n continue;\n }\n case 8: {\n if (tag !== 64) {\n break;\n }\n\n message.is_dir = reader.bool();\n continue;\n }\n case 9: {\n if (tag !== 72) {\n break;\n }\n\n message.created = longToNumber(reader.int64());\n continue;\n }\n case 10: {\n if (tag !== 80) {\n break;\n }\n\n message.modified = longToNumber(reader.int64());\n continue;\n }\n case 11: {\n if (tag !== 88) {\n break;\n }\n\n message.accessed = longToNumber(reader.int64());\n continue;\n }\n case 12: {\n if (tag !== 96) {\n break;\n }\n\n message.size = longToNumber(reader.int64());\n continue;\n }\n }\n if ((tag & 7) === 4 || tag === 0) {\n break;\n }\n reader.skip(tag & 7);\n }\n return message;\n },\n\n fromJSON(object: any): FSEntry {\n return {\n uuid: isSet(object.uuid) ? globalThis.String(object.uuid) : \"\",\n uid: isSet(object.uid) ? globalThis.String(object.uid) : \"\",\n name: isSet(object.name) ? globalThis.String(object.name) : \"\",\n path: isSet(object.path) ? globalThis.String(object.path) : \"\",\n parent_uuid: isSet(object.parent_uuid) ? globalThis.String(object.parent_uuid) : \"\",\n parent_uid: isSet(object.parent_uid) ? globalThis.String(object.parent_uid) : \"\",\n parent_id: isSet(object.parent_id) ? globalThis.String(object.parent_id) : \"\",\n is_dir: isSet(object.is_dir) ? globalThis.Boolean(object.is_dir) : false,\n created: isSet(object.created) ? globalThis.Number(object.created) : 0,\n modified: isSet(object.modified) ? globalThis.Number(object.modified) : 0,\n accessed: isSet(object.accessed) ? globalThis.Number(object.accessed) : 0,\n size: isSet(object.size) ? globalThis.Number(object.size) : 0,\n };\n },\n\n toJSON(message: FSEntry): unknown {\n const obj: any = {};\n if (message.uuid !== \"\") {\n obj.uuid = message.uuid;\n }\n if (message.uid !== \"\") {\n obj.uid = message.uid;\n }\n if (message.name !== \"\") {\n obj.name = message.name;\n }\n if (message.path !== \"\") {\n obj.path = message.path;\n }\n if (message.parent_uuid !== \"\") {\n obj.parent_uuid = message.parent_uuid;\n }\n if (message.parent_uid !== \"\") {\n obj.parent_uid = message.parent_uid;\n }\n if (message.parent_id !== \"\") {\n obj.parent_id = message.parent_id;\n }\n if (message.is_dir !== false) {\n obj.is_dir = message.is_dir;\n }\n if (message.created !== 0) {\n obj.created = Math.round(message.created);\n }\n if (message.modified !== 0) {\n obj.modified = Math.round(message.modified);\n }\n if (message.accessed !== 0) {\n obj.accessed = Math.round(message.accessed);\n }\n if (message.size !== 0) {\n obj.size = Math.round(message.size);\n }\n return obj;\n },\n\n create(base?: DeepPartial): FSEntry {\n return FSEntry.fromPartial(base ?? {});\n },\n fromPartial(object: DeepPartial): FSEntry {\n const message = createBaseFSEntry();\n message.uuid = object.uuid ?? \"\";\n message.uid = object.uid ?? \"\";\n message.name = object.name ?? \"\";\n message.path = object.path ?? \"\";\n message.parent_uuid = object.parent_uuid ?? \"\";\n message.parent_uid = object.parent_uid ?? \"\";\n message.parent_id = object.parent_id ?? \"\";\n message.is_dir = object.is_dir ?? false;\n message.created = object.created ?? 0;\n message.modified = object.modified ?? 0;\n message.accessed = object.accessed ?? 0;\n message.size = object.size ?? 0;\n return message;\n },\n};\n\ntype Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;\n\nexport type DeepPartial = T extends Builtin ? T\n : T extends globalThis.Array ? globalThis.Array>\n : T extends ReadonlyArray ? ReadonlyArray>\n : T extends {} ? { [K in keyof T]?: DeepPartial }\n : Partial;\n\nfunction longToNumber(int64: { toString(): string }): number {\n const num = globalThis.Number(int64.toString());\n if (num > globalThis.Number.MAX_SAFE_INTEGER) {\n throw new globalThis.Error(\"Value is larger than Number.MAX_SAFE_INTEGER\");\n }\n if (num < globalThis.Number.MIN_SAFE_INTEGER) {\n throw new globalThis.Error(\"Value is smaller than Number.MIN_SAFE_INTEGER\");\n }\n return num;\n}\n\nfunction isSet(value: any): boolean {\n return value !== null && value !== undefined;\n}\n\nexport interface MessageFns {\n encode(message: T, writer?: BinaryWriter): BinaryWriter;\n decode(input: BinaryReader | Uint8Array, length?: number): T;\n fromJSON(object: any): T;\n toJSON(message: T): unknown;\n create(base?: DeepPartial): T;\n fromPartial(object: DeepPartial): T;\n}\n"
},
{
"path": "src/backend/src/filesystem/hl_operations/definitions.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } created or existing directory node\n */\nasync function createDirOrUseExisting ({ parent, selector, actor, fs }) {\n try {\n const ll_mkdir = new LLMkdir();\n return await ll_mkdir.run({\n parent,\n name: selector.name,\n actor,\n });\n } catch ( error ) {\n // This \"error\" can occur when multiple `hl_mkdir` operations are being\n // run at the same time with the `createMissingParents` option enabled.\n const errorCode = error.code || error.fields?.code;\n if ( errorCode === 'item_with_same_name_exists' ) {\n const existing_node = await fs.node(selector);\n\n // Wait for the entry to be stable (it might still be in the process\n // of being created by another parallel request)\n await existing_node.awaitStableEntry();\n await existing_node.fetchEntry();\n\n // If this is a file we need to re-throw the error\n if ( await existing_node.get('type') !== FSNodeContext.TYPE_DIRECTORY ) {\n throw error;\n }\n\n return existing_node;\n }\n\n throw error;\n }\n}\n\nclass MkTree extends HLFilesystemOperation {\n static DESCRIPTION = `\n High-level operation for making directory trees\n\n The following input for 'tree':\n ['a/b/c', ['i/j/k'], ['p', ['q'], ['r/s']]]]\n\n Would create a directory tree like this:\n a\n └── b\n └── c\n ├── i\n │ └── j\n │ └── k\n └── p\n ├── q\n └── r\n └── s\n `;\n\n static PARAMETERS = {\n parent: new FSNodeParam('parent', { optional: true }),\n };\n\n static PROPERTIES = {\n leaves: () => [],\n directories_created: () => [],\n };\n\n async _run () {\n const { values, context } = this;\n const fs = context.get('services').get('filesystem');\n\n await this.create_branch_({\n parent_node: values.parent || await fs.node(new RootNodeSelector()),\n tree: values.tree,\n parent_exists: true,\n });\n }\n\n async create_branch_ ({ parent_node, tree, parent_exists }) {\n const { context } = this;\n const fs = context.get('services').get('filesystem');\n const actor = context.get('actor');\n\n const trunk = tree[0];\n const branches = tree.slice(1);\n\n let current = parent_node.selector;\n\n // trunk = a/b/c\n\n const dirs = trunk === '.' ? []\n : trunk.split('/').filter(Boolean);\n\n // dirs = [a, b, c]\n\n let parent_did_exist = parent_exists;\n\n // This is just a loop that goes through each part of the path\n // until it finds the first directory that doesn't exist yet.\n let i = 0;\n if ( parent_exists ) {\n for ( ; i < dirs.length ; i++ ) {\n const dir = dirs[i];\n const currentParent = current;\n current = new NodeChildSelector(current, dir);\n\n const maybe_dir = await fs.node(current);\n\n if ( maybe_dir.isRoot ) continue;\n if ( await maybe_dir.isUserDirectory() ) continue;\n\n if ( await maybe_dir.exists() ) {\n\n if ( await maybe_dir.get('type') !== FSNodeContext.TYPE_DIRECTORY ) {\n throw APIError.create('dest_is_not_a_directory');\n }\n\n continue;\n }\n\n current = currentParent;\n parent_exists = false;\n break;\n }\n }\n\n if ( parent_did_exist && !parent_exists ) {\n const node = await fs.node(current);\n const has_perm = await chkperm(await node.get('entry'), actor.type.user.id, 'write');\n if ( ! has_perm ) throw APIError.create('permission_denied');\n }\n\n // This next loop creates the new directories\n\n // We break into a second loop because we know none of these directories\n // exist yet. If we continued those checks each child operation would\n // wait for the previous one to complete because FSNodeContext::fetchEntry\n // will notice ResourceService has a lock on the previous operation\n // we started.\n\n // In this way it goes nyyyoooom because all the database inserts\n // happen concurrently (and probably end up in the same batch).\n\n for ( ; i < dirs.length ; i++ ) {\n const dir = dirs[i];\n const currentParent = current;\n current = new NodeChildSelector(current, dir);\n\n const node = await createDirOrUseExisting({\n parent: await fs.node(currentParent),\n selector: current,\n actor,\n fs,\n });\n\n current = node.selector;\n\n this.directories_created.push(node);\n }\n\n const bottom_parent = await fs.node(current);\n\n if ( branches.length === 0 ) {\n this.leaves.push(bottom_parent);\n }\n\n for ( const branch of branches ) {\n await this.create_branch_({\n parent_node: bottom_parent,\n tree: branch,\n parent_exists,\n });\n }\n }\n}\n\nclass QuickMkdir extends HLFilesystemOperation {\n async _run () {\n const { context, values } = this;\n let { parent, path } = values;\n const { _path } = this.modules;\n const fs = context.get('services').get('filesystem');\n const actor = context.get('actor');\n\n parent = parent || await fs.node(new RootNodeSelector());\n\n let current = parent.selector;\n\n const dirs = path === '.' ? []\n : path.split('/').filter(Boolean);\n\n const api = require('@opentelemetry/api');\n const currentSpan = api.trace.getSpan(api.context.active());\n if ( currentSpan ) {\n currentSpan.setAttribute('path', path);\n currentSpan.setAttribute('dirs', dirs.join('/'));\n currentSpan.setAttribute('parent', parent.selector.describe());\n }\n\n for ( let i = 0 ; i < dirs.length ; i++ ) {\n const dir = dirs[i];\n const currentParent = current;\n current = new NodeChildSelector(current, dir);\n\n const node = await createDirOrUseExisting({\n parent: await fs.node(currentParent),\n selector: current,\n actor,\n fs,\n });\n\n current = node.selector;\n\n // this.directories_created.push(node);\n }\n\n this.created = await fs.node(current);\n }\n}\n\nclass HLMkdir extends HLFilesystemOperation {\n static DESCRIPTION = `\n High-level mkdir operation.\n\n This operation is a wrapper around the low-level mkdir operation.\n It provides the following features:\n - create missing parent directories\n - overwrite existing files\n - dedupe names\n - create shortcuts\n `;\n\n static PARAMETERS = {\n parent: new FSNodeParam('parent', { optional: true }),\n path: new StringParam('path'),\n overwrite: new FlagParam('overwrite', { optional: true }),\n create_missing_parents: new FlagParam('create_missing_parents', { optional: true }),\n user: new UserParam(),\n\n shortcut_to: new FSNodeParam('shortcut_to', { optional: true }),\n };\n\n static MODULES = {\n _path: require('path'),\n };\n\n static PROPERTIES = {\n parent_directories_created: () => [],\n };\n\n static FEATURES = [\n new OtelFeature([\n '_get_existing_parent',\n '_create_parents',\n ]),\n ];\n\n async _run () {\n const { context, values } = this;\n const { _path } = this.modules;\n const fs = context.get('services').get('filesystem');\n\n if ( ! is_valid_path(values.path, {\n no_relative_components: true,\n allow_path_fragment: true,\n }) ) {\n throw APIError.create('field_invalid', null, {\n key: 'path',\n expected: 'valid path',\n got: 'invalid path',\n });\n }\n\n // Unify the following formats:\n // - full path: {\"path\":\"/foo/bar\", args...}, used by apitest (./tools/api-tester/apitest.js)\n // - parent + path: {\"parent\": \"/foo\", \"path\":\"bar\", args...}, used by puter-js (puter.fs.mkdir(\"/foo/bar\"))\n if ( !values.parent && values.path ) {\n values.parent = await fs.node(new NodePathSelector(_path.dirname(values.path)));\n values.path = _path.basename(values.path);\n }\n\n let parent_node = values.parent || await fs.node(new RootNodeSelector());\n\n let target_basename = _path.basename(values.path);\n\n // \"top_parent\" is the immediate parent of the target directory\n // (e.g: /home/foo/bar -> /home/foo)\n const top_parent = values.create_missing_parents\n ? await this._create_dir(parent_node)\n : await this._get_existing_top_parent({ top_parent: parent_node })\n ;\n\n // TODO: this can be removed upon completion of: https://github.com/HeyPuter/puter/issues/1352\n if ( top_parent.isRoot ) {\n // root directory is read-only\n throw APIError.create('forbidden', null, {\n message: 'Cannot create directories in the root directory.',\n });\n }\n\n // `parent_node` becomes the parent of the last directory name\n // specified under `path`.\n parent_node = await this._create_parents({\n parent_node: top_parent,\n actor: values.actor,\n });\n\n const user_id = values.actor.type.user.id;\n\n const has_perm = await chkperm(await parent_node.get('entry'), user_id, 'write');\n if ( ! has_perm ) throw APIError.create('permission_denied');\n\n const existing = await fs.node(new NodeChildSelector(parent_node.selector, target_basename));\n\n await existing.fetchEntry();\n\n if ( existing.found ) {\n const { overwrite, dedupe_name, create_missing_parents } = values;\n if ( overwrite ) {\n // TODO: tag rm operation somehow\n const has_perm = await chkperm(await existing.get('entry'), user_id, 'write');\n if ( ! has_perm ) throw APIError.create('permission_denied');\n const hl_remove = new HLRemove();\n await hl_remove.run({\n target: existing,\n actor: values.actor,\n recursive: true,\n });\n }\n else if ( dedupe_name ) {\n const fs = context.get('services').get('filesystem');\n const parent_selector = parent_node.selector;\n for ( let i = 1 ;; i++ ) {\n let try_new_name = `${target_basename} (${i})`;\n const selector = new NodeChildSelector(parent_selector, try_new_name);\n const exists = await parent_node.provider.quick_check({\n selector,\n });\n if ( ! exists ) {\n target_basename = try_new_name;\n break;\n }\n }\n }\n else if ( create_missing_parents ) {\n if ( ! existing.entry.is_dir ) {\n throw APIError.create('dest_is_not_a_directory');\n }\n this.created = existing;\n this.used_existing = true;\n return await this.created.getSafeEntry();\n } else {\n throw APIError.create('item_with_same_name_exists', null, {\n entry_name: target_basename,\n });\n }\n }\n\n if ( values.shortcut_to ) {\n const shortcut_to = values.shortcut_to;\n if ( ! await shortcut_to.exists() ) {\n throw APIError.create('shortcut_to_does_not_exist');\n }\n if ( ! shortcut_to.entry.is_dir ) {\n throw APIError.create('shortcut_target_is_a_directory');\n }\n const has_perm = await chkperm(shortcut_to.entry, user_id, 'read');\n if ( ! has_perm ) throw APIError.create('forbidden');\n\n this.created = await fs.mkshortcut({\n parent: parent_node,\n name: target_basename,\n actor: values.actor,\n target: shortcut_to,\n });\n\n await this.created.awaitStableEntry();\n return await this.created.getSafeEntry();\n }\n\n let created_node;\n try {\n const ll_mkdir = new LLMkdir();\n created_node = await ll_mkdir.run({\n parent: parent_node,\n name: target_basename,\n actor: values.actor,\n });\n } catch ( error ) {\n // This \"error\" can occur when multiple `hl_mkdir` operations are being\n // run at the same time with the `createMissingParents` option enabled.\n const errorCode = error.code || error.fields?.code;\n if ( errorCode === 'item_with_same_name_exists' ) {\n const existing_node = await fs.node(new NodeChildSelector(parent_node.selector, target_basename));\n\n // Wait for the entry to be stable (it might still be in the process\n // of being created by another parallel request)\n await existing_node.awaitStableEntry();\n await existing_node.fetchEntry();\n\n // If this is a file we need to re-throw the error\n if ( await existing_node.get('type') !== FSNodeContext.TYPE_DIRECTORY ) {\n throw error;\n }\n\n created_node = existing_node;\n } else {\n throw error;\n }\n }\n\n this.created = created_node;\n\n const all_nodes = [\n ...this.parent_directories_created,\n this.created,\n ];\n\n await Promise.all(all_nodes.map(node => node.awaitStableEntry()));\n\n const response = await this.created.getSafeEntry();\n response.parent_dirs_created = [];\n for ( const node of this.parent_directories_created ) {\n response.parent_dirs_created.push(await node.getSafeEntry());\n }\n response.requested_path = values.path;\n\n return response;\n }\n\n async _create_parents ({ parent_node }) {\n const { context, values } = this;\n const { _path } = this.modules;\n\n const fs = context.get('services').get('filesystem');\n\n // Determine the deepest existing node\n let deepest_existing = parent_node;\n let remaining_path = _path.dirname(values.path).split('/').filter(Boolean);\n {\n const parts = remaining_path.slice();\n for ( ;; ) {\n if ( remaining_path.length === 0 ) {\n return deepest_existing;\n }\n const component = remaining_path[0];\n const next_selector = new NodeChildSelector(deepest_existing.selector, component);\n const next_node = await fs.node(next_selector);\n if ( ! await next_node.exists() ) {\n break;\n }\n deepest_existing = next_node;\n remaining_path.shift();\n }\n }\n\n const tree_op = new MkTree();\n await tree_op.run({\n parent: deepest_existing,\n tree: [remaining_path.join('/')],\n });\n\n this.parent_directories_created = tree_op.directories_created;\n\n return tree_op.leaves[0];\n }\n\n /**\n * Creates a directory and all its ancestors.\n *\n * @param {FSNodeContext} dir - The directory to create.\n * @returns {Promise} The created directory.\n */\n async _create_dir (dir) {\n if ( await dir.exists() ) {\n if ( ! dir.entry.is_dir ) {\n throw APIError.create('dest_is_not_a_directory');\n }\n return dir;\n }\n\n const maybe_path_selector =\n dir.get_selector_of_type(NodePathSelector);\n\n if ( ! maybe_path_selector ) {\n throw APIError.create('dest_does_not_exist', null, { what_dest: 'path from selector' });\n }\n\n const path = maybe_path_selector.value;\n\n const fs = this.context.get('services').get('filesystem');\n\n const tree_op = new MkTree();\n await tree_op.run({\n parent: await fs.node(new RootNodeSelector()),\n tree: [path],\n });\n\n return tree_op.leaves[0];\n }\n\n async _get_existing_top_parent ({ top_parent }) {\n if ( ! await top_parent.exists() ) {\n throw APIError.create('dest_does_not_exist', null, {\n // This seems verbose, but is necessary information when creating\n // shortcuts, otherwise the developer doesn't know if we're talking\n // about the shortcut's target directory or this parent directory.\n what_dest: 'parent directory of the new directory being created',\n });\n }\n\n if ( ! top_parent.entry.is_dir ) {\n throw APIError.create('dest_is_not_a_directory');\n }\n\n return top_parent;\n }\n}\n\nmodule.exports = {\n QuickMkdir,\n HLMkdir,\n MkTree,\n};\n"
},
{
"path": "src/backend/src/filesystem/hl_operations/hl_mklink.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } Result of the write operation\n * @throws {APIError} When the target node does not exist\n */\n async _run () {\n const node = this.values.node;\n\n // Embed fields into this.context\n this.context.set('immutable', this.values.immutable);\n this.context.set('tmp', this.values.tmp);\n this.context.set('fsentry_tmp', this.values.fsentry_tmp);\n this.context.set('message', this.values.message);\n this.context.set('actor', this.values.actor);\n this.context.set('app_id', this.values.app_id);\n\n // TODO: Add symlink write\n if ( ! await node.exists() ) {\n // TODO: different class of errors for low-level operations\n throw APIError.create('subject_does_not_exist');\n }\n\n return await node.provider.write_overwrite({\n context: this.context,\n node: node,\n file: this.values.file,\n });\n }\n}\n\n/**\n * The \"non-overwrite\" write operation.\n *\n * This operation is used to write a file to a non-existent path.\n *\n * @extends LLFilesystemOperation\n */\nclass LLCWrite extends LLFilesystemOperation {\n static MODULES = {\n _path: require('path'),\n uuidv4: require('uuid').v4,\n config: require('../../config.js'),\n };\n\n /**\n * Executes the create operation by writing a new file to the parent directory.\n * @returns {Promise} Result of the write operation\n * @throws {APIError} When the parent directory does not exist\n */\n async _run () {\n const parent = this.values.parent;\n\n // Embed fields into this.context\n this.context.set('immutable', this.context.get('immutable') ?? this.values.immutable);\n this.context.set('tmp', this.context.get('tmp') ?? this.values.tmp);\n this.context.set('fsentry_tmp', this.context.get('fsentry_tmp') ?? this.values.fsentry_tmp);\n this.context.set('message', this.context.get('message') ?? this.values.message);\n this.context.set('actor', this.context.get('actor') ?? this.values.actor);\n this.context.set('app_id', this.context.get('app_id') ?? this.values.app_id);\n\n if ( ! await parent.exists() ) {\n throw APIError.create('subject_does_not_exist');\n }\n\n return await parent.provider.write_new({\n context: this.context,\n parent,\n name: this.values.name,\n file: this.values.file,\n });\n }\n}\n\nmodule.exports = {\n LLCWrite,\n LLOWrite,\n};\n"
},
{
"path": "src/backend/src/filesystem/node/selectors.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see _`,\nwhere `` specifies broadly what that strategies contained within\nthe directory are concerned with (storage, fsentry, etc), and ``\nis a letter from A-Z indicating the layer/level of concern.\n\nThe class **A** indicates that this is the highest level of swappable\nbehaviour, which generally means there will be two strategies:\n- one which supports legacy behaviour that is coupled with multiple concerns\n- one which adapts more cohesive strategies to an interface which\n supports the case above.\n"
},
{
"path": "src/backend/src/filesystem/strategies/storage_a/LocalDiskStorageStrategy.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } Resolves when the upload is complete\n */\n async _run () {\n const { uid, file, storage_api } = this.values;\n\n const { progress_tracker } = storage_api;\n\n if ( file.buffer ) {\n await this.parent.svc_localDiskStorage.store_buffer({\n key: uid,\n buffer: file.buffer,\n });\n progress_tracker.set_total(file.buffer.length);\n progress_tracker.set(file.buffer.length);\n } else {\n await this.parent.svc_localDiskStorage.store_stream({\n key: uid,\n stream: file.stream,\n size: file.size,\n on_progress: evt => {\n progress_tracker.set_total(file.size);\n progress_tracker.set(evt.uploaded);\n },\n });\n }\n }\n\n /**\n * Hook called after the operation is inserted into the trace.\n */\n post_insert () {\n }\n}\n\n/**\n * Handles file copy operations within local disk storage.\n * Extends BaseOperation to provide copy functionality with progress tracking.\n */\nclass LocalDiskCopyStrategy extends BaseOperation {\n /**\n * Creates a new LocalDiskCopyStrategy instance.\n * @param {Object} parent - The parent storage strategy instance\n */\n constructor (parent) {\n super();\n this.parent = parent;\n }\n\n /**\n * Executes the copy operation by duplicating a file from source to destination.\n * Updates progress tracker to indicate completion.\n * @returns {Promise} Resolves when the copy is complete\n */\n async _run () {\n const { src_node, dst_storage, storage_api } = this.values;\n const { progress_tracker } = storage_api;\n\n await this.parent.svc_localDiskStorage.copy({\n src_key: await src_node.get('uid'),\n dst_key: dst_storage.key,\n });\n\n // for now we just copy the file, we don't care about the progress\n progress_tracker.set_total(1);\n progress_tracker.set(1);\n }\n\n /**\n * Hook called after the operation is inserted into the trace.\n */\n post_insert () {\n }\n}\n\n/**\n * Handles file deletion operations from local disk storage.\n * Extends BaseOperation to provide delete functionality.\n */\nclass LocalDiskDeleteStrategy extends BaseOperation {\n /**\n * Creates a new LocalDiskDeleteStrategy instance.\n * @param {Object} parent - The parent storage strategy instance\n */\n constructor (parent) {\n super();\n this.parent = parent;\n }\n\n /**\n * Executes the delete operation by removing a file from local disk storage.\n * @returns {Promise} Resolves when the deletion is complete\n */\n async _run () {\n const { node } = this.values;\n\n await this.parent.svc_localDiskStorage.delete({\n key: await node.get('uid'),\n });\n }\n}\n\n/**\n * Main strategy class for managing local disk storage operations.\n * Provides factory methods for creating upload, copy, and delete operations.\n */\nclass LocalDiskStorageStrategy {\n /**\n * Creates a new LocalDiskStorageStrategy instance.\n * @param {Object} config - Configuration object\n * @param {Object} config.services - Services container for dependency injection\n */\n constructor ({ services }) {\n this.svc_localDiskStorage = services.get('local-disk-storage');\n }\n\n /**\n * Creates a new upload operation instance.\n * @returns {LocalDiskUploadStrategy} A new upload strategy instance\n */\n create_upload () {\n return new LocalDiskUploadStrategy(this);\n }\n\n /**\n * Creates a new copy operation instance.\n * @returns {LocalDiskCopyStrategy} A new copy strategy instance\n */\n create_copy () {\n return new LocalDiskCopyStrategy(this);\n }\n\n /**\n * Creates a new delete operation instance.\n * @returns {LocalDiskDeleteStrategy} A new delete strategy instance\n */\n create_delete () {\n return new LocalDiskDeleteStrategy(this);\n }\n\n /**\n * Creates a readable stream for accessing file data from local disk storage.\n * @param {string} uid - The unique identifier of the file to read\n * @param {Object} [options={}] - Optional parameters for stream creation\n * @returns {Promise} A readable stream for the file data\n */\n async create_read_stream (uid, options = {}) {\n return await this.svc_localDiskStorage.create_read_stream(uid, options);\n }\n}\n\nmodule.exports = {\n LocalDiskStorageStrategy,\n};\n"
},
{
"path": "src/backend/src/filesystem/strategies/storage_a/README.md",
"content": "## Class A Storage Strategies\n\nThis is the broadest definition of storage strategies.\nThis is to allow swapping between the behaviour of the original\nPuter storage logic, and Class B storage strategies.\n\n- they know the UID of the file\n- they can perform post-operations after the fsentry is inserted\n- they can access the Puter database\n"
},
{
"path": "src/backend/src/filesystem/validation.bench.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see >}\n */\nexport const get_apps = spanify('get_apps', async (specifiers, options = {}) => {\n if ( ! Array.isArray(specifiers) ) {\n specifiers = [specifiers];\n }\n\n const decorateApp = (app) => {\n if ( ! app ) return app;\n const icon_url = get_app_icon_url(app.uid ?? app.uuid);\n if ( ! icon_url ) return { ...app };\n return { ...app, icon: icon_url };\n };\n const normalizeAppForCache = (app) => {\n if ( ! app ) return app;\n const normalized = { ...app };\n delete normalized.icon_is_base64;\n return normalized;\n };\n const isDecoratedAppCacheEntry = (app) => (\n !!app &&\n typeof app === 'object' &&\n Object.prototype.hasOwnProperty.call(app, 'icon_is_base64')\n );\n const cacheApp = async (app) => {\n if ( ! app ) return;\n AppRedisCacheSpace.setCachedApp(app, {\n ttlSeconds: 60,\n });\n };\n\n const normalized = specifiers.map(spec => spec ? { ...spec } : {});\n\n if ( options.follow_old_names ) {\n const svc_oldAppName = servicesContainer.services.get('old-app-name');\n for ( const spec of normalized ) {\n if ( spec.uid || !spec.name ) continue;\n const old_name = await svc_oldAppName.check_app_name(spec.name);\n if ( old_name ) {\n spec.uid = old_name.app_uid;\n delete spec.name;\n }\n }\n }\n\n const appByUid = new Map();\n const appByName = new Map();\n const appById = new Map();\n\n const addApp = (app) => {\n if ( ! app ) return;\n appByUid.set(app.uid, app);\n appByName.set(app.name, app);\n appById.set(app.id, app);\n };\n\n const pendingLookups = new Map();\n const pendingToResolve = new Map();\n const queryUids = new Set();\n const queryNames = new Set();\n const queryIds = new Set();\n\n const queueMissing = (type, value) => {\n const queryKey = `${type}:${value}`;\n if ( pendingToResolve.has(queryKey) || pendingLookups.has(queryKey) ) {\n return;\n }\n\n const separatorIndex = queryKey.indexOf(':');\n const lookup = queryKey.slice(0, separatorIndex);\n value = queryKey.slice(separatorIndex + 1);\n const pendingKey = AppRedisCacheSpace.pendingKey({\n lookup,\n value,\n });\n const pending = kv.get(pendingKey);\n if ( pending ) {\n pendingLookups.set(queryKey, pending);\n return;\n }\n\n let resolveQuery;\n let rejectQuery;\n const queryPromise = new Promise((resolve, reject) => {\n resolveQuery = resolve;\n rejectQuery = reject;\n });\n kv.set(pendingKey, queryPromise, { 'EX': PENDING_QUERY_TTL });\n pendingToResolve.set(queryKey, { resolveQuery, rejectQuery, pendingKey });\n\n if ( type === 'uid' ) {\n queryUids.add(value);\n } else if ( type === 'name' ) {\n queryNames.add(value);\n } else if ( type === 'id' ) {\n queryIds.add(value);\n }\n };\n\n const cacheLookupPlan = normalized.map((spec) => {\n if ( spec.uid ) {\n return {\n lookup: 'uid',\n value: spec.uid,\n cacheKey: AppRedisCacheSpace.key({\n lookup: 'uid',\n value: spec.uid,\n }),\n };\n }\n if ( spec.name ) {\n return {\n lookup: 'name',\n value: spec.name,\n cacheKey: AppRedisCacheSpace.key({\n lookup: 'name',\n value: spec.name,\n }),\n };\n }\n if ( spec.id ) {\n return {\n lookup: 'id',\n value: spec.id,\n cacheKey: AppRedisCacheSpace.key({\n lookup: 'id',\n value: spec.id,\n }),\n };\n }\n return null;\n });\n\n const cachedAppsByKey = await redisGetJsonMany(\n cacheLookupPlan.filter(Boolean).map(item => item.cacheKey),\n );\n\n for ( const plannedLookup of cacheLookupPlan ) {\n if ( ! plannedLookup ) continue;\n let cached = cachedAppsByKey.get(plannedLookup.cacheKey);\n if ( isDecoratedAppCacheEntry(cached) ) {\n AppRedisCacheSpace.invalidateCachedApp(cached);\n cached = null;\n }\n if ( cached ) {\n addApp(decorateApp(cached));\n } else {\n queueMissing(plannedLookup.lookup, plannedLookup.value);\n }\n }\n\n const pendingResultsPromise = pendingLookups.size\n ? Promise.all(Array.from(pendingLookups.values()))\n : Promise.resolve([]);\n\n if ( queryUids.size || queryNames.size || queryIds.size ) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'apps');\n\n const clauses = [];\n const params = [];\n\n if ( queryUids.size ) {\n const uids = Array.from(queryUids);\n clauses.push(`uid IN (${uids.map(() => '?').join(', ')})`);\n params.push(...uids);\n }\n if ( queryNames.size ) {\n const names = Array.from(queryNames);\n clauses.push(`name IN (${names.map(() => '?').join(', ')})`);\n params.push(...names);\n }\n if ( queryIds.size ) {\n const ids = Array.from(queryIds);\n clauses.push(`id IN (${ids.map(() => '?').join(', ')})`);\n params.push(...ids);\n }\n\n let rows = [];\n const resolvedKeys = new Set();\n try {\n rows = await db.read(\n `SELECT *, CASE WHEN icon LIKE 'data:%' THEN 1 ELSE 0 END AS icon_is_base64 FROM \\`apps\\` WHERE ${clauses.join(' OR ')}`,\n params,\n );\n for ( const app of rows ) {\n const appForCache = normalizeAppForCache(app);\n cacheApp(appForCache);\n const decorated_app = decorateApp(appForCache);\n addApp(decorated_app);\n\n const uidKey = `uid:${appForCache.uid}`;\n const nameKey = `name:${appForCache.name}`;\n const idKey = `id:${appForCache.id}`;\n\n if ( pendingToResolve.has(uidKey) ) {\n pendingToResolve.get(uidKey).resolveQuery(appForCache);\n resolvedKeys.add(uidKey);\n }\n if ( pendingToResolve.has(nameKey) ) {\n pendingToResolve.get(nameKey).resolveQuery(appForCache);\n resolvedKeys.add(nameKey);\n }\n if ( pendingToResolve.has(idKey) ) {\n pendingToResolve.get(idKey).resolveQuery(appForCache);\n resolvedKeys.add(idKey);\n }\n }\n\n for ( const [key, { resolveQuery }] of pendingToResolve.entries() ) {\n if ( ! resolvedKeys.has(key) ) {\n resolveQuery(null);\n }\n }\n } catch ( err ) {\n for ( const { rejectQuery } of pendingToResolve.values() ) {\n rejectQuery(err);\n }\n throw err;\n } finally {\n for ( const { pendingKey } of pendingToResolve.values() ) {\n kv.del(pendingKey);\n }\n }\n\n }\n\n const pendingResults = await pendingResultsPromise;\n for ( const app of pendingResults ) {\n addApp(decorateApp(app));\n }\n\n return normalized.map(spec => {\n let app;\n if ( spec.uid ) {\n app = appByUid.get(spec.uid);\n } else if ( spec.name ) {\n app = appByName.get(spec.name);\n } else if ( spec.id ) {\n app = appById.get(spec.id);\n }\n if ( ! app ) return null;\n const result = { ...app };\n delete result.icon_is_base64;\n return result;\n });\n\n});\n\n/**\n * Checks to see if an app exists\n *\n * @param {string} options - `options`\n * @returns {Promise}\n */\nexport async function app_exists (options) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'apps');\n\n let app;\n if ( options.uid )\n {\n app = await db.read('SELECT `id` FROM `apps` WHERE `uid` = ? LIMIT 1', [options.uid]);\n }\n else if ( options.name )\n {\n app = await db.read('SELECT `id` FROM `apps` WHERE `name` = ? LIMIT 1', [options.name]);\n }\n else if ( options.id )\n {\n app = await db.read('SELECT `id` FROM `apps` WHERE `id` = ? LIMIT 1', [options.id]);\n }\n\n return app[0];\n}\n\n/**\n * change username\n *\n * @param {string} options - `options`\n * @returns {Promise}\n */\nexport async function change_username (user_id, new_username) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_WRITE, 'auth');\n\n const old_username = (await get_user({ id: user_id })).username;\n\n // update username\n await db.write('UPDATE `user` SET username = ? WHERE `id` = ? LIMIT 1', [new_username, user_id]);\n // update root directory name for this user\n await db.write(\n 'UPDATE `fsentries` SET `name` = ?, `path` = ? ' +\n 'WHERE `user_id` = ? AND parent_uid IS NULL LIMIT 1',\n [new_username, `/${ new_username}`, user_id],\n );\n\n console.log(`User ${old_username} changed username to ${new_username}`);\n await servicesContainer.services.get('filesystem').update_child_paths(`/${old_username}`, `/${new_username}`, user_id);\n\n invalidate_cached_user_by_id(user_id);\n}\n\n/**\n * Find a FSEntry by its uuid\n *\n * @param {integer} id - `id` of FSEntry\n * @returns {Promise} Promise object represents the UUID of the FileSystem Entry\n * @deprecated Use fs middleware instead\n */\nexport async function uuid2fsentry (uuid, return_thumbnail) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n // todo optim, check if uuid is not exactly 36 characters long, if not it's invalid\n // and we can avoid one unnecessary DB lookup\n let fsentry = await db.requireRead(\n `SELECT\n id,\n associated_app_id,\n uuid,\n public_token,\n bucket,\n bucket_region,\n file_request_token,\n user_id,\n parent_uid,\n is_dir,\n is_public,\n is_shortcut,\n shortcut_to,\n sort_by,\n ${return_thumbnail ? 'thumbnail,' : ''}\n immutable,\n name,\n metadata,\n modified,\n created,\n accessed,\n size\n FROM fsentries WHERE uuid = ? LIMIT 1`,\n [uuid],\n );\n\n if ( ! fsentry[0] )\n {\n return false;\n }\n else\n {\n return fsentry[0];\n }\n}\n\n/**\n * Find a FSEntry by its id\n *\n * @param {integer} id - `id` of FSEntry\n * @returns {Promise} Promise object represents the UUID of the FileSystem Entry\n */\nexport async function id2fsentry (id, return_thumbnail) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n // todo optim, check if uuid is not exactly 36 characters long, if not it's invalid\n // and we can avoid one unnecessary DB lookup\n let fsentry = await db.requireRead(\n `SELECT\n id,\n uuid,\n public_token,\n file_request_token,\n associated_app_id,\n user_id,\n parent_uid,\n is_dir,\n is_public,\n is_shortcut,\n shortcut_to,\n sort_by,\n ${return_thumbnail ? 'thumbnail,' : ''}\n immutable,\n name,\n metadata,\n modified,\n created,\n accessed,\n size\n FROM fsentries WHERE id = ? LIMIT 1`,\n [id],\n );\n\n if ( ! fsentry[0] ) {\n return false;\n } else\n {\n return fsentry[0];\n }\n}\n\n/**\n * Takes a an absolute path and returns its corresponding FSEntry.\n *\n * @param {string} path - absolute path of the filesystem entry to be resolved\n * @param {boolean} return_content - if FSEntry is a file, determines whether its content should be returned\n * @returns {false|object} - `false` if path could not be resolved, otherwise an object representing the FSEntry\n * @deprecated Use fs middleware instead\n */\nexport async function convert_path_to_fsentry (path) {\n // todo optim, check if path is valid (e.g. contaisn valid characters)\n // if syntactical errors are found we can potentially avoid some expensive db lookups\n\n // '/' means that parent_uid is null\n // TODO: facade fsentry for root (devlog:2023-06-01)\n if ( path === '/' )\n {\n return null;\n }\n //first slash is redundant\n path = path.substr(path.indexOf('/') + 1);\n //last slash, if existing is redundant\n if ( path[path.length - 1] === '/' )\n {\n path = path.slice(0, -1);\n }\n //split path into parts\n const fsentry_names = path.split('/');\n\n // if no parts, return false\n if ( fsentry_names.length === 0 )\n {\n return false;\n }\n\n let parent_uid = null;\n let final_res = null;\n let is_public = false;\n let result;\n\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n // Try stored path first\n result = await db.read(\n 'SELECT * FROM fsentries WHERE path=? LIMIT 1',\n [`/${ path}`],\n );\n\n if ( result[0] ) {\n return result[0];\n }\n\n for ( let i = 0; i < fsentry_names.length; i++ ) {\n if ( parent_uid === null ) {\n result = await db.read(\n 'SELECT * FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1',\n [fsentry_names[i]],\n );\n }\n else {\n result = await db.read(\n 'SELECT * FROM fsentries WHERE parent_uid = ? AND name=? LIMIT 1',\n [parent_uid, fsentry_names[i]],\n );\n }\n\n if ( result[0] ) {\n parent_uid = result[0].uuid;\n // is_public is either directly specified or inherited from parent dir\n if ( result[0].is_public === null )\n {\n result[0].is_public = is_public;\n }\n else\n {\n is_public = result[0].is_public;\n }\n\n } else {\n return false;\n }\n final_res = result;\n }\n return final_res[0];\n}\n\n/**\n *\n * @param {integer} bytes - size in bytes\n * @returns {string} bytes in human-readable format\n */\nexport function byte_format (bytes) {\n // calculate and return bytes in human-readable format\n const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];\n if ( typeof bytes !== 'number' || bytes < 1 ) {\n return '0 B';\n }\n const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)));\n return `${Math.round(bytes / Math.pow(1024, i), 2) } ${ sizes[i]}`;\n};\n\nexport const get_descendants = spanify('get_descendants', async (...args) => {\n return await getDescendantsHelper(...args);\n});\n\n/**\n *\n * @param {integer} entry_id\n * @returns\n */\nexport const id2path = spanify('helpers:id2path', async (entry_uid) => {\n if ( entry_uid == null ) {\n throw new Error('got null or undefined entry id');\n }\n\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n const log = servicesContainer.services.get('log-service').create('helpers.id2path');\n log.traceOn();\n const errors = servicesContainer.services.get('error-service').create(log);\n log.called();\n\n let result;\n\n log.debug(`entry id: ${entry_uid}`);\n if ( typeof entry_uid === 'number' ) {\n const old = entry_uid;\n entry_uid = await id2uuid(entry_uid);\n log.debug(`entry id resolved: resolved ${old} ${entry_uid}`);\n }\n\n try {\n result = await db.read(`\n WITH RECURSIVE cte AS (\n SELECT uuid, parent_uid, name, name AS path\n FROM fsentries\n WHERE uuid = ?\n\n UNION ALL\n\n SELECT e.uuid, e.parent_uid, e.name, ${\n db.case({\n sqlite: 'e.name || \\'/\\' || cte.path',\n otherwise: 'CONCAT(e.name, \\'/\\', cte.path)',\n })\n }\n FROM fsentries e\n INNER JOIN cte ON cte.parent_uid = e.uuid\n )\n SELECT *\n FROM cte\n WHERE parent_uid IS NULL\n `, [entry_uid]);\n } catch (e) {\n errors.report('id2path.select', {\n alarm: true,\n source: e,\n message: `error while resolving path for ${entry_uid}: ${e.message}`,\n extra: {\n entry_uid,\n },\n });\n throw new ManagedError(`cannot create path for ${entry_uid}`);\n }\n\n if ( !result || !result[0] ) {\n errors.report('id2path.select', {\n alarm: true,\n message: `no result for ${entry_uid}`,\n extra: {\n entry_uid,\n },\n });\n throw new ManagedError(`cannot create path for ${entry_uid}`);\n }\n\n return `/${ result[0].path}`;\n});\n\n/**\n * Recursively retrieve all files, directories, and subdirectories under `path`.\n * Optionally the `depth` can be set.\n *\n * @param {string} path\n * @param {object} user\n * @param {integer} depth\n * @returns\n */\nasync function getDescendantsHelper (path, user, depth, return_thumbnail = false) {\n const log = servicesContainer.services.get('log-service').create('get_descendants');\n log.called();\n\n // decrement depth if it's set\n depth !== undefined && depth--;\n // turn path into absolute form\n path = _resolve('/', path);\n // get parent dir\n const parent = await convert_path_to_fsentry(path);\n // holds array that will be returned\n const ret = [];\n // holds immediate children of this path\n let children;\n\n // try to extract username from path\n let username;\n let split_path = path.split('/');\n if ( split_path.length === 2 && split_path[0] === '' )\n {\n username = split_path[1];\n }\n\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n // -------------------------------------\n // parent is root ('/')\n // -------------------------------------\n if ( parent === null ) {\n path = '';\n // direct children under root\n children = await db.read(\n `SELECT\n id, uuid, parent_uid, name, metadata, is_dir, bucket, bucket_region,\n modified, created, immutable, shortcut_to, is_shortcut, sort_by, associated_app_id,\n ${return_thumbnail ? 'thumbnail, ' : ''}\n accessed, size\n FROM fsentries\n WHERE user_id = ? AND parent_uid IS NULL`,\n [user.id],\n );\n // users that have shared files/dirs with this user\n const sharing_users = await db.read(\n `SELECT DISTINCT(owner_user_id), user.username\n FROM share\n INNER JOIN user ON user.id = share.owner_user_id\n WHERE share.recipient_user_id = ?`,\n [user.id],\n );\n if ( sharing_users.length > 0 ) {\n for ( let i = 0; i < sharing_users.length; i++ ) {\n let dir = {};\n dir.id = null;\n dir.uuid = null;\n dir.parent_uid = null;\n dir.name = sharing_users[i].username;\n dir.is_dir = true;\n dir.immutable = true;\n children.push(dir);\n }\n }\n }\n // -------------------------------------\n // parent doesn't exist\n // -------------------------------------\n else if ( parent === false ) {\n return [];\n }\n // -------------------------------------\n // Parent is a shared-user directory: /[some_username](/)\n // but make sure `[some_username]` is not the same as the requester's username\n // -------------------------------------\n else if ( username && username !== user.username ) {\n children = [];\n let sharing_user;\n sharing_user = await get_user({ username: username });\n if ( ! sharing_user )\n {\n return [];\n }\n\n // shared files/dirs with this user\n const shared_fsentries = await db.read(\n `SELECT\n fsentries.id, fsentries.user_id, fsentries.uuid, fsentries.parent_uid, fsentries.bucket, fsentries.bucket_region,\n fsentries.name, fsentries.shortcut_to, fsentries.is_shortcut, fsentries.metadata, fsentries.is_dir, fsentries.modified,\n fsentries.created, fsentries.accessed, fsentries.size, fsentries.sort_by, fsentries.associated_app_id,\n fsentries.is_symlink, fsentries.symlink_path,\n fsentries.immutable ${return_thumbnail ? ', fsentries.thumbnail' : ''}\n FROM share\n INNER JOIN fsentries ON fsentries.id = share.fsentry_id\n WHERE share.recipient_user_id = ? AND owner_user_id = ?`,\n [user.id, sharing_user.id],\n );\n // merge `children` and `shared_fsentries`\n if ( shared_fsentries.length > 0 ) {\n for ( let i = 0; i < shared_fsentries.length; i++ ) {\n shared_fsentries[i].path = await id2path(shared_fsentries[i].id);\n children.push(shared_fsentries[i]);\n }\n }\n }\n // -------------------------------------\n // All other cases\n // -------------------------------------\n else {\n children = [];\n let temp_children = await db.read(\n `SELECT\n id, user_id, uuid, parent_uid, name, metadata, is_shortcut,\n shortcut_to, is_dir, modified, created, accessed, size, sort_by, associated_app_id,\n is_symlink, symlink_path,\n immutable ${return_thumbnail ? ', thumbnail' : ''}\n FROM fsentries\n WHERE parent_uid = ?`,\n [parent.uuid],\n );\n // check if user has access to each file, if yes add it\n if ( temp_children.length > 0 ) {\n for ( let i = 0; i < temp_children.length; i++ ) {\n const tchild = temp_children[i];\n if ( await chkperm(tchild, user.id) )\n {\n children.push(tchild);\n }\n }\n }\n }\n\n // shortcut on empty result set\n if ( children.length === 0 ) return [];\n\n const ids = children.map(child => child.id);\n const qmarks = ids.map(() => '?').join(',');\n\n let rows = await db.read(\n `SELECT root_dir_id FROM subdomains WHERE root_dir_id IN (${qmarks}) AND user_id=?`,\n [...ids, user.id],\n );\n\n const websiteMap = {};\n for ( const row of rows ) websiteMap[row.root_dir_id] = true;\n\n for ( let i = 0; i < children.length; i++ ) {\n const contentType = _contentType(children[i].name);\n\n // has_website\n let has_website = false;\n if ( children[i].is_dir ) {\n has_website = websiteMap[children[i].id];\n }\n\n // object to return\n // TODO: DRY creation of response fsentry from db fsentry\n ret.push({\n path: children[i].path ?? (`${path }/${ children[i].name}`),\n name: children[i].name,\n metadata: children[i].metadata,\n _id: children[i].id,\n id: children[i].uuid,\n uid: children[i].uuid,\n is_shortcut: children[i].is_shortcut,\n shortcut_to: (children[i].shortcut_to ? await id2uuid(children[i].shortcut_to) : undefined),\n shortcut_to_path: (children[i].shortcut_to ? await id2path(children[i].shortcut_to) : undefined),\n is_symlink: children[i].is_symlink,\n symlink_path: children[i].symlink_path,\n immutable: children[i].immutable,\n is_dir: children[i].is_dir,\n modified: children[i].modified,\n created: children[i].created,\n accessed: children[i].accessed,\n size: children[i].size,\n sort_by: children[i].sort_by,\n thumbnail: children[i].thumbnail,\n associated_app_id: children[i].associated_app_id,\n type: contentType ? contentType : null,\n has_website: has_website,\n });\n if ( children[i].is_dir &&\n (depth === undefined || (depth !== undefined && depth > 0))\n ) {\n ret.push(await get_descendants(`${path }/${ children[i].name}`, user, depth));\n }\n }\n return ret.flat();\n};\n\nexport const get_dir_size = async (path, user) => {\n let size = 0;\n const descendants = await get_descendants(path, user);\n for ( let i = 0; i < descendants.length; i++ ) {\n if ( ! descendants[i].is_dir ) {\n size += descendants[i].size;\n }\n }\n\n return size;\n};\n\n/**\n *\n * @param {string} glob\n * @param {object} user\n * @returns\n */\nexport async function resolve_glob (glob, user) {\n //turn glob into abs path\n glob = _resolve('/', glob);\n //get base of glob\n const base = micromatch.scan(glob).base;\n //estimate needed depth\n let depth = 1;\n const dirs = glob.split('/');\n for ( let i = 0; i < dirs.length; i++ ) {\n if ( dirs[i].includes('**') ) {\n depth = undefined;\n break;\n } else {\n depth++;\n }\n }\n\n const descendants = await get_descendants(base, user, depth);\n\n return descendants.filter((fsentry) => {\n return fsentry.path && micromatch.isMatch(fsentry.path, glob);\n });\n}\n\nfunction isString (variable) {\n return typeof variable === 'string' || variable instanceof String;\n}\n\nexport const body_parser_error_handler = (err, req, res, next) => {\n if ( err instanceof SyntaxError && err.status === 400 && 'body' in err ) {\n return res.status(400).send(err); // Bad request\n }\n next();\n};\n\n/**\n * Given a uid, returns a file node.\n *\n * TODO (xiaochen): It only works for MemoryFSProvider currently.\n *\n * @param {string} uid - The uid of the file to get.\n * @returns {Promise} The file node, or null if the file does not exist.\n */\nasync function get_entry (uid) {\n const svc_mountpoint = Context.get('services').get('mountpoint');\n const uid_selector = new NodeUIDSelector(uid);\n const provider = await svc_mountpoint.get_provider(uid_selector);\n\n // NB: We cannot import MemoryFSProvider here because it will cause a circular dependency.\n if ( provider.constructor.name !== 'MemoryFSProvider' ) {\n return null;\n }\n\n return provider.stat({\n selector: uid_selector,\n });\n}\n\nexport async function is_ancestor_of (ancestor_uid, descendant_uid) {\n const ancestor = await get_entry(ancestor_uid);\n const descendant = await get_entry(descendant_uid);\n\n if ( ancestor && descendant ) {\n return descendant.path.startsWith(ancestor.path);\n }\n\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n // root is an ancestor to all FSEntries\n if ( ancestor_uid === null )\n {\n return true;\n }\n // root is never a descendant to any FSEntries\n if ( descendant_uid === null )\n {\n return false;\n }\n\n if ( typeof ancestor_uid === 'number' ) {\n ancestor_uid = await id2uuid(ancestor_uid);\n }\n if ( typeof descendant_uid === 'number' ) {\n descendant_uid = await id2uuid(descendant_uid);\n }\n\n let parent = await db.read('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]);\n if ( parent[0] === undefined )\n {\n parent = await db.pread('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]);\n }\n if ( parent[0].uuid === ancestor_uid || parent[0].parent_uid === ancestor_uid ) {\n return true;\n }\n // keep checking as long as parent of parent is not root\n while ( parent[0].parent_uid !== null ) {\n parent = await db.read('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [parent[0].parent_uid]);\n if ( parent[0] === undefined ) {\n parent = await db.pread('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]);\n }\n\n if ( parent[0].uuid === ancestor_uid || parent[0].parent_uid === ancestor_uid ) {\n return true;\n }\n }\n\n return false;\n}\n\nexport async function sign_file (fsentry, action) {\n\n // fsentry not found\n if ( fsentry === false ) {\n throw { message: 'No entry found with this uid' };\n }\n\n const uid = fsentry.uuid ?? (fsentry.uid ?? fsentry._id);\n const ttl = 9999999999999;\n const secret = config.url_signature_secret;\n const expires = Math.ceil(Date.now() / 1000) + ttl;\n const signature = sha256(`${uid}/${action}/${secret}/${expires}`);\n const contentType = _contentType(fsentry.name);\n\n // return\n return {\n uid: uid,\n expires: expires,\n signature: signature,\n url: `${config.api_base_url}/file?uid=${uid}&expires=${expires}&signature=${signature}`,\n read_url: `${config.api_base_url}/file?uid=${uid}&expires=${expires}&signature=${signature}`,\n write_url: `${config.api_base_url}/writeFile?uid=${uid}&expires=${expires}&signature=${signature}`,\n metadata_url: `${config.api_base_url}/itemMetadata?uid=${uid}&expires=${expires}&signature=${signature}`,\n fsentry_type: contentType,\n fsentry_is_dir: !!fsentry.is_dir,\n fsentry_name: fsentry.name,\n fsentry_size: fsentry.size,\n fsentry_accessed: fsentry.accessed,\n fsentry_modified: fsentry.modified,\n fsentry_created: fsentry.created,\n };\n}\n\nexport async function gen_public_token (file_uuid) {\n\n // get fsentry\n let fsentry = await uuid2fsentry(file_uuid);\n\n // fsentry not found\n if ( fsentry === false ) {\n throw { message: 'No entry found with this uid' };\n }\n\n const uid = fsentry.uuid;\n const token = v4();\n const contentType = _contentType(fsentry.name);\n\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_WRITE, 'filesystem');\n\n // insert into DB\n try {\n await db.write(\n 'UPDATE fsentries SET public_token = ? WHERE id = ?',\n [\n //token\n token,\n //fsentry_id\n fsentry.id,\n ],\n );\n } catch (e) {\n console.log(e);\n return false;\n }\n\n // return\n return {\n uid: uid,\n token: token,\n url: `${config.api_base_url}/pubfile?token=${token}`,\n fsentry_type: contentType,\n fsentry_is_dir: fsentry.is_dir,\n fsentry_name: fsentry.name,\n };\n}\n\nexport async function deleteUser (user_id) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n const svc_fs = servicesContainer.services.get('filesystem');\n\n // get a list of up to 5000 files owned by this user\n // eslint-disable-next-line no-constant-condition\n for ( let offset = 0; true; offset += 5000 ) {\n let files = await db.read(\n `SELECT uuid, bucket, bucket_region FROM fsentries WHERE user_id = ? AND is_dir = 0 LIMIT 5000 OFFSET ${ offset}`,\n [user_id],\n );\n\n if ( !files || files.length == 0 ) break;\n\n // delete all files from S3\n if ( files !== null && files.length > 0 ) {\n for ( let i = 0; i < files.length; i++ ) {\n const node = await svc_fs.node(new NodeUIDSelector(files[i].uuid));\n\n await node.provider.unlink({\n context: Context.get(),\n override_immutable: true,\n node,\n });\n }\n }\n }\n\n // delete all fsentries from DB\n await db.write('DELETE FROM fsentries WHERE user_id = ?', [user_id]);\n\n // delete user\n await db.write('DELETE FROM user WHERE id = ?', [user_id]);\n}\n\nexport function subdomain (req) {\n if ( config.experimental_no_subdomain ) return 'api';\n return req.hostname.slice(0, -1 * (config.domain.length + 1));\n}\n\nexport async function jwt_auth (req, authService) {\n let token;\n // HTTML Auth header\n if ( req.header && req.header('Authorization') )\n {\n token = req.header('Authorization');\n }\n // Cookie\n else if ( req.cookies && req.cookies[config.cookie_name] )\n {\n token = req.cookies[config.cookie_name];\n }\n // Auth token in URL\n else if ( req.query && req.query.auth_token )\n {\n token = req.query.auth_token;\n }\n // Socket\n else if ( req.handshake && req.handshake.auth && req.handshake.auth.auth_token )\n {\n token = req.handshake.auth.auth_token;\n }\n\n if ( !token || token === 'null' )\n {\n throw ('No auth token found');\n }\n else if ( typeof token !== 'string' )\n {\n throw ('token must be a string.');\n }\n else\n {\n token = token.replace('Bearer ', '');\n }\n\n try {\n if ( ! authService ) {\n throw new Error('jwt_auth requires authService');\n }\n\n const actor = await authService.authenticate_from_token(token);\n\n if ( !actor.type?.constructor?.name === 'UserActorType' ) {\n throw ({\n message: APIError.create('token_unsupported')\n .serialize(),\n });\n }\n\n return {\n actor,\n user: actor.type.user,\n token: token,\n };\n } catch (e) {\n if ( ! (e instanceof APIError) ) {\n console.log('ERROR', e);\n }\n throw (e.message);\n }\n}\n\n/**\n * returns all ancestors of an fsentry\n *\n * @param {*} fsentry_id\n */\nexport async function ancestors (fsentry_id) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n const ancestors = [];\n // first parent\n let parent = await db.read('SELECT * FROM `fsentries` WHERE `id` = ? LIMIT 1', [fsentry_id]);\n if ( parent.length === 0 ) {\n return ancestors;\n }\n // get all subsequent parents\n while ( parent[0].parent_uid !== null ) {\n const parent_fsentry = await uuid2fsentry(parent[0].parent_uid);\n parent = await db.read('SELECT * FROM `fsentries` WHERE `id` = ? LIMIT 1', [parent_fsentry.id]);\n if ( parent[0].length !== 0 ) {\n ancestors.push(parent[0]);\n }\n }\n\n return ancestors;\n}\n\nexport function hyphenize_confirm_code (email_confirm_code) {\n email_confirm_code = email_confirm_code.toString();\n email_confirm_code =\n `${email_confirm_code[0] +\n email_confirm_code[1] +\n email_confirm_code[2]\n }-${\n email_confirm_code[3]\n }${email_confirm_code[4]\n }${email_confirm_code[5]}`;\n return email_confirm_code;\n}\n\nexport async function username_exists (username) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n let rows = await db.read('SELECT EXISTS(SELECT 1 FROM user WHERE username=?) AS username_exists', [username]);\n if ( rows[0].username_exists )\n {\n return true;\n }\n}\n\nexport async function generate_random_username () {\n let username;\n do {\n username = generate_identifier();\n } while ( await username_exists(username) );\n return username;\n}\n\nexport async function app_name_exists (name) {\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');\n\n let rows = await db.read('SELECT EXISTS(SELECT 1 FROM apps WHERE apps.name=?) AS app_name_exists', [name]);\n if ( rows[0].app_name_exists )\n {\n return true;\n }\n\n const svc_oldAppName = servicesContainer.services.get('old-app-name');\n const name_info = await svc_oldAppName.check_app_name(name);\n if ( name_info ) return true;\n}\n\nexport function send_email_verification_code (email_confirm_code, email) {\n const svc_email = Context.get('services').get('email');\n svc_email.send_email({ email }, 'email_verification_code', {\n code: hyphenize_confirm_code(email_confirm_code),\n });\n}\n\nexport function send_email_verification_token (email_confirm_token, email, user_uuid) {\n const svc_email = Context.get('services').get('email');\n const link = `${config.origin}/confirm-email-by-token?user_uuid=${user_uuid}&token=${email_confirm_token}`;\n svc_email.send_email({ email }, 'email_verification_link', { link });\n}\n\nexport function generate_random_str (length) {\n let result = '';\n const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';\n const charactersLength = characters.length;\n for ( let i = 0; i < length; i++ ) {\n result += characters.charAt(Math.floor(Math.random() *\n charactersLength));\n }\n return result;\n}\n\n/**\n * Converts a given number of seconds into a human-readable string format.\n *\n * @param {number} seconds - The number of seconds to be converted.\n * @returns {string} The time represented in the format: 'X years Y days Z hours A minutes B seconds'.\n * @throws {TypeError} If the `seconds` parameter is not a number.\n */\nexport function seconds_to_string (seconds) {\n const numyears = Math.floor(seconds / 31536000);\n const numdays = Math.floor((seconds % 31536000) / 86400);\n const numhours = Math.floor(((seconds % 31536000) % 86400) / 3600);\n const numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60);\n const numseconds = (((seconds % 31536000) % 86400) % 3600) % 60;\n return `${numyears } years ${ numdays } days ${ numhours } hours ${ numminutes } minutes ${ numseconds } seconds`;\n}\n\n/**\n * returns a list of apps that could open the fsentry, ranked by relevance\n * @param {*} fsentry\n * @param {*} options\n */\nconst SUGGEST_APP_CODE_EXTS = [\n '.asm',\n '.asp',\n '.aspx',\n '.bash',\n '.c',\n '.cpp',\n '.css',\n '.csv',\n '.dhtml',\n '.f',\n '.go',\n '.h',\n '.htm',\n '.html',\n '.html5',\n '.java',\n '.jl',\n '.js',\n '.jsa',\n '.json',\n '.jsonld',\n '.jsf',\n '.jsp',\n '.kt',\n '.log',\n '.lock',\n '.lua',\n '.md',\n '.perl',\n '.phar',\n '.php',\n '.pl',\n '.py',\n '.r',\n '.rb',\n '.rdata',\n '.rda',\n '.rdf',\n '.rds',\n '.rs',\n '.rlib',\n '.rpy',\n '.scala',\n '.sc',\n '.scm',\n '.sh',\n '.sol',\n '.sql',\n '.ss',\n '.svg',\n '.swift',\n '.toml',\n '.ts',\n '.wasm',\n '.xhtml',\n '.xml',\n '.yaml',\n];\n\nconst buildSuggestedAppSpecifiers = async (fsentry) => {\n const name_specifiers = [];\n\n let content_type = _contentType(fsentry.name);\n if ( ! content_type ) content_type = '';\n\n // IIFE just so fsname can stay `const`\n const fsname = (() => {\n if ( ! fsentry.name ) {\n return 'missing-fsentry-name';\n }\n let fsname = fsentry.name.toLowerCase();\n // We add `.directory` so that this works as a file association\n if ( fsentry.is_dir ) fsname += '.directory';\n return fsname;\n })();\n const file_extension = extname(fsname).toLowerCase();\n\n const any_of = (list, name) => list.some(v => name.endsWith(v));\n\n //---------------------------------------------\n // Code\n //---------------------------------------------\n if ( any_of(SUGGEST_APP_CODE_EXTS, fsname) || !fsname.includes('.') ) {\n name_specifiers.push({ name: 'code' });\n name_specifiers.push({ name: 'editor' });\n }\n\n //---------------------------------------------\n // Editor\n //---------------------------------------------\n if (\n fsname.endsWith('.txt') ||\n // files with no extension\n !fsname.includes('.')\n ) {\n name_specifiers.push({ name: 'editor' });\n name_specifiers.push({ name: 'code' });\n }\n //---------------------------------------------\n // Markus\n //---------------------------------------------\n if ( fsname.endsWith('.md') ) {\n name_specifiers.push({ name: 'markus' });\n }\n //---------------------------------------------\n // Viewer\n //---------------------------------------------\n if (\n fsname.endsWith('.jpg') ||\n fsname.endsWith('.png') ||\n fsname.endsWith('.webp') ||\n fsname.endsWith('.svg') ||\n fsname.endsWith('.bmp') ||\n fsname.endsWith('.jpeg')\n ) {\n name_specifiers.push({ name: 'viewer' });\n }\n //---------------------------------------------\n // Draw\n //---------------------------------------------\n if (\n fsname.endsWith('.bmp') ||\n content_type.startsWith('image/')\n ) {\n name_specifiers.push({ name: 'draw' });\n }\n //---------------------------------------------\n // PDF\n //---------------------------------------------\n if ( fsname.endsWith('.pdf') ) {\n name_specifiers.push({ name: 'pdf' });\n }\n //---------------------------------------------\n // Player\n //---------------------------------------------\n if (\n fsname.endsWith('.mp4') ||\n fsname.endsWith('.webm') ||\n fsname.endsWith('.mpg') ||\n fsname.endsWith('.mpv') ||\n fsname.endsWith('.mp3') ||\n fsname.endsWith('.m4a') ||\n fsname.endsWith('.ogg')\n ) {\n name_specifiers.push({ name: 'player' });\n }\n\n //---------------------------------------------\n // 3rd-party apps\n //---------------------------------------------\n const apps = safe_json_parse(await redisClient.get(\n AppRedisCacheSpace.associationAppsKey(file_extension.slice(1)),\n ), []);\n /** @type {{id:string}[]} */\n const id_specifiers = apps.map(app_id => ({ id: app_id }));\n\n return { name_specifiers, id_specifiers };\n};\n\nconst buildSuggestedAppsFromResolved = (resolved, name_specifier_count, options) => {\n const suggested_apps = [];\n\n const name_apps = resolved.slice(0, name_specifier_count);\n suggested_apps.push(...name_apps);\n\n const third_party_apps = resolved.slice(name_specifier_count);\n for ( const third_party_app of third_party_apps ) {\n if ( ! third_party_app ) continue;\n if ( third_party_app.approved_for_opening_items ||\n (options?.user && options.user.id === third_party_app.owner_user_id) )\n {\n suggested_apps.push(third_party_app);\n }\n }\n\n const needs_codeapp = suggested_apps.some(app => app && app.name === 'editor');\n return { suggested_apps, needs_codeapp };\n};\n\nconst normalizeSuggestedApps = (suggested_apps) => (\n suggested_apps.filter((suggested_app, pos, self) => {\n // Remove any null values caused by calling `get_app()` for apps that don't exist.\n // This happens on self-host because we don't include `code`, among others.\n if ( ! suggested_app ) {\n return false;\n }\n\n // Remove any duplicate entries\n return self.indexOf(suggested_app) === pos;\n })\n);\n\nconst buildSuggestedAppsCacheKey = (fsentry, options) => {\n const user_id = options?.user?.id ?? '';\n const entry_id = fsentry?.uuid ?? fsentry?.uid ?? fsentry?.id ?? fsentry?.path ?? '';\n const entry_name = fsentry?.name ?? '';\n const entry_type = fsentry?.is_dir ? 'd' : 'f';\n return `${user_id}:${entry_id}:${entry_type}:${entry_name}`;\n};\n\nconst cloneSuggestedApps = (suggested_apps) => (\n Array.isArray(suggested_apps)\n ? suggested_apps.map(app => (app ? { ...app } : app))\n : suggested_apps\n);\n\nexport async function suggestedAppsForFsEntries (fsentries, options) {\n if ( ! Array.isArray(fsentries) ) {\n fsentries = [fsentries];\n }\n\n const batches = [];\n const specifiers = [];\n const results = new Array(fsentries.length);\n const cacheKeysByIndex = new Map();\n\n for ( let index = 0; index < fsentries.length; index++ ) {\n const fsentry = fsentries[index];\n if ( ! fsentry ) {\n results[index] = [];\n continue;\n }\n\n const cache_key = buildSuggestedAppsCacheKey(fsentry, options);\n const cached = suggestedAppsCache.get(cache_key);\n if ( cached !== undefined ) {\n results[index] = cloneSuggestedApps(cached);\n continue;\n }\n\n const { name_specifiers, id_specifiers } = await buildSuggestedAppSpecifiers(fsentry);\n const entry_specifiers = [...name_specifiers, ...id_specifiers];\n\n if ( entry_specifiers.length === 0 ) {\n results[index] = [];\n cacheKeysByIndex.set(index, cache_key);\n continue;\n }\n\n const offset = specifiers.length;\n specifiers.push(...entry_specifiers);\n batches.push({\n index,\n offset,\n count: entry_specifiers.length,\n name_count: name_specifiers.length,\n suggested_apps: [],\n needs_codeapp: false,\n });\n cacheKeysByIndex.set(index, cache_key);\n }\n\n let resolved = [];\n if ( specifiers.length > 0 ) {\n resolved = await get_apps(specifiers);\n }\n\n let any_needs_codeapp = false;\n for ( const batch of batches ) {\n const slice = resolved.slice(batch.offset, batch.offset + batch.count);\n const { suggested_apps, needs_codeapp } = buildSuggestedAppsFromResolved(\n slice,\n batch.name_count,\n options,\n );\n batch.suggested_apps = suggested_apps;\n batch.needs_codeapp = needs_codeapp;\n if ( needs_codeapp ) any_needs_codeapp = true;\n }\n\n let codeapp;\n if ( any_needs_codeapp ) {\n [codeapp] = await get_apps([{ name: 'codeapp' }]);\n }\n\n for ( const batch of batches ) {\n let suggested_apps = batch.suggested_apps;\n if ( batch.needs_codeapp && codeapp ) {\n suggested_apps = [...suggested_apps, codeapp];\n }\n results[batch.index] = normalizeSuggestedApps(suggested_apps);\n }\n\n // Deduplicate results by ID\n const deduplicatedResults = results.map(apps => {\n if ( ! Array.isArray(apps) ) return apps;\n const seen = new Set();\n return apps.filter(app => {\n if ( !app || !app.id ) return true;\n if ( seen.has(app.id) ) return false;\n seen.add(app.id);\n return true;\n });\n });\n\n for ( const [index, cache_key] of cacheKeysByIndex ) {\n const apps = deduplicatedResults[index];\n if ( apps !== undefined ) {\n suggestedAppsCache.set(cache_key, cloneSuggestedApps(apps));\n }\n }\n\n return deduplicatedResults;\n}\n\nexport async function suggestedAppForFsEntry (fsentry, options) {\n const [result] = await suggestedAppsForFsEntries([fsentry], options);\n return result;\n}\n\nexport async function get_taskbar_items (user, {\n icon_size: iconSizeFromSnake,\n iconSize: iconSizeFromCamel,\n no_icons,\n} = {}) {\n const iconSize = iconSizeFromCamel ?? iconSizeFromSnake;\n /** @type BaseDatabaseAccessService */\n const db = servicesContainer.services.get('database').get(DB_WRITE, 'filesystem');\n\n let taskbar_items_from_db = [];\n // If taskbar items don't exist (specifically NULL)\n // add default apps.\n if ( ! user.taskbar_items ) {\n taskbar_items_from_db = [\n { name: 'app-center', type: 'app' },\n { name: 'dev-center', type: 'app' },\n { name: 'editor', type: 'app' },\n { name: 'code', type: 'app' },\n { name: 'camera', type: 'app' },\n { name: 'recorder', type: 'app' },\n ];\n await db.write(\n 'UPDATE user SET taskbar_items = ? WHERE id = ?',\n [\n JSON.stringify(taskbar_items_from_db),\n user.id,\n ],\n );\n invalidate_cached_user(user);\n }\n // there are items from before\n else {\n try {\n taskbar_items_from_db = JSON.parse(user.taskbar_items);\n } catch (e) {\n // ignore errors\n }\n }\n\n const app_specifiers = taskbar_items_from_db.map((taskbar_item_from_db) => {\n if ( taskbar_item_from_db.type !== 'app' ) return {};\n if ( taskbar_item_from_db.name === 'explorer' ) return {};\n if ( taskbar_item_from_db.name ) {\n return { name: taskbar_item_from_db.name };\n }\n if ( taskbar_item_from_db.id ) {\n return { id: taskbar_item_from_db.id };\n }\n if ( taskbar_item_from_db.uid ) {\n return { uid: taskbar_item_from_db.uid };\n }\n return {};\n });\n\n const taskbar_apps = await get_apps(app_specifiers);\n\n // get apps that these taskbar items represent\n let taskbar_items = [];\n for ( let index = 0; index < taskbar_items_from_db.length; index++ ) {\n const taskbar_item_from_db = taskbar_items_from_db[index];\n if ( taskbar_item_from_db.type !== 'app' ) continue;\n if ( taskbar_item_from_db.name === 'explorer' ) continue;\n\n const item = taskbar_apps[index];\n\n // if item not found, skip it\n if ( ! item ) continue;\n\n // delete sensitive attributes\n delete item.id;\n delete item.owner_user_id;\n delete item.timestamp;\n // delete item.godmode;\n delete item.approved_for_listing;\n delete item.approved_for_opening_items;\n\n if ( no_icons ) {\n delete item.icon;\n } else {\n item.icon = get_app_icon_url(item, iconSize);\n }\n\n // add to final object\n taskbar_items.push(item);\n }\n\n return taskbar_items;\n}\n\nexport function validate_signature_auth (url, action, options = {}) {\n const query = new URL(url).searchParams;\n\n if ( ! query.get('uid') )\n {\n throw { message: '`uid` is required for signature-based authentication.' };\n }\n else if ( ! action )\n {\n throw { message: '`action` is required for signature-based authentication.' };\n }\n else if ( ! query.get('expires') )\n {\n throw { message: '`expires` is required for signature-based authentication.' };\n }\n else if ( ! query.get('signature') )\n {\n throw { message: '`signature` is required for signature-based authentication.' };\n }\n\n if ( options.uid ) {\n if ( query.get('uid') !== options.uid ) {\n throw { message: 'Authentication failed. `uid` does not match.' };\n }\n }\n\n const expired = query.get('expires') && (query.get('expires') < Date.now() / 1000);\n\n // expired?\n if ( expired )\n {\n throw { message: 'Authentication failed. Signature expired.' };\n }\n\n const uid = query.get('uid');\n const secret = config.url_signature_secret;\n\n // before doing anything, see if this signature is valid for 'write' action, if yes that means every action is allowed\n if ( !expired && query.get('signature') === sha256(`${uid}/write/${secret}/${query.get('expires')}`) )\n {\n return true;\n }\n // if not, check specific actions\n else if ( !expired && query.get('signature') === sha256(`${uid}/${action}/${secret}/${query.get('expires')}`) )\n {\n return true;\n }\n // auth failed\n else\n {\n throw { message: 'Authentication failed' };\n }\n}\n\nexport function get_url_from_req (req) {\n return `${req.protocol }://${ req.get('host') }${req.originalUrl}`;\n}\n\n/**\n * Formats a number with grouped thousands.\n *\n * @param {number|string} number - The number to be formatted. If a string is provided, it must only contain numerical characters, plus and minus signs, and the letter 'E' or 'e' (for scientific notation).\n * @param {number} decimals - The number of decimal points. If a non-finite number is provided, it defaults to 0.\n * @param {string} [dec_point='.'] - The character used for the decimal point. Defaults to '.' if not provided.\n * @param {string} [thousands_sep=','] - The character used for the thousands separator. Defaults to ',' if not provided.\n * @returns {string} The formatted number with grouped thousands, using the specified decimal point and thousands separator characters.\n * @throws {TypeError} If the `number` parameter cannot be converted to a finite number, or if the `decimals` parameter is non-finite and cannot be converted to an absolute number.\n */\nexport function number_format (number, decimals, dec_point, thousands_sep) {\n // Strip all characters but numerical ones.\n number = (`${number }`).replace(/[^0-9+\\-Ee.]/g, '');\n let n = !isFinite(+number) ? 0 : +number,\n prec = !isFinite(+decimals) ? 0 : Math.abs(decimals),\n sep = (typeof thousands_sep === 'undefined') ? ',' : thousands_sep,\n dec = (typeof dec_point === 'undefined') ? '.' : dec_point,\n s = '',\n toFixedFix = function (n, prec) {\n const k = Math.pow(10, prec);\n return `${ Math.round(n * k) / k}`;\n };\n // Fix for IE parseFloat(0.55).toFixed(0) = 0;\n s = (prec ? toFixedFix(n, prec) : `${ Math.round(n)}`).split('.');\n if ( s[0].length > 3 ) {\n s[0] = s[0].replace(/\\B(?=(?:\\d{3})+(?!\\d))/g, sep);\n }\n if ( (s[1] || '').length < prec ) {\n s[1] = s[1] || '';\n s[1] += new Array(prec - s[1].length + 1).join('0');\n }\n return s.join(dec);\n}\n"
},
{
"path": "src/backend/src/index.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } An object containing:\n * - {Object} open_count - Open counts for different time periods\n * - {Object} user_count - Uniqu>e user counts for different time periods\n * - {number|null} referral_count - The number of referrals (all-time only)\n */\n async get_stats (app_uid, options = {}) {\n let period = options.period ?? 'all';\n let stats_grouping = options.grouping;\n let app_creation_ts = options.created_at;\n const parse_cached_int = (value) => {\n if ( value === null || value === undefined ) return null;\n const parsed = parseInt(value, 10);\n return Number.isNaN(parsed) ? null : parsed;\n };\n\n // Check cache first if period is 'all' and no grouping is requested\n if ( period === 'all' && !stats_grouping ) {\n const key_open_count = AppRedisCacheSpace.openCountKey(app_uid);\n const key_user_count = AppRedisCacheSpace.userCountKey(app_uid);\n const key_referral_count = AppRedisCacheSpace.referralCountKey(app_uid);\n\n const [cached_open_count, cached_user_count, cached_referral_count] = await Promise.all([\n redisClient.get(key_open_count),\n redisClient.get(key_user_count),\n redisClient.get(key_referral_count),\n ]);\n\n const cached_open_count_parsed = parse_cached_int(cached_open_count);\n const cached_user_count_parsed = parse_cached_int(cached_user_count);\n if ( cached_open_count_parsed !== null && cached_user_count_parsed !== null ) {\n return {\n open_count: cached_open_count_parsed,\n user_count: cached_user_count_parsed,\n referral_count: parse_cached_int(cached_referral_count),\n };\n }\n }\n\n const db = this.services.get('database').get(DB_READ, 'apps');\n\n const getTimeRange = (period) => {\n const now = new Date();\n const today = new Date(now.getFullYear(), now.getMonth(), now.getDate());\n\n switch ( period ) {\n case 'today':\n return {\n start: today.getTime(),\n end: now.getTime(),\n };\n case 'yesterday': {\n const yesterday = new Date(today);\n yesterday.setDate(yesterday.getDate() - 1);\n return {\n start: yesterday.getTime(),\n end: today.getTime() - 1,\n };\n }\n case '7d': {\n const weekAgo = new Date(now);\n weekAgo.setDate(weekAgo.getDate() - 7);\n return {\n start: weekAgo.getTime(),\n end: now.getTime(),\n };\n }\n case '30d': {\n const monthAgo = new Date(now);\n monthAgo.setDate(monthAgo.getDate() - 30);\n return {\n start: monthAgo.getTime(),\n end: now.getTime(),\n };\n }\n case 'this_week': {\n const firstDayOfWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay());\n return {\n start: firstDayOfWeek.getTime(),\n end: now.getTime(),\n };\n }\n case 'last_week': {\n const firstDayOfLastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay() - 7);\n const firstDayOfThisWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay());\n return {\n start: firstDayOfLastWeek.getTime(),\n end: firstDayOfThisWeek.getTime() - 1,\n };\n }\n case 'this_month': {\n const firstDayOfMonth = new Date(now.getFullYear(), now.getMonth(), 1);\n return {\n start: firstDayOfMonth.getTime(),\n end: now.getTime(),\n };\n }\n case 'last_month': {\n const firstDayOfLastMonth = new Date(now.getFullYear(), now.getMonth() - 1, 1);\n const firstDayOfThisMonth = new Date(now.getFullYear(), now.getMonth(), 1);\n return {\n start: firstDayOfLastMonth.getTime(),\n end: firstDayOfThisMonth.getTime() - 1,\n };\n }\n case 'this_year': {\n const firstDayOfYear = new Date(now.getFullYear(), 0, 1);\n return {\n start: firstDayOfYear.getTime(),\n end: now.getTime(),\n };\n }\n case 'last_year': {\n const firstDayOfLastYear = new Date(now.getFullYear() - 1, 0, 1);\n const firstDayOfThisYear = new Date(now.getFullYear(), 0, 1);\n return {\n start: firstDayOfLastYear.getTime(),\n end: firstDayOfThisYear.getTime() - 1,\n };\n }\n case '12m': {\n const twelveMonthsAgo = new Date(now);\n twelveMonthsAgo.setMonth(twelveMonthsAgo.getMonth() - 12);\n return {\n start: twelveMonthsAgo.getTime(),\n end: now.getTime(),\n };\n }\n case 'all': {\n const start = new Date(app_creation_ts);\n return {\n start: start.getTime(),\n end: now.getTime(),\n };\n }\n default:\n return null;\n }\n };\n\n const timeRange = getTimeRange(period);\n\n // Handle time-based grouping if stats_grouping is specified\n if ( stats_grouping ) {\n const timeFormat = this.mysqlDateFormats[stats_grouping];\n if ( ! timeFormat ) {\n throw new Error(`Invalid stats_grouping: ${stats_grouping}. Supported values are: hour, day, week, month, year`);\n }\n\n // Generate all periods for the time range\n const allPeriods = this.generateAllPeriods(\n new Date(timeRange.start),\n new Date(timeRange.end),\n stats_grouping,\n );\n\n if ( global.clickhouseClient ) {\n const groupByFormat = this.clickhouseGroupByFormats[stats_grouping];\n const timeCondition = timeRange ?\n `AND ts >= ${Math.floor(timeRange.start / 1000)} AND ts < ${Math.floor(timeRange.end / 1000)}` : '';\n\n const [openResult, userResult] = await Promise.all([\n global.clickhouseClient.query({\n query: `\n SELECT \n ${groupByFormat} as period,\n COUNT(_id) as count\n FROM app_opens \n WHERE app_uid = '${app_uid}' \n ${timeCondition}\n GROUP BY period\n ORDER BY period\n `,\n format: 'JSONEachRow',\n }),\n global.clickhouseClient.query({\n query: `\n SELECT \n ${groupByFormat} as period,\n COUNT(DISTINCT user_id) as count\n FROM app_opens \n WHERE app_uid = '${app_uid}' \n ${timeCondition}\n GROUP BY period\n ORDER BY period\n `,\n format: 'JSONEachRow',\n }),\n ]);\n\n const openRows = await openResult.json();\n const userRows = await userResult.json();\n\n // Ensure counts are properly parsed as integers\n const processedOpenRows = openRows.map(row => ({\n period: new Date(row.period),\n count: parseInt(row.count),\n }));\n\n const processedUserRows = userRows.map(row => ({\n period: new Date(row.period),\n count: parseInt(row.count),\n }));\n\n // Calculate totals from the processed rows\n const totalOpenCount = processedOpenRows.reduce((sum, row) => sum + row.count, 0);\n const totalUserCount = processedUserRows.reduce((sum, row) => sum + row.count, 0);\n\n // Generate all periods and merge with actual data\n const allPeriods = this.generateAllPeriods(\n new Date(timeRange.start),\n new Date(timeRange.end),\n stats_grouping,\n );\n\n const completeOpenStats = this.mergeWithGeneratedPeriods(processedOpenRows, allPeriods, stats_grouping);\n const completeUserStats = this.mergeWithGeneratedPeriods(processedUserRows, allPeriods, stats_grouping);\n\n return {\n open_count: totalOpenCount,\n user_count: totalUserCount,\n grouped_stats: {\n open_count: completeOpenStats,\n user_count: completeUserStats,\n },\n referral_count: period === 'all'\n ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))\n : null,\n };\n }\n\n else {\n // MySQL queries for grouped stats\n const queryParams = timeRange ?\n [app_uid, timeRange.start / 1000, timeRange.end / 1000] :\n [app_uid];\n\n const [openResult, userResult] = await Promise.all([\n db.read(`\n SELECT ${db.case({\n mysql: `DATE_FORMAT(FROM_UNIXTIME(ts/1000), '${timeFormat}') as period, `,\n sqlite: `STRFTIME('%Y-%m-%d %H', datetime(ts/1000, 'unixepoch'), '${timeFormat}') as period, `,\n })\n }\n COUNT(_id) as count\n FROM app_opens \n WHERE app_uid = ?\n ${timeRange ? 'AND ts >= ? AND ts < ?' : ''}\n GROUP BY period\n ORDER BY period\n `, queryParams),\n db.read(`\n SELECT ${db.case({\n mysql: `DATE_FORMAT(FROM_UNIXTIME(ts/1000), '${timeFormat}') as period, `,\n sqlite: `STRFTIME('%Y-%m-%d %H', datetime(ts/1000, 'unixepoch'), '${timeFormat}') as period, `,\n })\n }\n COUNT(DISTINCT user_id) as count\n FROM app_opens \n WHERE app_uid = ?\n ${timeRange ? 'AND ts >= ? AND ts < ?' : ''}\n GROUP BY period\n ORDER BY period\n `, queryParams),\n ]);\n\n // Calculate totals\n const totalOpenCount = openResult.reduce((sum, row) => sum + parseInt(row.count), 0);\n const totalUserCount = userResult.reduce((sum, row) => sum + parseInt(row.count), 0);\n\n // Convert MySQL results to the same format as needed\n const openRows = openResult.map(row => ({\n period: row.period,\n count: parseInt(row.count),\n }));\n const userRows = userResult.map(row => ({\n period: row.period,\n count: parseInt(row.count),\n }));\n\n // Merge with generated periods to include zero-value periods\n const completeOpenStats = this.mergeWithGeneratedPeriods(openRows, allPeriods, stats_grouping);\n const completeUserStats = this.mergeWithGeneratedPeriods(userRows, allPeriods, stats_grouping);\n\n return {\n open_count: totalOpenCount,\n user_count: totalUserCount,\n grouped_stats: {\n open_count: completeOpenStats,\n user_count: completeUserStats,\n },\n referral_count: period === 'all'\n ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))\n : null,\n };\n }\n }\n\n // Handle non-grouped stats\n if ( global.clickhouseClient ) {\n const openCountQuery = timeRange\n ? `SELECT COUNT(_id) AS open_count FROM app_opens \n WHERE app_uid = '${app_uid}' \n AND ts >= ${Math.floor(timeRange.start / 1000)} \n AND ts < ${Math.floor(timeRange.end / 1000)}`\n : `SELECT COUNT(_id) AS open_count FROM app_opens \n WHERE app_uid = '${app_uid}'`;\n\n const userCountQuery = timeRange\n ? `SELECT COUNT(DISTINCT user_id) AS uniqueUsers FROM app_opens \n WHERE app_uid = '${app_uid}' \n AND ts >= ${Math.floor(timeRange.start / 1000)} \n AND ts < ${Math.floor(timeRange.end / 1000)}`\n : `SELECT COUNT(DISTINCT user_id) AS uniqueUsers FROM app_opens \n WHERE app_uid = '${app_uid}'`;\n\n const [openResult, userResult] = await Promise.all([\n global.clickhouseClient.query({\n query: openCountQuery,\n format: 'JSONEachRow',\n }),\n global.clickhouseClient.query({\n query: userCountQuery,\n format: 'JSONEachRow',\n }),\n ]);\n\n const openRows = await openResult.json();\n const userRows = await userResult.json();\n\n const results = {\n open_count: parseInt(openRows[0].open_count),\n user_count: parseInt(userRows[0].uniqueUsers),\n referral_count: period === 'all'\n ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))\n : null,\n };\n\n // Cache the results if period is 'all'\n if ( period === 'all' ) {\n const key_open_count = AppRedisCacheSpace.openCountKey(app_uid);\n const key_user_count = AppRedisCacheSpace.userCountKey(app_uid);\n void Promise.all([\n setRedisCacheValue(key_open_count, results.open_count),\n setRedisCacheValue(key_user_count, results.user_count),\n ]);\n }\n\n return results;\n } else {\n // Regular MySQL queries for non-grouped stats\n const baseOpenQuery = 'SELECT COUNT(_id) AS open_count FROM app_opens WHERE app_uid = ?';\n const baseUserQuery = 'SELECT COUNT(DISTINCT user_id) AS user_count FROM app_opens WHERE app_uid = ?';\n\n const generateQuery = (baseQuery, timeRange) => {\n if ( ! timeRange ) return baseQuery;\n return `${baseQuery} AND ts >= ? AND ts < ?`;\n };\n\n const openQuery = generateQuery(baseOpenQuery, timeRange);\n const userQuery = generateQuery(baseUserQuery, timeRange);\n const queryParams = timeRange ? [app_uid, timeRange.start, timeRange.end] : [app_uid];\n\n const [openResult, userResult] = await Promise.all([\n db.read(openQuery, queryParams),\n db.read(userQuery, queryParams),\n ]);\n\n const results = {\n open_count: parseInt(openResult[0].open_count),\n user_count: parseInt(userResult[0].user_count),\n referral_count: period === 'all'\n ? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))\n : null,\n };\n\n // Cache the results if period is 'all'\n if ( period === 'all' ) {\n const key_open_count = AppRedisCacheSpace.openCountKey(app_uid);\n const key_user_count = AppRedisCacheSpace.userCountKey(app_uid);\n void Promise.all([\n setRedisCacheValue(key_open_count, results.open_count),\n setRedisCacheValue(key_user_count, results.user_count),\n ]);\n }\n\n return results;\n }\n }\n\n /**\n * Refreshes the cache of app statistics including open and user counts.\n *\n * @notes\n * - This method logs a tick event for performance monitoring.\n *\n * @async\n * @returns {Promise} A promise that resolves when the cache refresh operation is complete.\n */\n async _refresh_app_stats () {\n this.log.tick('refresh app stats');\n\n const db = this.services.get('database').get(DB_READ, 'apps');\n\n let openCountMap;\n let userCountMap;\n\n if ( global.clickhouseClient ) {\n const [openResult, userResult] = await Promise.all([\n global.clickhouseClient.query({\n query: `\n SELECT app_uid, COUNT(_id) AS open_count\n FROM app_opens\n GROUP BY app_uid\n `,\n format: 'JSONEachRow',\n }),\n global.clickhouseClient.query({\n query: `\n SELECT app_uid, COUNT(DISTINCT user_id) AS user_count\n FROM app_opens\n GROUP BY app_uid\n `,\n format: 'JSONEachRow',\n }),\n ]);\n const openRows = await openResult.json();\n const userRows = await userResult.json();\n openCountMap = new Map(openRows.map(row => [row.app_uid, parseInt(row.open_count, 10)]));\n userCountMap = new Map(userRows.map(row => [row.app_uid, parseInt(row.user_count, 10)]));\n } else {\n const [openCounts, userCounts] = await Promise.all([\n db.read(`\n SELECT app_uid, COUNT(_id) AS open_count \n FROM app_opens \n GROUP BY app_uid\n `),\n db.read(`\n SELECT app_uid, COUNT(DISTINCT user_id) AS user_count \n FROM app_opens \n GROUP BY app_uid\n `),\n ]);\n openCountMap = new Map(openCounts.map(row => [row.app_uid, row.open_count]));\n userCountMap = new Map(userCounts.map(row => [row.app_uid, row.user_count]));\n }\n\n // Get all app UIDs and update the cache (apps list lives in MySQL)\n const apps = await db.read('SELECT uid FROM apps');\n\n for ( const app of apps ) {\n const key_open_count = AppRedisCacheSpace.openCountKey(app.uid);\n const key_user_count = AppRedisCacheSpace.userCountKey(app.uid);\n\n // Background refresh writes should stay local to avoid broadcast churn.\n void Promise.all([\n setRedisCacheValue(key_open_count, openCountMap.get(app.uid) ?? 0, { emitEvent: false }),\n setRedisCacheValue(key_user_count, userCountMap.get(app.uid) ?? 0, { emitEvent: false }),\n ]);\n }\n }\n\n /**\n * Refreshes the cache of app referral statistics.\n *\n * This method queries the database for user counts referred by each app's origin URL\n * and updates the cache with the referral counts for each app.\n *\n * @notes\n * - This method logs a tick event for performance monitoring.\n *\n * @async\n * @returns {Promise} A promise that resolves when the cache refresh operation is complete.\n */\n async _refresh_app_stat_referrals () {\n this.log.tick('refresh app stat referrals');\n\n const db = this.services.get('database').get(DB_READ, 'apps');\n\n const apps = await db.read('SELECT uid, index_url FROM apps');\n\n // First, build a map of valid app origins to UIDs\n const validApps = [];\n const svc_auth = this.services.get('auth');\n\n for ( const app of apps ) {\n const origin = origin_from_url(app.index_url);\n\n // only count the referral if the origin hashes to the app's uid\n let expected_uid;\n try {\n expected_uid = await svc_auth.app_uid_from_origin(origin);\n } catch (e) {\n // This happens if the app origin isn't valid\n continue;\n }\n if ( expected_uid !== app.uid ) {\n continue;\n }\n\n validApps.push({ uid: app.uid, origin });\n }\n\n if ( validApps.length === 0 ) {\n return;\n }\n\n // Build a single query to get all referral counts\n const likeConditions = validApps.map(() => 'referrer LIKE ?').join(' OR ');\n const queryParams = validApps.map(app => `${app.origin}%`);\n\n const referralResults = await db.read(`\n SELECT \n referrer,\n COUNT(id) as referral_count \n FROM user \n WHERE ${likeConditions}\n GROUP BY referrer\n `, queryParams);\n\n // Create a map to store referral counts by origin\n const referralMap = new Map();\n\n for ( const result of referralResults ) {\n // Find which app this referrer belongs to\n for ( const app of validApps ) {\n if ( result.referrer.startsWith(app.origin) ) {\n const currentCount = referralMap.get(app.uid) || 0;\n referralMap.set(app.uid, currentCount + parseInt(result.referral_count));\n break;\n }\n }\n }\n\n // Update cache with results\n for ( const app of validApps ) {\n const key_referral_count = AppRedisCacheSpace.referralCountKey(app.uid);\n const count = referralMap.get(app.uid) || 0;\n // Background refresh writes should stay local to avoid broadcast churn.\n await setRedisCacheValue(key_referral_count, count, { emitEvent: false });\n }\n\n this.log.info('DONE refresh app stat referrals');\n }\n\n /**\n * Deletes an application from the system.\n *\n * This method performs the following actions:\n * - Retrieves the app data from cache or database if not provided.\n * - Deletes the app record from the database.\n * - Removes the app from all relevant caches (by name, id, and uid).\n * - Removes the app from any associated tags.\n *\n * @param {string} app_uid - The unique identifier of the app to be deleted.\n * @param {Object} [app] - The app object, if already fetched. If not provided, it will be retrieved.\n * @param {Object} [options] - Optional delete behavior flags.\n * @throws {Error} If the app is not found in either cache or database.\n * @returns {Promise} A promise that resolves when the app has been successfully deleted.\n */\n async delete_app (app_uid, app, options = {}) {\n const db = this.services.get('database').get(DB_READ, 'apps');\n\n if ( ! app ) {\n app = await AppRedisCacheSpace.getCachedApp({\n lookup: 'uid',\n value: app_uid,\n });\n }\n if ( ! app ) {\n app = (await db.read(\n 'SELECT * FROM apps WHERE uid = ?',\n [app_uid],\n ))[0];\n }\n\n if ( ! app ) {\n throw new Error('app not found');\n }\n\n const associationRows = await db.read(\n 'SELECT type FROM app_filetype_association WHERE app_id = ?',\n [app.id],\n );\n\n await db.write(\n 'DELETE FROM apps WHERE uid = ? LIMIT 1',\n [app_uid],\n );\n\n if ( ! options.preserveCanonicalUidAlias ) {\n await this.cleanupCanonicalAppUidAliases_(app_uid);\n }\n\n // remove from caches\n AppRedisCacheSpace.invalidateCachedApp(app, {\n includeStats: true,\n });\n const associationKeys = associationRows\n .map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\\./, ''))\n .filter(Boolean)\n .map(ext => AppRedisCacheSpace.associationAppsKey(ext));\n if ( associationKeys.length ) {\n await deleteRedisKeys(associationKeys);\n }\n\n // remove from tags\n const app_tags = (app.tags ?? '').split(',')\n .map(tag => tag.trim())\n .filter(tag => tag.length > 0);\n for ( const tag of app_tags ) {\n if ( ! this.tags[tag] ) continue;\n const index = this.tags[tag].indexOf(app_uid);\n if ( index >= 0 ) {\n this.tags[tag].splice(index, 1);\n }\n }\n\n const svc_event = this.services.get('event');\n await svc_event.emit('app.changed', {\n app_uid: app.uid,\n action: 'deleted',\n app,\n });\n }\n\n buildCanonicalAppUidAliasKey_ (appUid) {\n return `${APP_UID_ALIAS_KEY_PREFIX}:${appUid}`;\n }\n\n buildCanonicalAppUidAliasReverseKey_ (canonicalAppUid) {\n return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`;\n }\n\n normalizeCanonicalAliasUidList_ (value) {\n if ( ! Array.isArray(value) ) return [];\n const normalizedList = [];\n const seen = new Set();\n for ( const item of value ) {\n if ( typeof item !== 'string' || !item ) continue;\n if ( seen.has(item) ) continue;\n seen.add(item);\n normalizedList.push(item);\n }\n return normalizedList;\n }\n\n async cleanupCanonicalAppUidAliases_ (appUid) {\n if ( typeof appUid !== 'string' || !appUid ) return;\n\n const kvStore = this.services.get('puter-kvstore');\n const suService = this.services.get('su');\n if ( !kvStore || typeof kvStore.get !== 'function' || typeof kvStore.del !== 'function' ) return;\n if ( !suService || typeof suService.sudo !== 'function' ) return;\n\n const selfAliasKey = this.buildCanonicalAppUidAliasKey_(appUid);\n const reverseKey = this.buildCanonicalAppUidAliasReverseKey_(appUid);\n\n try {\n await suService.sudo(async () => {\n const reverseValue = await kvStore.get({ key: reverseKey });\n const reverseAliases = this.normalizeCanonicalAliasUidList_(reverseValue);\n\n const deleteOps = [\n kvStore.del({ key: selfAliasKey }),\n kvStore.del({ key: reverseKey }),\n ];\n for ( const oldUid of reverseAliases ) {\n deleteOps.push(kvStore.del({\n key: this.buildCanonicalAppUidAliasKey_(oldUid),\n }));\n }\n await Promise.all(deleteOps);\n });\n } catch {\n // KV cleanup is best-effort.\n }\n }\n\n // Helper function to generate array of all periods between start and end dates\n generateAllPeriods (startDate, endDate, grouping) {\n const periods = [];\n let currentDate = new Date(startDate);\n\n // ???: In local debugging, `currentDate` evaluates to `Invalid Date`.\n // Does this work in prod?\n\n while ( currentDate <= endDate ) {\n let period;\n switch ( grouping ) {\n case 'hour':\n period = `${currentDate.toISOString().slice(0, 13)}:00:00`;\n currentDate.setHours(currentDate.getHours() + 1);\n break;\n case 'day':\n period = currentDate.toISOString().slice(0, 10);\n currentDate.setDate(currentDate.getDate() + 1);\n break;\n case 'week': {\n // Get the ISO week number\n const weekNum = String(this.getWeekNumber(currentDate)).padStart(2, '0');\n period = `${currentDate.getFullYear()}-${weekNum}`;\n currentDate.setDate(currentDate.getDate() + 7);\n break;\n }\n case 'month':\n period = currentDate.toISOString().slice(0, 7);\n currentDate.setMonth(currentDate.getMonth() + 1);\n break;\n case 'year':\n period = currentDate.getFullYear().toString();\n currentDate.setFullYear(currentDate.getFullYear() + 1);\n break;\n }\n periods.push({ period, count: 0 });\n }\n return periods;\n }\n\n // Helper function to get ISO week number\n getWeekNumber (date) {\n const target = new Date(date.valueOf());\n const dayNumber = (date.getDay() + 6) % 7;\n target.setDate(target.getDate() - dayNumber + 3);\n const firstThursday = target.valueOf();\n target.setMonth(0, 1);\n if ( target.getDay() !== 4 ) {\n target.setMonth(0, 1 + ((4 - target.getDay()) + 7) % 7);\n }\n return 1 + Math.ceil((firstThursday - target) / 604800000);\n }\n\n // Helper function to merge actual data with generated periods\n mergeWithGeneratedPeriods (actualData, allPeriods, stats_grouping) {\n // Create a map of period to count from actual data\n // First normalize the period format from both MySQL and ClickHouse\n const dataMap = new Map(actualData.map(item => {\n let period = item.period;\n // For ClickHouse results, convert the timestamp to match the expected format\n if ( item.period instanceof Date ) {\n switch ( stats_grouping ) {\n case 'hour':\n period = `${item.period.toISOString().slice(0, 13)}:00:00`;\n break;\n case 'day':\n period = item.period.toISOString().slice(0, 10);\n break;\n case 'week': {\n const weekNum = String(this.getWeekNumber(item.period)).padStart(2, '0');\n period = `${item.period.getFullYear()}-${weekNum}`;\n break;\n }\n case 'month':\n period = item.period.toISOString().slice(0, 7);\n break;\n case 'year':\n period = item.period.getFullYear().toString();\n break;\n }\n }\n return [period, parseInt(item.count)];\n }));\n\n // Map the generated periods to include actual counts where they exist\n return allPeriods.map(periodObj => {\n const count = dataMap.get(periodObj.period);\n return {\n period: periodObj.period,\n count: count !== undefined ? count : 0,\n };\n });\n }\n\n}\n\nmodule.exports = {\n AppInformationService,\n};\n"
},
{
"path": "src/backend/src/modules/apps/AppPermissionService.js",
"content": "const { UserActorType } = require('../../services/auth/Actor');\nconst { PermissionImplicator, PermissionUtil } = require('../../services/auth/permissionUtils.mjs');\nconst BaseService = require('../../services/BaseService');\n\nclass AppPermissionService extends BaseService {\n async _init () {\n const svc_permission = this.services.get('permission');\n svc_permission.register_implicator(PermissionImplicator.create({\n id: 'user-can-grant-read-own-apps',\n matcher: permission => {\n return permission.startsWith('apps-of-user:') ||\n permission.startsWith('subdomains-of-user:');\n },\n checker: async ({ actor, permission }) => {\n if ( ! (actor.type instanceof UserActorType) ) {\n return undefined;\n }\n\n const parts = PermissionUtil.split(permission);\n if ( parts[1] === actor.type.user.uuid ) {\n return {};\n }\n },\n }));\n }\n}\n\nmodule.exports = {\n AppPermissionService,\n};\n"
},
{
"path": "src/backend/src/modules/apps/AppRedisCacheSpace.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves when the initialization is complete.\n */\n async _init () {\n const svc_permission = this.services.get('permission');\n\n svc_permission.register_rewriter(PermissionRewriter.create({\n matcher: permission => {\n if ( ! permission.startsWith('app:') ) return false;\n const [_, specifier] = PermissionUtil.split(permission);\n if ( specifier.startsWith('uid#') ) return false;\n return true;\n },\n rewriter: async permission => {\n const [_1, name, ...rest] = PermissionUtil.split(permission);\n const app = await get_app({ name });\n return PermissionUtil.join(_1, `uid#${app.uid}`, ...rest);\n },\n }));\n\n // track: object description in comment\n // Owner of procted app has implicit permission to access it\n svc_permission.register_implicator(PermissionImplicator.create({\n matcher: permission => {\n return permission.startsWith('app:') || permission.startsWith('manage:app');\n },\n checker: async ({ actor, permission }) => {\n if ( ! (actor.type instanceof UserActorType) ) {\n return undefined;\n }\n\n const parts = PermissionUtil.split(permission);\n\n if ( parts[0] === 'manage' ) parts.shift();\n\n if ( parts.length < 2 ) return undefined;\n\n const [_, uid_part] = parts;\n\n // track: slice a prefix\n const uid = uid_part.slice('uid#'.length);\n\n const app = await get_app({ uid });\n\n if ( app.owner_user_id !== actor.type.user.id ) {\n return undefined;\n }\n\n return {};\n },\n }));\n }\n}\n\nmodule.exports = {\n ProtectedAppService,\n};\n"
},
{
"path": "src/backend/src/modules/apps/RecommendedAppsRedisCacheSpace.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n async _init () {\n const services = this.services;\n this.pager = services.get('pager');\n\n // TODO:[self-hosted] fix this properly\n this.known_errors = [];\n\n }\n\n /**\n * AlarmService registers its commands at the consolidation phase because\n * the '_init' method of CommandService may not have been called yet.\n */\n '__on_boot.consolidation' () {\n this._register_commands(this.services.get('commands'));\n }\n\n adapt_id_ (id) {\n let shorten = true;\n\n if ( shorten ) {\n const rng = seedrandom(id);\n id = this.identutil.generate_identifier('-', rng);\n }\n\n return id;\n }\n\n /**\n * Method to create an alarm with the given ID, message, and fields.\n * If the ID already exists, it will be updated with the new fields\n * and the occurrence count will be incremented.\n *\n * @param {string} id - Unique identifier for the alarm.\n * @param {string} message - Message associated with the alarm.\n * @param {object} fields - Additional information about the alarm.\n */\n create (id, message, fields) {\n if ( this.config.log_upcoming_alarms ) {\n this.log.error(`upcoming alarm: ${id}: ${message}`);\n }\n let existing = false;\n /**\n * Method to create an alarm with the given ID, message, and fields.\n * If the ID already exists, it will be updated with the new fields.\n * @param {string} id - Unique identifier for the alarm.\n * @param {string} message - Message associated with the alarm.\n * @param {object} fields - Additional information about the alarm.\n * @returns {void}\n */\n const alarm = (() => {\n const short_id = this.adapt_id_(id);\n\n if ( this.alarms[id] ) {\n existing = true;\n return this.alarms[id];\n }\n\n const alarm = this.alarms[id] = this.alarm_aliases[short_id] = {\n id,\n short_id,\n started: Date.now(),\n occurrences: [],\n };\n\n Object.defineProperty(alarm, 'count', {\n /**\n * Method to create a new alarm.\n *\n * This method takes an id, message, and optional fields as parameters.\n * It creates a new alarm object with the provided id and message,\n * and adds it to the alarms object. It also keeps track of the number of occurrences of the alarm.\n * If the alarm already exists, it increments the occurrence count and calls the handle\\_alarm\\_repeat\\_ method.\n * If it's a new alarm, it calls the handle\\_alarm\\_on\\_ method.\n *\n * @param {string} id - The unique identifier for the alarm.\n * @param {string} message - The message associated with the alarm.\n * @param {object} [fields] - Optional fields associated with the alarm.\n * @returns {void}\n */\n get () {\n return alarm.timestamps?.length ?? 0;\n },\n });\n\n Object.defineProperty(alarm, 'id_string', {\n /**\n * Method to handle creating a new alarm with given parameters.\n * This method adds the alarm to the `alarms` object, updates the occurrences count,\n * and processes any known errors that may apply to the alarm.\n * @param {string} id - The unique identifier for the alarm.\n * @param {string} message - The message associated with the alarm.\n * @param {Object} fields - Additional fields to associate with the alarm.\n */\n get () {\n if ( alarm.id.length < 20 ) {\n return alarm.id;\n }\n\n const truncatedLongId = `${alarm.id.slice(0, 20) }...`;\n\n return `${alarm.short_id} (${truncatedLongId})`;\n },\n });\n\n return alarm;\n })();\n\n const occurance = {\n message,\n fields,\n timestamp: Date.now(),\n };\n\n // Keep logs from the previous occurrence if:\n // - it's one of the first 3 occurrences\n // - the 10th, 100th, 1000th...etc occurrence\n if ( alarm.count > 3 && Math.log10(alarm.count) % 1 !== 0 ) {\n delete alarm.occurrences[alarm.occurrences.length - 1].logs;\n }\n occurance.logs = this.log.get_log_buffer();\n\n alarm.message = message;\n alarm.fields = { ...alarm.fields, ...fields };\n alarm.timestamps = (alarm.timestamps ?? []).concat(Date.now());\n alarm.occurrences.push(occurance);\n\n if ( fields?.error ) {\n alarm.error = fields.error;\n }\n\n if ( alarm.source ) {\n console.error(alarm.error);\n }\n\n if ( existing ) {\n this.handle_alarm_repeat_(alarm);\n } else {\n this.handle_alarm_on_(alarm);\n }\n }\n\n /**\n * Method to clear an alarm with the given ID.\n * @param {*} id - The ID of the alarm to clear.\n * @returns {void}\n */\n clear (id) {\n const alarm = this.alarms[id];\n if ( ! alarm ) {\n return;\n }\n delete this.alarms[id];\n this.handle_alarm_off_(alarm);\n }\n\n apply_known_errors_ (alarm) {\n const rule_matches = rule => {\n const match = rule.match;\n if ( match.id !== alarm.id ) return false;\n if ( match.message && match.message !== alarm.message ) return false;\n if ( match.fields ) {\n for ( const [key, value] of Object.entries(match.fields) ) {\n if ( alarm.fields[key] !== value ) return false;\n }\n }\n return true;\n };\n\n const rule_actions = {\n 'no-alert': () => alarm.no_alert = true,\n 'severity': action => alarm.severity = action.value,\n };\n\n const apply_action = action => {\n rule_actions[action.type](action);\n };\n\n for ( const rule of this.known_errors ) {\n if ( rule_matches(rule) ) apply_action(rule.action);\n }\n }\n\n handle_alarm_repeat_ (alarm) {\n this.log.warn(`REPEAT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,\n alarm.fields);\n\n this.apply_known_errors_(alarm);\n\n if ( alarm.no_alert ) return;\n\n const severity = alarm.severity ?? 'critical';\n\n const fields_clean = {};\n for ( const [key, value] of Object.entries(alarm.fields) ) {\n fields_clean[key] = util.inspect(value);\n }\n\n this.pager.alert({\n id: alarm.id ?? 'something-bad',\n message: alarm.message ?? alarm.id ?? 'something bad happened',\n source: 'alarm-service',\n severity,\n custom: {\n fields: fields_clean,\n trace: alarm.error?.stack,\n repeat_count: alarm.count,\n },\n });\n }\n\n handle_alarm_on_ (alarm) {\n this.log.error(`ACTIVE ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,\n alarm.fields);\n\n this.apply_known_errors_(alarm);\n\n if ( this.global_config.env === 'dev' && !this.attached_dev ) {\n this.attached_dev = true;\n const realConsole = globalThis.original_console_object ?? console;\n realConsole.error('\\x1B[33;1m[alarm]\\x1B[0m Active alarms detected; see logs for details.');\n }\n\n const args = this.Context.get('args') ?? {};\n if ( args['quit-on-alarm'] ) {\n const svc_shutdown = this.services.get('shutdown');\n svc_shutdown.shutdown({\n reason: '--quit-on-alarm is set',\n code: 1,\n });\n }\n\n if ( alarm.no_alert ) return;\n\n const severity = alarm.severity ?? 'critical';\n\n const fields_clean = {};\n for ( const [key, value] of Object.entries(alarm.fields) ) {\n fields_clean[key] = util.inspect(value);\n }\n\n this.pager.alert({\n id: alarm.id ?? 'something-bad',\n message: alarm.message ?? alarm.id ?? 'something bad happened',\n source: 'alarm-service',\n severity,\n custom: {\n fields: fields_clean,\n trace: alarm.error?.stack,\n },\n });\n\n // Write a .log file for the alert that happened\n try {\n const lines = [];\n lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`);\n lines.push(`started: ${new Date(alarm.started).toISOString()}`);\n lines.push(`short id: ${alarm.short_id}`);\n lines.push(`original id: ${alarm.id}`);\n lines.push(`severity: ${severity}`);\n lines.push(`message: ${alarm.message}`);\n lines.push(`fields: ${JSON.stringify(fields_clean)}`);\n\n const alert_info = lines.join('\\n');\n\n (async () => {\n try {\n fs.appendFileSync(`alert_${alarm.id}.log`, `${alert_info }\\n`);\n } catch (e) {\n this.log.error(`failed to write alert log: ${e.message}`);\n }\n })();\n } catch (e) {\n this.log.error(`failed to write alert log: ${e.message}`);\n }\n }\n\n handle_alarm_off_ (alarm) {\n this.log.info(`CLEAR ${alarm.id} :: ${alarm.message} (${alarm.count})`,\n alarm.fields);\n }\n\n /**\n * Method to get an alarm by its ID.\n *\n * @param {*} id - The ID of the alarm to get.\n * @returns\n */\n get_alarm (id) {\n return this.alarms[id] ?? this.alarm_aliases[id];\n }\n\n _register_commands (commands) {\n // Function to handle a specific alarm event.\n // This comment can be added above line 320.\n // This function is responsible for processing specific events related to alarms.\n // It can be used for tasks such as updating alarm status, sending notifications, or triggering actions.\n // This function is called internally by the AlarmService class.\n\n // /*\n // * handleAlarmEvent - Handles a specific alarm event.\n // *\n // * @param {Object} alarm - The alarm object containing relevant information.\n // * @param {Function} callback - Optional callback function to be called when the event is handled.\n // */\n // function handleAlarmEvent(alarm, callback) {\n // // Implementation goes here.\n // }\n const completeAlarmID = (args) => {\n // The alarm ID is the first argument, so return no results if we're on the second or later.\n if ( args.length > 1 )\n {\n return;\n }\n const lastArg = args[args.length - 1];\n\n const results = [];\n for ( const alarm of Object.values(this.alarms) ) {\n if ( alarm.id.startsWith(lastArg) ) {\n results.push(alarm.id);\n }\n if ( alarm.short_id?.startsWith(lastArg) ) {\n results.push(alarm.short_id);\n }\n }\n return results;\n };\n\n commands.registerCommands('alarm', [\n {\n id: 'list',\n description: 'list alarms',\n handler: async (args, log) => {\n for ( const alarm of Object.values(this.alarms) ) {\n log.log(`${alarm.id_string}: ${alarm.message} (${alarm.count})`);\n }\n },\n },\n {\n id: 'info',\n description: 'show info about an alarm',\n handler: async (args, log) => {\n const [id] = args;\n const alarm = this.get_alarm(id);\n if ( ! alarm ) {\n log.log(`no alarm with id ${id}`);\n return;\n }\n log.log(`\\x1B[33;1m${alarm.id_string}\\x1B[0m :: ${alarm.message} (${alarm.count})`);\n log.log(`started: ${new Date(alarm.started).toISOString()}`);\n log.log(`short id: ${alarm.short_id}`);\n log.log(`original id: ${alarm.id}`);\n\n // print stack trace of alarm error\n if ( alarm.error ) {\n log.log(alarm.error.stack);\n }\n // print other fields\n for ( const [key, value] of Object.entries(alarm.fields) ) {\n log.log(`- ${key}: ${util.inspect(value)}`);\n }\n },\n completer: completeAlarmID,\n },\n {\n id: 'clear',\n description: 'clear an alarm',\n handler: async (args, log) => {\n const [id] = args;\n const alarm = this.get_alarm(id);\n if ( ! alarm ) {\n log.log(`no alarm with id ${id}; ` +\n `but calling clear(${JSON.stringify(id)}) anyway.`);\n }\n this.clear(id);\n },\n completer: completeAlarmID,\n },\n {\n id: 'clear-all',\n description: 'clear all alarms',\n handler: async (_args, _log) => {\n const alarms = Object.values(this.alarms);\n this.alarms = {};\n for ( const alarm of alarms ) {\n this.handle_alarm_off_(alarm);\n }\n },\n },\n {\n id: 'sound',\n description: 'sound an alarm',\n handler: async (args, _log) => {\n const [id, message] = args;\n this.create(id ?? 'test', message, {});\n },\n },\n {\n id: 'inspect',\n description: 'show logs that happened an alarm',\n handler: async (args, log) => {\n const [id, occurance_idx] = args;\n const alarm = this.get_alarm(id);\n if ( ! alarm ) {\n log.log(`no alarm with id ${id}`);\n return;\n }\n const occurance = alarm.occurrences[occurance_idx];\n if ( ! occurance ) {\n log.log(`no occurance with index ${occurance_idx}`);\n return;\n }\n log.log(`┏━━ Logs before: ${alarm.id_string} ━━━━`);\n for ( const lg of occurance.logs ) {\n log.log(`┃ ${ this.logutil.stringify_log_entry(lg)}`);\n }\n log.log(`┗━━ Logs before: ${alarm.id_string} ━━━━`);\n },\n completer: completeAlarmID,\n },\n ]);\n }\n}\n\nmodule.exports = {\n AlarmService,\n};\n"
},
{
"path": "src/backend/src/modules/core/ContextService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves when the initialization is complete.\n */\n async init () {\n const services = this.services;\n this.alarm = services.get('alarm');\n this.backupLogger = services.get('log-service').create('error-service');\n }\n\n /**\n * Creates an ErrorContext instance with the provided logging context.\n *\n * @param {*} log_context The logging context to associate with the error reports.\n * @returns {ErrorContext} An ErrorContext instance.\n */\n create (log_context) {\n return new ErrorContext(this, log_context);\n }\n\n /**\n * Reports an error with the specified location and details.\n * The \"location\" is a string up to the callers discretion to identify\n * the source of the error.\n *\n * @param {*} location The location where the error occurred.\n * @param {*} fields The error details to report.\n * @param {boolean} [alarm=true] Whether to raise an alarm for the error.\n * @returns {void}\n */\n report (location, { source, logger, trace, extra, message }, alarm = true) {\n message = message ?? source?.message;\n logger = logger ?? this.backupLogger;\n logger.error(`Error @ ${location}: ${message}; ${ source?.stack}`);\n\n if ( alarm ) {\n const alarm_id = `${location}:${message}`;\n this.alarm.create(alarm_id, message, {\n error: source,\n ...extra,\n });\n }\n }\n}\n\nmodule.exports = { ErrorService };\n"
},
{
"path": "src/backend/src/modules/core/ExpectationService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves when initialization is complete.\n */\n async _init () {\n // TODO: service to track all interval functions?\n /**\n * Initializes the service by setting up interval functions and registering commands.\n * This method sets up a periodic interval function to purge expectations and registers\n * a command to list pending expectations.\n *\n * @returns {void}\n */\n\n // The comment should be placed above the method at line 68\n setInterval(() => {\n this.purgeExpectations_();\n }, 1000);\n }\n\n /**\n * Purges expectations that have been met.\n *\n * This method iterates through the list of expectations and removes\n * those that have been satisfied. Currently, this functionality is\n * disabled and needs to be re-enabled.\n *\n * @returns {void} This method does not return anything.\n */\n purgeExpectations_ () {\n return;\n // TODO: Re-enable this\n // for ( let i=0 ; i < this.expectations_.length ; i++ ) {\n // if ( this.expectations_[i].check() ) {\n // this.expectations_[i] = null;\n // }\n // }\n // this.expectations_ = this.expectations_.filter(v => v !== null);\n }\n\n /**\n * Registers an expectation to be tracked by the service.\n *\n * @param {Object} workUnit - The work unit to track\n * @param {string} checkpoint - The checkpoint to expect\n * @returns {void}\n */\n expect_eventually ({ workUnit, checkpoint }) {\n this.expectations_.push(new this.expect.CheckpointExpectation(workUnit, checkpoint));\n }\n}\n\nmodule.exports = {\n ExpectationService,\n};"
},
{
"path": "src/backend/src/modules/core/LogService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n async _init () {\n this.alertHandlers_ = [];\n\n if ( ! this.config ) {\n return;\n }\n\n this.onInit();\n }\n\n /**\n * Initializes PagerDuty configuration and registers alert handlers.\n * If PagerDuty is enabled in the configuration, it sets up an alert handler\n * to send alerts to PagerDuty.\n *\n * @method onInit\n */\n onInit () {\n if ( this.config.pagerduty && this.config.pagerduty.enabled ) {\n this.alertHandlers_.push(async alert => {\n const event = pdjs.event;\n\n const fields_clean = {};\n for ( const [key, value] of Object.entries(alert?.fields ?? {}) ) {\n fields_clean[key] = util.inspect(value);\n }\n\n const custom_details = {\n ...(alert.custom || {}),\n server_id: this.global_config.server_id,\n };\n\n const ctx = this.Context.get(undefined, { allow_fallback: true });\n\n // Add request payload if any exists\n const req = ctx.get('req');\n if ( req ) {\n if ( req.body ) {\n // Remove fields which may contain sensitive information\n delete req.body.password;\n delete req.body.email;\n\n // Add the request body to the custom details\n custom_details.request_body = req.body;\n }\n }\n\n this.log.info('it is sending to PD');\n await event({\n data: {\n routing_key: this.config.pagerduty.routing_key,\n event_action: 'trigger',\n dedup_key: alert.id,\n payload: {\n summary: alert.message,\n source: alert.source,\n severity: alert.severity,\n custom_details,\n },\n },\n });\n });\n }\n }\n\n /**\n * Sends an alert to all registered alert handlers.\n *\n * This method iterates through all alert handlers and attempts to send the alert.\n * If any handler fails to send the alert, an error message is logged.\n *\n * @param {Object} alert - The alert object containing details about the alert.\n */\n async alert (alert) {\n for ( const handler of this.alertHandlers_ ) {\n try {\n await handler(alert);\n } catch (e) {\n this.log.error(`failed to send pager alert: ${e?.message}`);\n }\n }\n }\n\n _register_commands (commands) {\n commands.registerCommands('pager', [\n {\n id: 'test-alert',\n description: 'create a test alert',\n handler: async (args, log) => {\n const [severity] = args;\n await this.alert({\n id: 'test-alert',\n message: 'test alert',\n source: 'test',\n severity,\n });\n },\n },\n ]);\n }\n\n}\n\nmodule.exports = {\n PagerService,\n};\n"
},
{
"path": "src/backend/src/modules/core/ParameterService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } */\n this.parameters_ = [];\n }\n\n /**\n * Initializes the service by registering commands with the command service.\n * This method is called during service startup to set up command handlers\n * for parameter management.\n * @private\n */\n '__on_boot.consolidation' () {\n this._registerCommands(this.services.get('commands'));\n }\n\n createParameters (serviceName, parameters, opt_instance) {\n for ( const parameter of parameters ) {\n this.log.debug(`registering parameter ${serviceName}:${parameter.id}`);\n this.parameters_.push(new Parameter({\n ...parameter,\n id: `${serviceName}:${parameter.id}`,\n }));\n if ( opt_instance ) {\n this.bindToInstance(`${serviceName}:${parameter.id}`,\n opt_instance,\n parameter.id);\n }\n }\n }\n\n /**\n * Gets the value of a parameter by its ID\n * @param {string} id - The unique identifier of the parameter to retrieve\n * @returns {Promise<*>} The current value of the parameter\n * @throws {Error} If parameter with given ID is not found\n */\n async get (id) {\n const parameter = this._get_param(id);\n return await parameter.get();\n }\n\n bindToInstance (id, instance, name) {\n const parameter = this._get_param(id);\n return parameter.bindToInstance(instance, name);\n }\n\n subscribe (id, listener) {\n const parameter = this._get_param(id);\n return parameter.subscribe(listener);\n }\n\n _get_param (id) {\n const parameter = this.parameters_.find(p => p.spec_.id === id);\n if ( ! parameter ) {\n throw new Error(`unknown parameter: ${id}`);\n }\n return parameter;\n }\n\n /**\n * Registers parameter-related commands with the command service\n * @param {Object} commands - The command service instance to register with\n */\n _registerCommands (commands) {\n const completeParameterName = (args) => {\n // The parameter name is the first argument, so return no results if we're on the second or later.\n if ( args.length > 1 )\n {\n return;\n }\n const lastArg = args[args.length - 1];\n\n return this.parameters_\n .map(parameter => parameter.spec_.id)\n .filter(parameterName => parameterName.startsWith(lastArg));\n };\n\n commands.registerCommands('params', [\n {\n id: 'get',\n description: 'get a parameter',\n handler: async (args, log) => {\n const [name] = args;\n const value = await this.get(name);\n log.log(value);\n },\n completer: completeParameterName,\n },\n {\n id: 'set',\n description: 'set a parameter',\n handler: async (args, log) => {\n const [name, value] = args;\n const parameter = this._get_param(name);\n parameter.set(value);\n log.log(value);\n },\n completer: completeParameterName,\n },\n {\n id: 'list',\n description: 'list parameters',\n handler: async (args, log) => {\n const [prefix] = args;\n let parameters = this.parameters_;\n if ( prefix ) {\n parameters = parameters\n .filter(p => p.spec_.id.startsWith(prefix));\n }\n log.log(`available parameters${\n prefix ? ` (starting with: ${prefix})` : ''\n }:`);\n for ( const parameter of parameters ) {\n // log.log(`- ${parameter.spec_.id}: ${parameter.spec_.description}`);\n // Log parameter description and value\n const value = await parameter.get();\n log.log(`- ${parameter.spec_.id} = ${value}`);\n log.log(` ${parameter.spec_.description}`);\n }\n },\n },\n ]);\n }\n}\n\nmodule.exports = {\n ParameterService,\n};\n"
},
{
"path": "src/backend/src/modules/core/ProcessEventService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n await this.Context.allow_fallback(async () => {\n errors.report('process:uncaughtException', {\n source: err,\n origin,\n trace: true,\n alarm: true,\n });\n });\n\n });\n\n process.on('unhandledRejection', async (reason, promise) => {\n /**\n * Handles unhandled promise rejections by reporting them to the error service\n * @param {*} reason - The rejection reason/error\n * @param {Promise} promise - The rejected promise\n * @returns {Promise} Resolves when error is reported\n */\n await this.Context.allow_fallback(async () => {\n errors.report('process:unhandledRejection', {\n source: reason,\n promise,\n trace: true,\n alarm: true,\n });\n });\n });\n }\n}\n\nmodule.exports = {\n ProcessEventService,\n};\n"
},
{
"path": "src/backend/src/modules/core/README.md",
"content": "# Core2Module\n\nA replacement for CoreModule with as few external relative requires as possible.\nThis will eventually be the successor to CoreModule, the main module for Puter's backend.\n\n## Services\n\n### AlarmService\n\nAlarmService class is responsible for managing alarms.\nIt provides methods for creating, clearing, and handling alarms.\n\n#### Listeners\n\n##### `boot.consolidation`\n\nAlarmService registers its commands at the consolidation phase because\nthe '_init' method of CommandService may not have been called yet.\n\n#### Methods\n\n##### `create`\n\nMethod to create an alarm with the given ID, message, and fields.\nIf the ID already exists, it will be updated with the new fields\nand the occurrence count will be incremented.\n\n###### Parameters\n\n- **id:** Unique identifier for the alarm.\n- **message:** Message associated with the alarm.\n- **fields:** Additional information about the alarm.\n\n##### `clear`\n\nMethod to clear an alarm with the given ID.\n\n###### Parameters\n\n- **id:** The ID of the alarm to clear.\n\n##### `get_alarm`\n\nMethod to get an alarm by its ID.\n\n###### Parameters\n\n- **id:** The ID of the alarm to get.\n\n### ErrorService\n\nThe ErrorService class is responsible for handling and reporting errors within the system.\nIt provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms.\n\n#### Methods\n\n##### `init`\n\nInitializes the ErrorService, setting up the alarm and backup logger services.\n\n##### `create`\n\nCreates an ErrorContext instance with the provided logging context.\n\n###### Parameters\n\n- **log_context:** The logging context to associate with the error reports.\n\n##### `report`\n\nReports an error with the specified location and details.\nThe \"location\" is a string up to the callers discretion to identify\nthe source of the error.\n\n###### Parameters\n\n- **location:** The location where the error occurred.\n- **fields:** The error details to report.\n\n### ExpectationService\n\n\n\n#### Listeners\n\n##### `boot.consolidation`\n\nExpectationService registers its commands at the consolidation phase because\nthe '_init' method of CommandService may not have been called yet.\n\n#### Methods\n\n##### `expect_eventually`\n\nRegisters an expectation to be tracked by the service.\n\n###### Parameters\n\n- **workUnit:** The work unit to track\n- **checkpoint:** The checkpoint to expect\n\n### LogService\n\nThe `LogService` class extends `BaseService` and is responsible for managing and \norchestrating various logging functionalities within the application. It handles \nlog initialization, middleware registration, log directory management, and \nprovides methods for creating log contexts and managing log output levels.\n\n#### Listeners\n\n##### `boot.consolidation`\n\nRegisters logging commands with the command service.\n\n#### Methods\n\n##### `register_log_middleware`\n\nRegisters a custom logging middleware with the LogService.\n\n###### Parameters\n\n- **callback:** The callback function that modifies log parameters before delegation.\n\n##### `create`\n\nCreate a new log context with the specified prefix\n\n###### Parameters\n\n- **prefix:** The prefix for the log context\n- **fields:** Optional fields to include in the log context\n\n##### `get_log_file`\n\nGenerates a sanitized file path for log files.\n\n###### Parameters\n\n- **name:** The name of the log file, which will be sanitized to remove any path characters.\n\n##### `get_log_buffer`\n\nGet the most recent log entries from the buffer maintained by the LogService.\nBy default, the buffer contains the last 20 log entries.\n\n### PagerService\n\n\n\n#### Listeners\n\n##### `boot.consolidation`\n\nPagerService registers its commands at the consolidation phase because\nthe '_init' method of CommandService may not have been called yet.\n\n#### Methods\n\n##### `onInit`\n\nInitializes PagerDuty configuration and registers alert handlers.\nIf PagerDuty is enabled in the configuration, it sets up an alert handler\nto send alerts to PagerDuty.\n\n##### `alert`\n\nSends an alert to all registered alert handlers.\n\nThis method iterates through all alert handlers and attempts to send the alert.\nIf any handler fails to send the alert, an error message is logged.\n\n###### Parameters\n\n- **alert:** The alert object containing details about the alert.\n\n### ProcessEventService\n\nService class that handles process-wide events and errors.\nProvides centralized error handling for uncaught exceptions and unhandled promise rejections.\nSets up event listeners on the process object to capture and report critical errors\nthrough the logging and error reporting services.\n\n## Libraries\n\n### core.expect\n\n### core.util.identutil\n\n#### Functions\n\n##### `randomItem`\n\nSelect a random item from an array using a random number generator function.\n\n###### Parameters\n\n- **arr:** The array to select an item from\n\n### core.util.logutil\n\n#### Functions\n\n##### `stringify_log_entry`\n\nStringifies a log entry into a formatted string for console output.\n\n###### Parameters\n\n- **logEntry:** The log entry object containing:\n\n### stdio\n\n#### Functions\n\n##### `visible_length`\n\nMETADATA // {\"ai-commented\":{\"service\":\"claude\"}}\n\n##### `split_lines`\n\nSplit a string into lines according to the terminal width,\npreserving ANSI escape sequences, and return an array of lines.\n\n###### Parameters\n\n- **str:** The string to split into lines\n\n### core.util.strutil\n\n#### Functions\n\n##### `quot`\n\nMETADATA // {\"def\":\"core.util.strutil\",\"ai-params\":{\"service\":\"claude\"},\"ai-commented\":{\"service\":\"claude\"}}\n\n## Notes\n\n### Outside Imports\n\nThis module has external relative imports. When these are\nremoved it may become possible to move this module to an\nextension.\n\n**Imports:**\n- `../../services/BaseService.js`\n- `../../util/context.js`\n- `../../services/BaseService` (use.BaseService)\n- `../../services/BaseService` (use.BaseService)\n- `../../util/context`\n- `../../services/BaseService` (use.BaseService)\n- `../../services/BaseService` (use.BaseService)\n- `../../services/BaseService` (use.BaseService)\n"
},
{
"path": "src/backend/src/modules/core/ServerHealthService/ServerHealthRedisCacheKeys.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }: An array of names of failed health checks, if any.\n */\n async get_status () {\n const cacheKey = ServerHealthRedisCacheKeys.status;\n\n // Check cache first\n const cached = await redisClient.get(cacheKey);\n if ( cached ) {\n try {\n return JSON.parse(cached);\n } catch (e) {\n // no op cache is in an invalid state\n }\n }\n\n // Compute status\n const failures = this.failures_.map(v => v.name);\n const status = {\n ok: failures.length === 0,\n ...(failures.length ? { failed: failures } : {}),\n };\n\n // Cache with 5 second TTL\n await setRedisCacheValue(cacheKey, JSON.stringify(status), {\n ttlSeconds: 5,\n eventData: status,\n });\n\n return status;\n }\n}\n\nmodule.exports = { ServerHealthService };\n"
},
{
"path": "src/backend/src/modules/core/lib/__lib__.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } arr - The array to select an item from\n * @param {function} [random=Math.random] - Random number generator function\n * @returns {T} A random item from the array\n */\nconst randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)];\n\n/**\n * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999).\n * The result is returned as a string with components separated by the specified separator.\n * It is useful when you need to create unique identifiers that are also human-friendly.\n *\n * @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided.\n * @param {function} [rng=Math.random] - Random number generator function\n * @returns {string} A unique, human-friendly identifier.\n *\n * @example\n *\n * let identifier = window.generate_identifier();\n * // identifier would be something like 'clever-idea-123'\n *\n */\nfunction generate_identifier (separator = '_', rng = Math.random) {\n // return a random combination of first_adj + noun + number (between 0 and 9999)\n // e.g. clever-idea-123\n return [\n randomItem(adjectives, rng),\n randomItem(nouns, rng),\n Math.floor(rng() * 10000),\n ].join(separator);\n}\n\n// Character set used for generating human-readable, case-insensitive random codes\nconst HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';\n\nfunction generate_random_code (n, {\n rng = Math.random,\n chars = HUMAN_READABLE_CASE_INSENSITIVE,\n} = {}) {\n let code = '';\n for ( let i = 0 ; i < n ; i++ ) {\n code += randomItem(chars, rng);\n }\n return code;\n}\n\n/**\n* Composes a code by combining a mask string with a base-36 converted number\n* @param {string} mask - Initial string template to use as base\n* @param {number} value - Number to convert to base-36 and append to the right\n* @returns {string} Combined uppercase code\n*/\nfunction compose_code (mask, value) {\n const right_str = value.toString(36);\n let out_str = mask;\n console.log('right_str', right_str);\n console.log('out_str', out_str);\n for ( let i = 0 ; i < right_str.length ; i++ ) {\n out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i];\n }\n\n out_str = out_str.toUpperCase();\n return out_str;\n}\n\nmodule.exports = {\n randomItem,\n generate_identifier,\n generate_random_code,\n};\n"
},
{
"path": "src/backend/src/modules/core/lib/linux.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see : to fs::\n svc_permission.register_rewriter(PermissionRewriter.create({\n matcher: permission => permission.startsWith('app-root-dir:'),\n rewriter: async permission => {\n const context = Context.get();\n\n // Only \"AppUnderUser\" scope is allowed to have this permission rewritten to\n // an actual filesystem permission - this is because apps will still be limited\n // baesd on a user's own access.\n const actor = context.get('actor');\n if ( ! Context.get('is_grant_user_app_permission') ) {\n return PERMISSION_FOR_NOTHING_IN_PARTICULAR;\n }\n\n const parts = PermissionUtil.split(permission);\n if ( parts.length < 3 ) {\n throw APIError.create('field_invalid', null, { key: 'permission', got: permission });\n }\n\n // <>::\n const target_app_uid = parts[1];\n const access = parts[2];\n if ( ! target_app_uid ) {\n throw APIError.create('field_invalid', null, { key: 'target_app_uid', got: target_app_uid });\n }\n\n if ( ! (actor.type instanceof UserActorType) ) {\n throw APIError.create('forbidden');\n }\n\n const target_app = await get_app({ uid: target_app_uid });\n if ( ! target_app ) {\n throw APIError.create('entity_not_found', null, { identifier: `app:${target_app_uid}` });\n }\n if ( target_app.owner_user_id !== actor.type.user.id ) {\n throw APIError.create('forbidden');\n }\n\n const root_dir_id = await svc_app.getAppRootDirId(target_app);\n const svc_fs = context.get('services').get('filesystem');\n const node = await svc_fs.node(new NodeInternalIDSelector('mysql', root_dir_id));\n await node.fetchEntry();\n if ( ! node.found ) throw APIError.create('subject_does_not_exist');\n\n const node_uid = await node.get('uid');\n return PermissionUtil.join('fs', node_uid, access);\n },\n }));\n\n }\n\n static PROTECTED_FIELDS = ['last_review'];\n static READ_ONLY_FIELDS = [\n 'approved_for_listing',\n 'approved_for_opening_items',\n 'approved_for_incentive_program',\n 'godmode',\n 'is_private',\n ];\n static WRITE_ALL_OWNER_PERMISSION = 'system:es:write-all-owners';\n\n static IMPLEMENTS = {\n 'crud-q': {\n async create ({ object, options }) {\n return await this.#create({ object, options });\n },\n async update ({ object, id, options }) {\n return await this.#update({ object, id, options });\n },\n async upsert ({ object, id, options }) {\n // Try to find an existing entity\n let existing = null;\n\n if ( object.uid !== undefined || id !== undefined ) {\n try {\n existing = await this.#read({\n uid: object.uid,\n id,\n });\n } catch ( error ) {\n // If entity not found, we'll create it\n if ( error.fields?.code !== 'entity_not_found' ) {\n throw error;\n }\n }\n }\n\n if ( existing ) {\n // Entity exists, call update\n return await this.#update({ object, id, options });\n } else {\n // Entity doesn't exist, call create\n return await this.#create({ object, options });\n }\n },\n async read ({ uid, id, params = {} }) {\n return this.#read({ uid, id, params });\n },\n async select (options) {\n return this.#select(options);\n },\n async delete ({ uid, id }) {\n return await this.#delete({ uid, id });\n },\n },\n };\n\n // value of require('om/mappings/app.js').redundant_identifiers\n static REDUNDANT_IDENTIFIERS = ['name'];\n\n async #select ({ predicate, params, ..._rest }) {\n const db = this.db;\n\n if ( predicate === undefined ) predicate = [];\n if ( params === undefined ) params = {};\n if ( ! Array.isArray(predicate) ) throw new Error('predicate must be an array');\n\n const userCanEditOnly = Array.prototype.includes.call(predicate, 'user-can-edit');\n\n const stmt = 'SELECT apps.*, ' +\n 'CASE WHEN apps.icon LIKE \\'data:%\\' THEN 1 ELSE 0 END AS icon_is_base64, ' +\n 'owner_user.username AS owner_user_username, ' +\n 'owner_user.uuid AS owner_user_uuid, ' +\n 'app_owner.uid AS app_owner_uid ' +\n 'FROM apps ' +\n 'LEFT JOIN user owner_user ON apps.owner_user_id = owner_user.id ' +\n 'LEFT JOIN apps app_owner ON apps.app_owner = app_owner.id ' +\n `${userCanEditOnly ? 'WHERE apps.owner_user_id=?' : ''} ` +\n 'LIMIT 5000';\n const values = userCanEditOnly ? [Context.get('user').id] : [];\n const rows = await db.read(stmt, values);\n\n const shouldFetchFiletypes = rows.some(row => typeof row.filetypes !== 'string');\n const filetypesByAppId = shouldFetchFiletypes\n ? await this.#getFiletypeAssociationsByAppIds(rows.map(row => row.id))\n : new Map();\n\n const iconSize = params.icon_size;\n const shouldResolveIconPath = Boolean(iconSize)\n || rows.some(row => isStoredBase64AppIcon(row));\n const svc_appIcon = shouldResolveIconPath\n ? this.context.get('services').get('app-icon')\n : null;\n const svc_error = shouldResolveIconPath\n ? this.context.get('services').get('error-service')\n : null;\n\n const appAndOwnerIds = [];\n for ( const row of rows ) {\n const app = {};\n\n // FROM ROW\n app.approved_for_incentive_program = as_bool(row.approved_for_incentive_program);\n app.approved_for_listing = as_bool(row.approved_for_listing);\n app.approved_for_opening_items = as_bool(row.approved_for_opening_items);\n app.background = as_bool(row.background);\n app.created_at = row.created_at;\n app.created_from_origin = row.created_from_origin;\n app.description = row.description;\n app.godmode = as_bool(row.godmode);\n app.icon = row.icon;\n app.is_private = as_bool(row.is_private);\n app.index_url = row.index_url;\n app.maximize_on_start = as_bool(row.maximize_on_start);\n app.metadata = row.metadata;\n app.name = row.name;\n app.protected = as_bool(row.protected);\n app.stats = row.stats;\n app.title = row.title;\n app.uid = row.uid;\n\n // REQURIES OTHER DATA\n // app.app_owner;\n // app.filetype_associations = row.filetype_associations;\n // app.owner = row.owner;\n\n app.app_owner = {\n uid: row.app_owner_uid,\n };\n\n {\n const owner_user = extract_from_prefix(row, 'owner_user_');\n app.owner = user_to_client(owner_user);\n }\n\n try {\n if ( typeof row.filetypes === 'string' ) {\n app.filetype_associations = this.#parseFiletypeAssociationsJson(row.filetypes);\n } else {\n app.filetype_associations = this.#normalizeFiletypeAssociations(filetypesByAppId.get(row.id) ?? []);\n }\n } catch (e) {\n throw new Error(`failed to get app filetype associations: ${e.message}`, { cause: e });\n }\n\n // REFINED BY OTHER DATA\n // app.icon;\n if ( svc_appIcon && (iconSize || isStoredBase64AppIcon(row)) ) {\n try {\n const iconPath = svc_appIcon.getAppIconPath({\n appUid: row.uid,\n size: iconSize,\n });\n if ( iconPath ) {\n app.icon = iconPath;\n }\n } catch (e) {\n svc_error?.report('AppES:read_transform', { source: e });\n }\n }\n\n appAndOwnerIds.push({\n app,\n ownerUserId: row.owner_user_id,\n });\n }\n\n // Check protected app access in parallel for faster large selections.\n const allowed_apps = await Promise.all(appAndOwnerIds.map(async ({ app, ownerUserId }) => {\n if ( await this.#check_protected_app_access(app, ownerUserId) ) {\n return null;\n }\n return app;\n }));\n\n return allowed_apps.filter(Boolean);\n }\n\n async #read ({ uid, id, params = {}, backend_only_options = {} }) {\n const db = this.db;\n\n if ( uid === undefined && id === undefined ) {\n throw new Error('read requires either uid or id');\n }\n\n // Build WHERE clause based on identifier type\n let whereClause;\n let whereValues;\n let canonicalUidAliasPromise = null;\n\n if ( uid !== undefined ) {\n // Simple uid lookup\n whereClause = 'apps.uid = ?';\n whereValues = [uid];\n canonicalUidAliasPromise = this.#readCanonicalAppUidAlias(uid);\n } else if ( id !== null && typeof id === 'object' && !Array.isArray(id) ) {\n // Complex id lookup (e.g., { name: 'editor' })\n const { clause, values } = this.#build_complex_id_where(id);\n whereClause = clause;\n whereValues = values;\n } else {\n throw APIError.create('invalid_id', null, { id });\n }\n\n const stmt = 'SELECT apps.*, ' +\n 'CASE WHEN apps.icon LIKE \\'data:%\\' THEN 1 ELSE 0 END AS icon_is_base64, ' +\n 'owner_user.username AS owner_user_username, ' +\n 'owner_user.uuid AS owner_user_uuid, ' +\n 'app_owner.uid AS app_owner_uid ' +\n 'FROM apps ' +\n 'LEFT JOIN user owner_user ON apps.owner_user_id = owner_user.id ' +\n 'LEFT JOIN apps app_owner ON apps.app_owner = app_owner.id ' +\n `WHERE ${whereClause} ` +\n 'LIMIT 1';\n\n let rows = await db.read(stmt, whereValues);\n\n if ( rows.length === 0 && canonicalUidAliasPromise ) {\n const canonicalUid = await canonicalUidAliasPromise;\n if (\n typeof canonicalUid === 'string'\n && canonicalUid\n && canonicalUid !== uid\n ) {\n rows = await db.read(stmt, [canonicalUid]);\n }\n }\n\n if ( rows.length === 0 ) {\n throw APIError.create('entity_not_found', null, {\n identifier: uid || JSON.stringify(id),\n });\n }\n\n const row = rows[0];\n const app = {};\n\n app.approved_for_incentive_program = as_bool(row.approved_for_incentive_program);\n app.approved_for_listing = as_bool(row.approved_for_listing);\n app.approved_for_opening_items = as_bool(row.approved_for_opening_items);\n app.background = as_bool(row.background);\n app.created_at = row.created_at;\n app.created_from_origin = row.created_from_origin;\n app.description = row.description;\n app.godmode = as_bool(row.godmode);\n app.icon = row.icon;\n app.is_private = as_bool(row.is_private);\n app.index_url = row.index_url;\n app.maximize_on_start = as_bool(row.maximize_on_start);\n app.metadata = row.metadata;\n app.name = row.name;\n app.protected = as_bool(row.protected);\n app.stats = row.stats;\n app.title = row.title;\n app.uid = row.uid;\n\n app.app_owner = {\n uid: row.app_owner_uid,\n };\n\n {\n const owner_user = extract_from_prefix(row, 'owner_user_');\n if ( backend_only_options.no_filter_owner ) app.owner = owner_user;\n else app.owner = user_to_client(owner_user);\n }\n\n let protectedAccessPromise;\n try {\n if ( typeof row.filetypes === 'string' ) {\n app.filetype_associations = this.#parseFiletypeAssociationsJson(row.filetypes);\n } else {\n protectedAccessPromise = this.#check_protected_app_access(app, row.owner_user_id);\n const filetypeAssociations = await this.#getFiletypeAssociationsByAppId(row.id);\n app.filetype_associations = this.#normalizeFiletypeAssociations(filetypeAssociations);\n }\n } catch (e) {\n throw new Error(`failed to get app filetype associations: ${e.message}`, { cause: e });\n }\n\n // Check protected app access as soon as dependent fields are resolved.\n if ( ! protectedAccessPromise ) {\n protectedAccessPromise = this.#check_protected_app_access(app, row.owner_user_id);\n }\n if ( await protectedAccessPromise ) {\n // App should not be accessible\n throw APIError.create('entity_not_found', null, {\n identifier: uid || JSON.stringify(id),\n });\n }\n\n const iconSize = params.icon_size;\n if ( iconSize || isStoredBase64AppIcon(row) ) {\n const svc_appIcon = this.context.get('services').get('app-icon');\n if ( svc_appIcon ) {\n try {\n const iconPath = svc_appIcon.getAppIconPath({\n appUid: row.uid,\n size: iconSize,\n });\n if ( iconPath ) {\n app.icon = iconPath;\n }\n } catch (e) {\n const svc_error = this.context.get('services').get('error-service');\n svc_error.report('AppES:read_transform', { source: e });\n }\n }\n }\n\n return app;\n }\n\n #parseFiletypeAssociationsJson (filetypes) {\n return this.#normalizeFiletypeAssociations(JSON.parse(filetypes));\n }\n\n async #getFiletypeAssociationsByAppId (appId) {\n if ( appId === undefined || appId === null ) return [];\n\n const rows = await this.db.read(\n 'SELECT type FROM app_filetype_association WHERE app_id = ?',\n [appId],\n );\n return rows\n .map(row => row.type)\n .filter(type => typeof type === 'string' || type === null);\n }\n\n #normalizeFiletypeAssociations (filetypesAsJSON) {\n filetypesAsJSON = Array.isArray(filetypesAsJSON)\n ? filetypesAsJSON\n : [];\n filetypesAsJSON = filetypesAsJSON.filter(ft => ft !== null);\n for ( let i = 0 ; i < filetypesAsJSON.length ; i++ ) {\n if ( typeof filetypesAsJSON[i] !== 'string' ) {\n throw new Error(`expected filetypesAsJSON[${i}] to be a string, got: ${filetypesAsJSON[i]}`);\n }\n if ( String.prototype.startsWith.call(filetypesAsJSON[i], '.') ) {\n filetypesAsJSON[i] = filetypesAsJSON[i].slice(1);\n }\n }\n return filetypesAsJSON;\n }\n\n async #getFiletypeAssociationsByAppIds (appIds) {\n appIds = [...new Set(appIds.filter(appId => appId !== undefined && appId !== null))];\n if ( appIds.length === 0 ) return new Map();\n\n const filetypesByAppId = new Map();\n for ( const appId of appIds ) {\n filetypesByAppId.set(appId, []);\n }\n\n // SQLite has a low bind-parameter limit; chunk to avoid oversized IN lists.\n const chunkSize = 500;\n for ( let i = 0 ; i < appIds.length ; i += chunkSize ) {\n const chunk = appIds.slice(i, i + chunkSize);\n const placeholders = chunk.map(() => '?').join(', ');\n const rows = await this.db.read(\n `SELECT app_id, type FROM app_filetype_association WHERE app_id IN (${placeholders})`,\n chunk,\n );\n for ( const row of rows ) {\n if ( ! filetypesByAppId.has(row.app_id) ) {\n filetypesByAppId.set(row.app_id, []);\n }\n filetypesByAppId.get(row.app_id).push(row.type);\n }\n }\n\n return filetypesByAppId;\n }\n\n async #create ({ object, options }) {\n // Only UserActorType and AppUnderUserActorType are allowed to do this\n const actor = Context.get('actor');\n if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) {\n throw APIError.create('forbidden');\n }\n\n const user = actor.type.user;\n\n // Remove protected/read_only fields from the input (ValidationES behavior)\n {\n object = { ...object };\n for ( const field of this.constructor.PROTECTED_FIELDS ) {\n delete object[field];\n }\n for ( const field of this.constructor.READ_ONLY_FIELDS ) {\n delete object[field];\n }\n }\n\n // Validate required fields\n {\n if ( object.name === undefined ) {\n throw APIError.create('field_missing', null, { key: 'name' });\n }\n if ( object.title === undefined ) {\n throw APIError.create('field_missing', null, { key: 'title' });\n }\n if ( object.index_url === undefined ) {\n throw APIError.create('field_missing', null, { key: 'index_url' });\n }\n }\n\n // Validate fields\n {\n validate_string(object.name, {\n key: 'name',\n maxlen: config.app_name_max_length,\n regex: config.app_name_regex,\n });\n\n validate_string(object.title, {\n key: 'title',\n maxlen: config.app_title_max_length,\n });\n\n if ( object.description !== undefined && object.description !== null ) {\n validate_string(object.description, {\n key: 'description',\n maxlen: 7000,\n });\n }\n\n if ( object.icon !== undefined && object.icon !== null ) {\n if ( typeof object.icon === 'string' ) {\n object.icon = normalizeRawBase64ImageString(object.icon);\n object.icon = migrateRelativeAppIconEndpointUrl(object.icon);\n }\n if ( typeof object.icon !== 'string' ) {\n throw APIError.create('field_invalid', null, { key: 'icon' });\n }\n object.icon = object.icon.trim();\n if ( ! object.icon ) {\n // Empty icon is allowed to clear current icon.\n } else if ( object.icon.startsWith('data:') ) {\n validate_image_base64(object.icon, { key: 'icon' });\n } else if ( ! isAllowedAppIconEndpointUrl(object.icon) ) {\n throw APIError.create('field_invalid', null, { key: 'icon' });\n }\n }\n\n validate_url(object.index_url, {\n key: 'index_url',\n maxlen: 3000,\n });\n\n if ( object.maximize_on_start !== undefined ) {\n object.maximize_on_start = as_bool(object.maximize_on_start);\n }\n if ( object.background !== undefined ) {\n object.background = as_bool(object.background);\n }\n\n if ( object.metadata !== undefined && object.metadata !== null ) {\n validate_json(object.metadata, { key: 'metadata' });\n }\n\n if ( object.filetype_associations !== undefined ) {\n validate_array_of_strings(object.filetype_associations, {\n key: 'filetype_associations',\n });\n }\n }\n\n // Ensure puter.site subdomain is owned by user (if index_url uses it)\n await this.#ensure_puter_site_subdomain_is_owned(object.index_url, user);\n const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlAppOnCreate({\n object,\n options,\n user,\n });\n if ( joinedApp ) {\n return joinedApp;\n }\n await this.#ensureIndexUrlNotAlreadyInUse({\n indexUrl: object.index_url,\n });\n\n // Handle app name conflicts (AppES behavior)\n if ( await app_name_exists(object.name) ) {\n if ( options?.dedupe_name ) {\n const base = object.name;\n let number = 1;\n while ( await app_name_exists(`${base}-${number}`) ) {\n number++;\n }\n object.name = `${base}-${number}`;\n } else {\n throw APIError.create('app_name_already_in_use', null, {\n name: object.name,\n });\n }\n }\n\n // Generate UID for the new app (puter-uuid format: app-{uuid})\n const uid = `app-${uuidv4()}`;\n\n // Determine app_owner if actor is AppUnderUserActorType (SetOwnerES behavior)\n let app_owner_id = null;\n if ( actor.type instanceof AppUnderUserActorType ) {\n app_owner_id = actor.type.app.id;\n }\n\n // Execute SQL INSERT\n const insert_id = await this.#execute_insert(object, uid, user.id, app_owner_id);\n\n // Handle file type associations\n if ( object.filetype_associations ) {\n await this.#update_filetype_associations(insert_id, object.filetype_associations);\n }\n\n // Emit icon event if icon is set\n if ( object.icon ) {\n const svc_event = this.services.get('event');\n const event = {\n app_uid: uid,\n data_url: object.icon,\n url: '',\n };\n await svc_event.emit('app.new-icon', event);\n if ( typeof event.url === 'string' && event.url ) {\n this.db_write.write(\n 'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1',\n [event.url, uid],\n );\n }\n }\n\n // Return the created app\n return await this.#read({ uid });\n }\n\n async #execute_insert (object, uid, owner_user_id, app_owner_id) {\n const columns = ['uid', 'owner_user_id'];\n const values = [uid, owner_user_id];\n\n if ( app_owner_id !== null ) {\n columns.push('app_owner');\n values.push(app_owner_id);\n }\n\n const sql_column_map = {\n name: 'name',\n title: 'title',\n description: 'description',\n icon: 'icon',\n index_url: 'index_url',\n maximize_on_start: 'maximize_on_start',\n background: 'background',\n metadata: 'metadata',\n };\n\n for ( const [field, column] of Object.entries(sql_column_map) ) {\n if ( object[field] === undefined ) continue;\n\n let value = object[field];\n\n // Handle JSON fields\n if ( field === 'metadata' && value !== null ) {\n value = JSON.stringify(value);\n }\n\n // Handle boolean fields\n if ( field === 'maximize_on_start' || field === 'background' ) {\n value = value ? 1 : 0;\n }\n\n columns.push(column);\n values.push(value);\n }\n\n const placeholders = columns.map(() => '?').join(', ');\n const stmt = `INSERT INTO apps (${columns.join(', ')}) VALUES (${placeholders})`;\n const result = await this.db_write.write(stmt, values);\n\n return result.insertId;\n }\n\n async #delete ({ uid, id }) {\n // Only UserActorType and AppUnderUserActorType are allowed to do this\n const actor = Context.get('actor');\n if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) {\n throw APIError.create('forbidden');\n }\n\n // Read the existing app\n const old_app = await this.#read({\n uid,\n id,\n backend_only_options: { no_filter_owner: true },\n });\n if ( ! old_app ) {\n throw APIError.create('entity_not_found', null, {\n identifier: uid || JSON.stringify(id),\n });\n }\n\n // Check owner permission (WriteByOwnerOnlyES behavior)\n await this.#check_owner_permission(old_app);\n\n // If actor is AppUnderUserActorType, check app_owner (AppLimitedES behavior)\n if ( actor.type instanceof AppUnderUserActorType ) {\n await this.#check_app_owner_permission(old_app, actor);\n }\n\n // Call app-information service to perform the deletion (AppES behavior)\n const svc_appInformation = this.services.get('app-information');\n await svc_appInformation.delete_app(old_app.uid);\n\n return { success: true, uid: old_app.uid };\n }\n\n async #check_app_owner_permission (old_app, actor) {\n // Check if app has write permission to all user's apps\n const svc_permission = this.services.get('permission');\n const user = actor.type.user;\n const perm = `es:app:${user.uuid}:write`;\n const can_write_any = await svc_permission.check(actor, perm);\n if ( can_write_any ) {\n return;\n }\n\n // Otherwise verify the app owns this entity\n const app = actor.type.app;\n const app_owner = old_app.app_owner;\n const app_owner_uid = app_owner?.uid;\n\n if ( !app_owner_uid || app_owner_uid !== app.uid ) {\n throw APIError.create('forbidden');\n }\n }\n\n async #update ({ object, id, options }) {\n const old_app = await this.#read({\n uid: object.uid,\n id,\n backend_only_options: { no_filter_owner: true },\n });\n if ( ! old_app ) {\n throw APIError.create('entity_not_found', null, {\n identifier: object.uid || JSON.stringify(id),\n });\n }\n\n // Only UserActorType and AppUnderUserActorType are allowed to do this\n const actor = Context.get('actor');\n if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) {\n throw APIError.create('forbidden');\n }\n\n // Check owner permission (WriteByOwnerOnlyES behavior)\n await this.#check_owner_permission(old_app);\n\n // If actor is AppUnderUserActorType, check app_owner (AppLimitedES behavior)\n if ( actor.type instanceof AppUnderUserActorType ) {\n await this.#check_app_owner_permission(old_app, actor);\n }\n\n // Remove protected/read_only fields from the update (ValidationES behavior)\n {\n object = { ...object };\n for ( const field of this.constructor.PROTECTED_FIELDS ) {\n delete object[field];\n }\n for ( const field of this.constructor.READ_ONLY_FIELDS ) {\n delete object[field];\n }\n }\n\n // Validate fields\n {\n if ( object.name !== undefined ) {\n validate_string(object.name, {\n key: 'name',\n maxlen: config.app_name_max_length,\n regex: config.app_name_regex,\n });\n }\n\n if ( object.title !== undefined ) {\n validate_string(object.title, {\n key: 'title',\n maxlen: config.app_title_max_length,\n });\n }\n\n if ( object.description !== undefined && object.description !== null ) {\n validate_string(object.description, {\n key: 'description',\n maxlen: 7000,\n });\n }\n\n if ( object.icon !== undefined && object.icon !== null ) {\n if ( typeof object.icon === 'string' ) {\n object.icon = normalizeRawBase64ImageString(object.icon);\n object.icon = migrateRelativeAppIconEndpointUrl(object.icon);\n }\n if ( typeof object.icon !== 'string' ) {\n throw APIError.create('field_invalid', null, { key: 'icon' });\n }\n object.icon = object.icon.trim();\n if ( ! object.icon ) {\n // Empty icon is allowed to clear current icon.\n } else if ( object.icon.startsWith('data:') ) {\n validate_image_base64(object.icon, { key: 'icon' });\n } else if ( ! isAllowedAppIconEndpointUrl(object.icon) ) {\n throw APIError.create('field_invalid', null, { key: 'icon' });\n }\n }\n\n if ( object.index_url !== undefined ) {\n validate_url(object.index_url, {\n key: 'index_url',\n maxlen: 3000,\n });\n }\n\n // Flag type - adapt values using as_bool\n if ( object.maximize_on_start !== undefined ) {\n object.maximize_on_start = as_bool(object.maximize_on_start);\n }\n if ( object.background !== undefined ) {\n object.background = as_bool(object.background);\n }\n\n if ( object.metadata !== undefined && object.metadata !== null ) {\n validate_json(object.metadata, { key: 'metadata' });\n }\n\n if ( object.filetype_associations !== undefined ) {\n validate_array_of_strings(object.filetype_associations, {\n key: 'filetype_associations',\n });\n }\n }\n\n // Handle app-specific logic (AppES behavior)\n const user = actor.type.user;\n const oldAppId = await this.#resolveAppId(old_app);\n\n // Ensure puter.site subdomain is owned by user (if index_url changed)\n if ( object.index_url && object.index_url !== old_app.index_url ) {\n await this.#ensure_puter_site_subdomain_is_owned(object.index_url, user);\n const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlAppOnCreate({\n object,\n options,\n user,\n excludeAppId: oldAppId,\n });\n if ( joinedApp ) {\n return joinedApp;\n }\n await this.#ensureIndexUrlNotAlreadyInUse({\n indexUrl: object.index_url,\n excludeAppId: oldAppId,\n });\n }\n\n // Handle app name conflicts\n if ( object.name !== undefined ) {\n await this.#handle_name_conflict(object, old_app, options);\n }\n\n // Build and execute SQL UPDATE\n const { insert_id } = await this.#execute_update(object, old_app);\n\n // Handle file type associations\n if ( object.filetype_associations !== undefined ) {\n await this.#update_filetype_associations(insert_id, object.filetype_associations);\n }\n\n // Emit events for icon/name or app changes\n await this.#emit_change_events(object, old_app);\n\n // Return the updated app (re-fetch for client-safe output)\n // TODO: optimize this\n return await this.#read({ uid: old_app.uid });\n }\n\n async #resolveAppId (app) {\n const appId = Number(app?.id);\n if ( Number.isInteger(appId) && appId > 0 ) return appId;\n if ( typeof app?.uid !== 'string' || !app.uid ) return undefined;\n\n const rows = await this.db.read(\n 'SELECT id FROM apps WHERE uid = ? LIMIT 1',\n [app.uid],\n );\n const resolvedId = Number(rows?.[0]?.id);\n if ( Number.isInteger(resolvedId) && resolvedId > 0 ) return resolvedId;\n return undefined;\n }\n\n async #check_owner_permission (old_app) {\n const svc_permission = this.services.get('permission');\n const actor = Context.get('actor');\n\n // Check if user has system-wide write permission\n {\n // We need to fix eslint rule for multi-line calls\n const has_permission_to_write_all = await svc_permission.check(\n actor,\n this.constructor.WRITE_ALL_OWNER_PERMISSION,\n );\n\n if ( has_permission_to_write_all ) {\n return;\n }\n }\n\n // Check if user owns the app\n {\n const user = Context.get('user');\n if ( ! old_app.owner ) {\n throw APIError.create('forbidden');\n }\n if ( user.id !== old_app.owner.id ) {\n throw APIError.create('forbidden');\n }\n }\n }\n\n /**\n * Resolves an app's subdomain to its puter.site root_dir_id.\n * Tries associated_app_id first, then falls back to index_url-based lookup.\n * @param {Object} app - App object with id, index_url, uid\n * @returns {Promise} root_dir_id\n * @throws {APIError} entity_not_found if the app has no subdomain / root directory\n */\n async getAppRootDirId (app) {\n const db_sites = this.services.get('database').get(DB_READ, 'sites');\n const rows = await db_sites.read(\n 'SELECT root_dir_id FROM subdomains WHERE associated_app_id = ? AND root_dir_id IS NOT NULL LIMIT 1',\n [app.id],\n );\n if ( rows?.[0]?.root_dir_id != null ) {\n return rows[0].root_dir_id;\n }\n\n let hostname;\n try {\n hostname = (new URL(app.index_url)).hostname.toLowerCase();\n } catch {\n throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` });\n }\n const hosting_domain = config.static_hosting_domain?.toLowerCase();\n if ( !hosting_domain || !hostname.endsWith(`.${hosting_domain}`) ) {\n throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` });\n }\n const subdomain = hostname.slice(0, hostname.length - hosting_domain.length - 1);\n const site = await this.services.get('puter-site').get_subdomain(subdomain, { is_custom_domain: false });\n if ( ! site?.root_dir_id ) {\n throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` });\n }\n return site.root_dir_id;\n }\n\n async #ensure_puter_site_subdomain_is_owned (index_url, user) {\n if ( ! user ) return;\n const subdomain = this.#extractPuterHostedSubdomain(index_url);\n if ( ! subdomain ) return;\n\n const svc_puterSite = this.services.get('puter-site');\n const site = await svc_puterSite.get_subdomain(subdomain, { is_custom_domain: false });\n\n if ( !site || site.user_id !== user.id ) {\n throw APIError.create('subdomain_not_owned', null, { subdomain });\n }\n }\n\n #normalizeConfiguredHostedDomain (domainValue) {\n if ( typeof domainValue !== 'string' ) return null;\n const normalizedDomain = domainValue.trim().toLowerCase().replace(/^\\./, '');\n if ( ! normalizedDomain ) return null;\n return normalizedDomain.split(':')[0] || null;\n }\n\n #getPuterHostedDomains () {\n const domains = new Set();\n for ( const configuredDomain of [\n config.static_hosting_domain,\n config.static_hosting_domain_alt,\n config.private_app_hosting_domain,\n config.private_app_hosting_domain_alt,\n ] ) {\n const normalizedConfiguredDomain = this.#normalizeConfiguredHostedDomain(configuredDomain);\n if ( normalizedConfiguredDomain ) {\n domains.add(normalizedConfiguredDomain);\n }\n }\n return [...domains];\n }\n\n #extractPuterHostedSubdomain (indexUrl) {\n if ( typeof indexUrl !== 'string' || !indexUrl ) return null;\n\n let hostname;\n try {\n hostname = (new URL(indexUrl)).hostname.toLowerCase();\n } catch {\n return null;\n }\n\n const hostedDomains = this.#getPuterHostedDomains();\n hostedDomains.sort((domainA, domainB) => domainB.length - domainA.length);\n\n for ( const hostedDomain of hostedDomains ) {\n const suffix = `.${hostedDomain}`;\n if ( hostname.endsWith(suffix) ) {\n const subdomain = hostname.slice(0, hostname.length - suffix.length);\n return subdomain || null;\n }\n }\n\n return null;\n }\n\n #isPuterHostedIndexUrl (indexUrl) {\n return !!this.#extractPuterHostedSubdomain(indexUrl);\n }\n\n #buildEquivalentIndexUrlCandidates (indexUrl) {\n if ( typeof indexUrl !== 'string' || !indexUrl.trim() ) {\n return [];\n }\n\n try {\n const parsedIndexUrl = new URL(indexUrl);\n const origin = `${parsedIndexUrl.protocol}//${parsedIndexUrl.host.toLowerCase()}`;\n const pathname = parsedIndexUrl.pathname || '/';\n\n const candidates = new Set();\n if ( pathname === '/' || pathname.toLowerCase() === '/index.html' ) {\n candidates.add(origin);\n candidates.add(`${origin}/`);\n candidates.add(`${origin}/index.html`);\n } else {\n const normalizedPath = pathname.endsWith('/')\n ? pathname.slice(0, -1)\n : pathname;\n candidates.add(`${origin}${normalizedPath}`);\n candidates.add(`${origin}${normalizedPath}/`);\n }\n\n return [...candidates];\n } catch {\n return [indexUrl.trim()];\n }\n }\n\n async #findIndexUrlConflictRow ({ indexUrl, excludeAppId } = {}) {\n if ( ! this.#isPuterHostedIndexUrl(indexUrl) ) {\n return null;\n }\n\n const indexUrlCandidates = this.#buildEquivalentIndexUrlCandidates(indexUrl);\n if ( indexUrlCandidates.length === 0 ) return null;\n if ( hasIndexUrlUniquenessExemption(indexUrlCandidates) ) return null;\n\n const placeholders = indexUrlCandidates.map(() => '?').join(', ');\n const parameters = [...indexUrlCandidates];\n let query = `SELECT id, uid, owner_user_id, index_url FROM apps WHERE index_url IN (${placeholders})`;\n\n if ( Number.isInteger(excludeAppId) && excludeAppId > 0 ) {\n query += ' AND id != ?';\n parameters.push(excludeAppId);\n }\n\n query += ' ORDER BY timestamp ASC, id ASC LIMIT 1';\n\n const rows = await this.db.read(query, parameters);\n const conflictRow = rows.find(row => {\n if (\n Number.isInteger(excludeAppId)\n && excludeAppId > 0\n && Number(row?.id) === excludeAppId\n ) {\n return false;\n }\n if ( typeof row?.index_url === 'string' ) {\n return indexUrlCandidates.includes(row.index_url);\n }\n return true;\n });\n return conflictRow || null;\n }\n\n async #ensureIndexUrlNotAlreadyInUse ({ indexUrl, excludeAppId } = {}) {\n const conflictRow = await this.#findIndexUrlConflictRow({ indexUrl, excludeAppId });\n if ( conflictRow ) {\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: indexUrl,\n app_uid: conflictRow.uid,\n });\n }\n }\n\n async #claimAppOwnershipByIdForUser ({ appId, userId }) {\n if ( !Number.isInteger(appId) || appId <= 0 ) return;\n if ( !Number.isInteger(userId) || userId <= 0 ) return;\n\n await this.db_write.write(\n 'UPDATE apps SET owner_user_id = ? WHERE id = ? AND owner_user_id IS NULL',\n [userId, appId],\n );\n }\n\n #buildCanonicalAppUidAliasKey (oldAppUid) {\n return `${APP_UID_ALIAS_KEY_PREFIX}:${oldAppUid}`;\n }\n\n #buildCanonicalAppUidAliasReverseKey (canonicalAppUid) {\n return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`;\n }\n\n #normalizeCanonicalAliasUidList (value) {\n if ( ! Array.isArray(value) ) return [];\n const normalizedList = [];\n const seen = new Set();\n for ( const item of value ) {\n if ( typeof item !== 'string' || !item ) continue;\n if ( seen.has(item) ) continue;\n seen.add(item);\n normalizedList.push(item);\n }\n return normalizedList;\n }\n\n async #readCanonicalAppUidAlias (oldAppUid) {\n if ( typeof oldAppUid !== 'string' || !oldAppUid ) return null;\n\n const kvStore = this.services.get('puter-kvstore');\n const suService = this.services.get('su');\n if ( !kvStore || typeof kvStore.get !== 'function' ) return null;\n if ( !suService || typeof suService.sudo !== 'function' ) return null;\n\n const key = this.#buildCanonicalAppUidAliasKey(oldAppUid);\n try {\n const canonicalAppUid = await suService.sudo(() => kvStore.get({ key }));\n if ( typeof canonicalAppUid === 'string' && canonicalAppUid ) {\n return canonicalAppUid;\n }\n } catch {\n // Alias reads are best-effort.\n }\n return null;\n }\n\n async #writeCanonicalAppUidAlias ({ oldAppUid, canonicalAppUid }) {\n if ( typeof oldAppUid !== 'string' || !oldAppUid ) return;\n if ( typeof canonicalAppUid !== 'string' || !canonicalAppUid ) return;\n if ( oldAppUid === canonicalAppUid ) return;\n\n const kvStore = this.services.get('puter-kvstore');\n const suService = this.services.get('su');\n if ( !kvStore || typeof kvStore.set !== 'function' ) return;\n if ( !suService || typeof suService.sudo !== 'function' ) return;\n\n const key = this.#buildCanonicalAppUidAliasKey(oldAppUid);\n const reverseKey = this.#buildCanonicalAppUidAliasReverseKey(canonicalAppUid);\n const expireAt = Math.floor(Date.now() / 1000) + APP_UID_ALIAS_TTL_SECONDS;\n try {\n await suService.sudo(async () => {\n const reverseValue = await kvStore.get({ key: reverseKey });\n const reverseAliases = this.#normalizeCanonicalAliasUidList(reverseValue);\n if ( ! reverseAliases.includes(oldAppUid) ) {\n reverseAliases.push(oldAppUid);\n }\n\n await kvStore.set({\n key,\n value: canonicalAppUid,\n expireAt,\n });\n await kvStore.set({\n key: reverseKey,\n value: reverseAliases,\n expireAt,\n });\n });\n } catch {\n // Alias writes are best-effort.\n }\n }\n\n async #maybeJoinOwnedHostedIndexUrlAppOnCreate ({\n object,\n options,\n user,\n excludeAppId,\n } = {}) {\n const indexUrl = object?.index_url;\n const sourceAppUid = object?.uid;\n if ( ! this.#isPuterHostedIndexUrl(indexUrl) ) {\n return null;\n }\n\n const conflictRow = await this.#findIndexUrlConflictRow({\n indexUrl,\n excludeAppId,\n });\n if ( ! conflictRow ) {\n return null;\n }\n\n const conflictOwnerUserId = Number(conflictRow.owner_user_id);\n if (\n Number.isInteger(conflictOwnerUserId)\n && conflictOwnerUserId > 0\n && conflictOwnerUserId !== user.id\n ) {\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: indexUrl,\n app_uid: conflictRow.uid,\n });\n }\n\n if ( !Number.isInteger(conflictOwnerUserId) || conflictOwnerUserId <= 0 ) {\n await this.#claimAppOwnershipByIdForUser({\n appId: conflictRow.id,\n userId: user.id,\n });\n }\n\n const appToJoin = await this.#read({\n uid: conflictRow.uid,\n backend_only_options: {\n no_filter_owner: true,\n },\n });\n if ( !appToJoin || appToJoin.uid !== conflictRow.uid ) {\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: indexUrl,\n app_uid: conflictRow.uid,\n });\n }\n const appToJoinOwnerId = Number(appToJoin.owner?.id);\n if ( !Number.isInteger(appToJoinOwnerId) || appToJoinOwnerId !== user.id ) {\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: indexUrl,\n app_uid: conflictRow.uid,\n });\n }\n if (\n Number.isInteger(conflictOwnerUserId)\n && conflictOwnerUserId === user.id\n && !this.#isOriginBootstrapApp(appToJoin)\n ) {\n // Prevent merging arbitrary same-owner apps; only allow the\n // auto-created origin bootstrap app to be absorbed.\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: indexUrl,\n app_uid: conflictRow.uid,\n });\n }\n\n const joinedObject = {\n ...object,\n uid: appToJoin.uid,\n };\n const requestedJoinedName = (\n typeof joinedObject.name === 'string'\n ? joinedObject.name.trim()\n : ''\n ) || null;\n const shouldReapplyRequestedNameAfterMerge = (\n !!object?.uid\n && !!requestedJoinedName\n );\n if ( object?.uid && joinedObject.name !== undefined ) {\n delete joinedObject.name;\n }\n\n let joinedApp = await this.#update({\n object: joinedObject,\n options,\n });\n\n if ( sourceAppUid && sourceAppUid !== appToJoin.uid ) {\n await this.#writeCanonicalAppUidAlias({\n oldAppUid: sourceAppUid,\n canonicalAppUid: appToJoin.uid,\n });\n const svc_appInformation = this.services.get('app-information');\n if ( svc_appInformation?.delete_app ) {\n await svc_appInformation.delete_app(sourceAppUid, undefined, {\n preserveCanonicalUidAlias: true,\n });\n }\n }\n\n if ( shouldReapplyRequestedNameAfterMerge ) {\n joinedApp = await this.#update({\n object: {\n uid: appToJoin.uid,\n name: requestedJoinedName,\n },\n options,\n });\n }\n\n return joinedApp;\n }\n\n #isOriginBootstrapApp (app) {\n if ( !app || typeof app !== 'object' ) return false;\n if ( typeof app.uid !== 'string' || !app.uid ) return false;\n if ( app.name !== app.uid ) return false;\n if ( app.title !== app.uid ) return false;\n if ( typeof app.description !== 'string' ) return false;\n return app.description.startsWith('App created from origin ');\n }\n\n async #handle_name_conflict (object, old_app, options) {\n const new_name = object.name;\n const old_name = old_app.name;\n\n // If the name hasn't changed, nothing to do\n if ( new_name === old_name ) {\n delete object.name;\n return;\n }\n\n // Check if the name is taken\n if ( await app_name_exists(new_name) ) {\n if ( options?.dedupe_name ) {\n // Auto-deduplicate the name\n let number = 1;\n while ( await app_name_exists(`${new_name}-${number}`) ) {\n number++;\n }\n object.name = `${new_name}-${number}`;\n } else {\n // Check if this is an old name of the same app\n const svc_oldAppName = this.services.get('old-app-name');\n const name_info = await svc_oldAppName.check_app_name(new_name);\n if ( !name_info || name_info.app_uid !== old_app.uid ) {\n throw APIError.create('app_name_already_in_use', null, {\n name: new_name,\n });\n }\n // Remove the old name from the old-app-name service\n await svc_oldAppName.remove_name(name_info.id);\n }\n }\n }\n\n async #execute_update (object, old_app) {\n // Map object fields to SQL columns\n const sql_column_map = {\n name: 'name',\n title: 'title',\n description: 'description',\n icon: 'icon',\n index_url: 'index_url',\n maximize_on_start: 'maximize_on_start',\n background: 'background',\n metadata: 'metadata',\n };\n\n const set_clauses = [];\n const values = [];\n\n for ( const [field, column] of Object.entries(sql_column_map) ) {\n if ( object[field] === undefined ) continue;\n\n let value = object[field];\n\n // Handle JSON fields\n if ( field === 'metadata' && value !== null ) {\n value = JSON.stringify(value);\n }\n\n // Handle boolean fields\n if ( field === 'maximize_on_start' || field === 'background' ) {\n value = value ? 1 : 0;\n }\n\n set_clauses.push(`${column} = ?`);\n values.push(value);\n }\n\n if ( set_clauses.length > 0 ) {\n values.push(old_app.uid);\n const stmt = `UPDATE apps SET ${set_clauses.join(', ')} WHERE uid = ? LIMIT 1`;\n await this.db_write.write(stmt, values);\n }\n\n // Fetch the internal ID\n const rows = await this.db.read(\n 'SELECT id FROM apps WHERE uid = ?',\n [old_app.uid],\n );\n return { insert_id: rows[0]?.id };\n }\n\n async #update_filetype_associations (app_id, filetype_associations) {\n const oldAssociations = await this.db.read(\n 'SELECT type FROM app_filetype_association WHERE app_id = ?',\n [app_id],\n );\n const normalizedOld = oldAssociations\n .map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\\./, ''))\n .filter(Boolean);\n const normalizedNew = (filetype_associations ?? [])\n .map(ft => String(ft).trim().toLowerCase().replace(/^\\./, ''))\n .filter(Boolean);\n\n // Remove old file associations\n await this.db_write.write(\n 'DELETE FROM app_filetype_association WHERE app_id = ?',\n [app_id],\n );\n\n // Add new file associations\n if ( ! normalizedNew.length ) {\n const affectedExtensions = new Set(normalizedOld);\n if ( affectedExtensions.size ) {\n await deleteRedisKeys(Array.from(affectedExtensions)\n .map(ext => AppRedisCacheSpace.associationAppsKey(ext)));\n }\n return;\n }\n\n const stmt =\n `INSERT INTO app_filetype_association (app_id, type) VALUES ${\n normalizedNew.map(() => '(?, ?)').join(', ')}`;\n const values = normalizedNew.flatMap(ft => [app_id, ft]);\n await this.db_write.write(stmt, values);\n\n const affectedExtensions = new Set([...normalizedOld, ...normalizedNew]);\n if ( affectedExtensions.size ) {\n await deleteRedisKeys(Array.from(affectedExtensions)\n .map(ext => AppRedisCacheSpace.associationAppsKey(ext)));\n }\n }\n\n async #emit_change_events (object, old_app) {\n const svc_event = this.services.get('event');\n const app = {\n ...old_app,\n ...object,\n uid: old_app.uid,\n };\n\n await svc_event.emit('app.changed', {\n app_uid: old_app.uid,\n action: 'updated',\n app,\n old_app,\n });\n\n // Emit icon change event\n if ( object.icon !== undefined && object.icon !== old_app.icon ) {\n const event = {\n app_uid: old_app.uid,\n data_url: object.icon,\n };\n await svc_event.emit('app.new-icon', event);\n if ( typeof event.url === 'string' && event.url ) {\n await this.db_write.write(\n 'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1',\n [event.url, old_app.uid],\n );\n }\n }\n\n // Emit name change event\n if ( object.name !== undefined && object.name !== old_app.name ) {\n const event = {\n app_uid: old_app.uid,\n new_name: object.name,\n old_name: old_app.name,\n };\n await svc_event.emit('app.rename', event);\n }\n }\n\n #build_complex_id_where (id) {\n const id_keys = Object.keys(id);\n id_keys.sort();\n\n // 1. Validate the identifier key from `id`\n\n const redundant_identifiers = this.constructor.REDUNDANT_IDENTIFIERS;\n let match_found = false;\n\n for ( let key_set of redundant_identifiers ) {\n key_set = Array.isArray(key_set) ? key_set : [key_set];\n const sorted_key_set = [...key_set].sort();\n\n // Check if id_keys matches this key_set exactly\n if ( id_keys.length === sorted_key_set.length &&\n id_keys.every((k, i) => k === sorted_key_set[i]) ) {\n match_found = true;\n break;\n }\n }\n\n if ( ! match_found ) {\n throw new Error(`Invalid complex id keys: ${id_keys.join(', ')}. ` +\n `Allowed: ${redundant_identifiers.join(', ')}`);\n }\n\n // 2. Build the SQL string for the predicate\n\n const conditions = [];\n const values = [];\n\n for ( const key of id_keys ) {\n conditions.push(`apps.${key} = ?`);\n values.push(id[key]);\n }\n\n return {\n clause: conditions.join(' AND '),\n values,\n };\n }\n\n /**\n * Checks if a protected app should be filtered out (not accessible to the current actor).\n * Returns true if the app should be filtered out, false if it's accessible.\n *\n * @param {Object} app - The app object with protected, uid, and owner fields\n * @param {number} owner_user_id - The database ID of the app owner (for accurate comparison)\n * @returns {Promise} true if app should be filtered out, false if accessible\n */\n async #check_protected_app_access (app, owner_user_id) {\n // If it's not a protected app, no worries - allow it\n if ( ! app.protected ) {\n return false;\n }\n\n const actor = Context.get('actor');\n const services = this.services;\n\n // If actor is this app itself, allow it\n if (\n actor.type instanceof AppUnderUserActorType &&\n app.uid === actor.type.app.uid\n ) {\n return false;\n }\n\n // If actor is owner of this app, allow it\n // Compare using owner_user_id from database for accuracy\n if (\n actor.type instanceof UserActorType &&\n owner_user_id &&\n owner_user_id === actor.type.user.id\n ) {\n return false;\n }\n\n // Now we need to check for permission\n const app_uid = app.uid;\n const svc_permission = services.get('permission');\n const permission_to_check = `app:uid#${app_uid}:access`;\n\n // If they have permission, allow it\n if ( await svc_permission.check(actor, permission_to_check) ) {\n return false;\n }\n\n // No access - filter it out\n return true;\n }\n}\n"
},
{
"path": "src/backend/src/modules/data-access/AppService.test.js",
"content": "import { beforeEach, describe, expect, it, vi } from 'vitest';\nimport AppService from './AppService.js';\n\n// Mock the Context module\nvi.mock('../../util/context.js', () => ({\n Context: {\n get: vi.fn(),\n },\n}));\n\n// Mock the helpers module\nvi.mock('../../helpers.js', () => ({\n app_name_exists: vi.fn(),\n}));\n\n// Mock the Actor module\nvi.mock('../../services/auth/Actor.js', () => ({\n UserActorType: class UserActorType {\n },\n AppUnderUserActorType: class AppUnderUserActorType {\n },\n}));\n\n// Mock the validation module\nvi.mock('./lib/validation.js', () => ({\n validate_string: vi.fn(),\n validate_url: vi.fn(),\n validate_image_base64: vi.fn(),\n validate_json: vi.fn(),\n validate_array_of_strings: vi.fn(),\n}));\n\n// Mock config\nvi.mock('../../config.js', () => ({\n default: {\n app_name_max_length: 100,\n app_name_regex: /^[a-z0-9-]+$/,\n app_title_max_length: 200,\n static_hosting_domain: 'puter.site',\n static_hosting_domain_alt: 'puter.host',\n private_app_hosting_domain: 'puter.app',\n private_app_hosting_domain_alt: 'puter.dev',\n origin: 'https://puter.localhost',\n api_base_url: 'https://api.puter.localhost',\n },\n}));\n\nimport { app_name_exists } from '../../helpers.js';\nimport { AppUnderUserActorType, UserActorType } from '../../services/auth/Actor.js';\nimport { Context } from '../../util/context.js';\nimport {\n validate_string,\n validate_url,\n} from './lib/validation.js';\n\ndescribe('AppService', () => {\n let appService;\n let mockDb;\n let mockDbWrite;\n let mockServices;\n let mockEventService;\n let mockPermissionService;\n let mockPuterSiteService;\n let mockOldAppNameService;\n let mockAppInformationService;\n let mockKvStoreService;\n let mockSuService;\n\n // Helper to create a mock database row\n const createMockAppRow = (overrides = {}) => ({\n id: 1,\n uid: 'app-uid-123',\n name: 'test-app',\n title: 'Test App',\n description: 'A test application',\n icon: 'icon.png',\n index_url: 'https://example.com/app',\n created_at: '2024-01-01T00:00:00Z',\n created_from_origin: 'localhost',\n metadata: '{}',\n stats: '{}',\n approved_for_incentive_program: 0,\n approved_for_listing: 1,\n approved_for_opening_items: 1,\n background: 0,\n godmode: 0,\n is_private: 0,\n maximize_on_start: 0,\n protected: 0,\n owner_user_id: 1,\n owner_user_username: 'testuser',\n owner_user_uuid: 'user-uuid-456',\n app_owner_uid: 'owner-app-uid-789',\n filetypes: '[\"txt\", \"doc\"]',\n ...overrides,\n });\n\n // Helper to create a mock actor\n const createMockUserActor = (userId = 1) => ({\n type: Object.assign(new UserActorType(), { user: { id: userId } }),\n });\n\n const createMockAppUnderUserActor = (userId = 1, appId = 100) => ({\n type: Object.assign(new AppUnderUserActorType(), {\n user: { id: userId },\n app: { id: appId, uid: 'creator-app-uid' },\n }),\n });\n\n // Helper to setup Context.get mock for create/update tests\n const setupContextForWrite = (actor, user = { id: 1 }) => {\n Context.get.mockImplementation((key) => {\n if ( key === 'actor' ) return actor;\n if ( key === 'user' ) return user;\n return null;\n });\n };\n\n beforeEach(() => {\n // Reset mocks\n vi.clearAllMocks();\n\n // Reset helper mocks\n app_name_exists.mockResolvedValue(false);\n\n // Mock database (read)\n mockDb = {\n read: vi.fn(),\n case: vi.fn().mockImplementation(({ sqlite }) => sqlite),\n };\n\n // Mock database (write)\n mockDbWrite = {\n write: vi.fn().mockResolvedValue({ insertId: 1 }),\n };\n\n // Mock event service\n mockEventService = {\n emit: vi.fn().mockResolvedValue(undefined),\n };\n\n // Mock permission service\n mockPermissionService = {\n check: vi.fn().mockResolvedValue(false),\n scan: vi.fn().mockResolvedValue([]),\n };\n\n // Mock puter-site service\n mockPuterSiteService = {\n get_subdomain: vi.fn().mockResolvedValue(null),\n };\n\n // Mock old-app-name service\n mockOldAppNameService = {\n check_app_name: vi.fn().mockResolvedValue(null),\n remove_name: vi.fn().mockResolvedValue(undefined),\n };\n\n // Mock app-information service\n mockAppInformationService = {\n delete_app: vi.fn().mockResolvedValue(undefined),\n };\n\n mockKvStoreService = {\n get: vi.fn().mockResolvedValue(null),\n set: vi.fn().mockResolvedValue(true),\n };\n\n mockSuService = {\n sudo: vi.fn(async (actorOrCallback, maybeCallback) => {\n const callback = maybeCallback || actorOrCallback;\n return await callback();\n }),\n };\n\n // Mock services\n mockServices = {\n get: vi.fn().mockImplementation((serviceName) => {\n if ( serviceName === 'database' ) {\n return {\n get: vi.fn().mockImplementation((mode) => {\n if ( mode === 'write' ) return mockDbWrite;\n return mockDb;\n }),\n };\n }\n if ( serviceName === 'event' ) return mockEventService;\n if ( serviceName === 'permission' ) return mockPermissionService;\n if ( serviceName === 'puter-site' ) return mockPuterSiteService;\n if ( serviceName === 'old-app-name' ) return mockOldAppNameService;\n if ( serviceName === 'app-information' ) return mockAppInformationService;\n if ( serviceName === 'puter-kvstore' ) return mockKvStoreService;\n if ( serviceName === 'su' ) return mockSuService;\n return null;\n }),\n };\n\n // Create AppService instance\n appService = new AppService({\n services: mockServices,\n config: {},\n name: 'app-service',\n args: {},\n context: {\n get: vi.fn().mockReturnValue(mockServices),\n },\n });\n\n // Manually call _init to set up the service\n appService.repository = {};\n appService.db = mockDb;\n appService.db_write = mockDbWrite;\n });\n\n describe('#read', () => {\n it('should read an app by uid', async () => {\n const mockRow = createMockAppRow();\n mockDb.read.mockResolvedValueOnce([mockRow]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(mockDb.read).toHaveBeenCalledTimes(1);\n expect(mockDb.read).toHaveBeenNthCalledWith(\n 1,\n expect.stringContaining('WHERE apps.uid = ?'),\n ['app-uid-123'],\n );\n expect(result).toBeDefined();\n expect(result.uid).toBe('app-uid-123');\n expect(result.name).toBe('test-app');\n expect(result.title).toBe('Test App');\n });\n\n it('should read an app by complex id (name)', async () => {\n const mockRow = createMockAppRow();\n mockDb.read.mockResolvedValueOnce([mockRow]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { id: { name: 'test-app' } });\n\n expect(mockDb.read).toHaveBeenCalledTimes(1);\n expect(mockDb.read).toHaveBeenNthCalledWith(\n 1,\n expect.stringContaining('WHERE apps.name = ?'),\n ['test-app'],\n );\n expect(result).toBeDefined();\n expect(result.name).toBe('test-app');\n });\n\n it('should throw entity_not_found when no app is found', async () => {\n mockDb.read.mockResolvedValue([]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.read.call(appService, { uid: 'nonexistent-uid' })).rejects.toMatchObject({\n fields: { code: 'entity_not_found' },\n });\n });\n\n it('should resolve app by canonical uid alias when old uid is missing', async () => {\n const canonicalRow = createMockAppRow({\n uid: 'app-canonical-uid-123',\n });\n mockDb.read\n .mockResolvedValueOnce([])\n .mockResolvedValueOnce([canonicalRow]);\n mockKvStoreService.get.mockResolvedValue('app-canonical-uid-123');\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-old-uid-123' });\n\n expect(result.uid).toBe('app-canonical-uid-123');\n expect(mockSuService.sudo).toHaveBeenCalled();\n expect(mockKvStoreService.get).toHaveBeenCalledWith({\n key: 'app:canonicalUidAlias:app-old-uid-123',\n });\n expect(mockDb.read).toHaveBeenNthCalledWith(\n 2,\n expect.stringContaining('WHERE apps.uid = ?'),\n ['app-canonical-uid-123'],\n );\n });\n\n it('should throw an error when neither uid nor id is provided', async () => {\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.read.call(appService, {})).rejects.toThrow(\n 'read requires either uid or id',\n );\n });\n\n it('should throw an error for invalid complex id keys', async () => {\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.read.call(appService, { id: { invalidKey: 'value' } })).rejects.toThrow('Invalid complex id keys');\n });\n\n it('should correctly coerce boolean fields from database', async () => {\n const mockRow = createMockAppRow({\n approved_for_incentive_program: 1,\n approved_for_listing: '1',\n approved_for_opening_items: 0,\n background: '0',\n godmode: 1,\n is_private: '1',\n maximize_on_start: '1',\n protected: 0,\n });\n mockDb.read.mockResolvedValue([mockRow]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(result.approved_for_incentive_program).toBe(true);\n expect(result.approved_for_listing).toBe(true);\n expect(result.approved_for_opening_items).toBe(false);\n expect(result.background).toBe(false);\n expect(result.godmode).toBe(true);\n expect(result.is_private).toBe(true);\n expect(result.maximize_on_start).toBe(true);\n expect(result.protected).toBe(false);\n });\n\n it('should parse filetypes JSON and strip leading dots', async () => {\n const mockRow = createMockAppRow({\n filetypes: '[\".txt\", \".doc\", \"pdf\"]',\n });\n mockDb.read.mockResolvedValueOnce([mockRow]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(result.filetype_associations).toEqual(['txt', 'doc', 'pdf']);\n expect(mockDb.read).toHaveBeenCalledTimes(1);\n });\n\n it('should filter out null values in filetypes array', async () => {\n const mockRow = createMockAppRow({\n filetypes: '[\".txt\", null, \"pdf\"]',\n });\n mockDb.read.mockResolvedValueOnce([mockRow]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(result.filetype_associations).toEqual(['txt', 'pdf']);\n expect(mockDb.read).toHaveBeenCalledTimes(1);\n });\n\n it('should query filetype associations table when filetypes JSON is missing', async () => {\n const mockRow = createMockAppRow({ filetypes: null });\n mockDb.read\n .mockResolvedValueOnce([mockRow])\n .mockResolvedValueOnce([\n { type: '.txt' },\n { type: null },\n { type: 'pdf' },\n ]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(result.filetype_associations).toEqual(['txt', 'pdf']);\n expect(mockDb.read).toHaveBeenCalledTimes(2);\n expect(mockDb.read).toHaveBeenNthCalledWith(\n 2,\n 'SELECT type FROM app_filetype_association WHERE app_id = ?',\n [mockRow.id],\n );\n });\n\n it('should have owner parameter', async () => {\n const mockRow = createMockAppRow();\n mockDb.read.mockResolvedValue([mockRow]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(result.owner).toEqual({\n username: 'testuser',\n uuid: 'user-uuid-456',\n });\n });\n\n it('should include app_owner in the result', async () => {\n const mockRow = createMockAppRow();\n mockDb.read.mockResolvedValue([mockRow]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(result.app_owner).toEqual({\n uid: 'owner-app-uid-789',\n });\n });\n\n it('should fetch icon with size when icon_size param is provided', async () => {\n const mockRow = createMockAppRow();\n mockDb.read.mockResolvedValue([mockRow]);\n\n const mockIconService = {\n getAppIconPath: vi.fn().mockReturnValue('/app-icon/app-uid-123/64'),\n };\n\n appService.context = {\n get: vi.fn().mockImplementation((key) => {\n if ( key === 'services' ) {\n return {\n get: vi.fn().mockImplementation((name) => {\n if ( name === 'app-icon' ) return mockIconService;\n return null;\n }),\n };\n }\n return null;\n }),\n };\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, {\n uid: 'app-uid-123',\n params: { icon_size: 64 },\n });\n\n expect(mockIconService.getAppIconPath).toHaveBeenCalledWith({\n appUid: 'app-uid-123',\n size: 64,\n });\n expect(result.icon).toBe('/app-icon/app-uid-123/64');\n });\n\n it('should route base64 icons through app-icon endpoint even without icon_size', async () => {\n const mockRow = createMockAppRow({\n icon: 'data:image/png;base64,abc123',\n icon_is_base64: 1,\n });\n mockDb.read.mockResolvedValue([mockRow]);\n\n const mockIconService = {\n getAppIconPath: vi.fn().mockReturnValue('/app-icon/app-uid-123/128'),\n };\n\n appService.context = {\n get: vi.fn().mockImplementation((key) => {\n if ( key === 'services' ) {\n return {\n get: vi.fn().mockImplementation((name) => {\n if ( name === 'app-icon' ) return mockIconService;\n return null;\n }),\n };\n }\n return null;\n }),\n };\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });\n\n expect(mockIconService.getAppIconPath).toHaveBeenCalledWith({\n appUid: 'app-uid-123',\n size: undefined,\n });\n expect(result.icon).toBe('/app-icon/app-uid-123/128');\n });\n\n it('should keep original icon when icon service throws', async () => {\n const mockRow = createMockAppRow();\n mockDb.read.mockResolvedValue([mockRow]);\n\n const mockErrorService = {\n report: vi.fn(),\n };\n\n const mockIconService = {\n getAppIconPath: vi.fn().mockImplementation(() => {\n throw new Error('Icon fetch failed');\n }),\n };\n\n appService.context = {\n get: vi.fn().mockImplementation((key) => {\n if ( key === 'services' ) {\n return {\n get: vi.fn().mockImplementation((name) => {\n if ( name === 'app-icon' ) return mockIconService;\n if ( name === 'error-service' ) return mockErrorService;\n return null;\n }),\n };\n }\n return null;\n }),\n };\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.read.call(appService, {\n uid: 'app-uid-123',\n params: { icon_size: 64 },\n });\n\n expect(mockErrorService.report).toHaveBeenCalledWith(\n 'AppES:read_transform',\n expect.objectContaining({ source: expect.any(Error) }),\n );\n expect(result.icon).toBe('icon.png');\n });\n\n });\n\n describe('#select', () => {\n it('should select all apps with default parameters', async () => {\n const mockRows = [\n createMockAppRow({ id: 1, uid: 'app-1', name: 'app-one' }),\n createMockAppRow({ id: 2, uid: 'app-2', name: 'app-two' }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {});\n\n expect(mockDb.read).toHaveBeenCalledTimes(1);\n expect(mockDb.read).toHaveBeenCalledWith(\n expect.not.stringContaining('WHERE'),\n [],\n );\n expect(result).toHaveLength(2);\n expect(result[0].uid).toBe('app-1');\n expect(result[1].uid).toBe('app-2');\n });\n\n it('should filter by user-can-edit predicate', async () => {\n const mockUser = { id: 42 };\n Context.get.mockReturnValue(mockUser);\n\n const mockRows = [createMockAppRow()];\n mockDb.read.mockResolvedValue(mockRows);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {\n predicate: ['user-can-edit'],\n });\n\n expect(mockDb.read).toHaveBeenCalledWith(\n expect.stringContaining('WHERE apps.owner_user_id=?'),\n [42],\n );\n expect(result).toHaveLength(1);\n });\n\n it('should throw error when predicate is not an array', async () => {\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.select.call(appService, { predicate: 'invalid' })).rejects.toThrow('predicate must be an array');\n });\n\n it('should correctly coerce boolean fields for all selected apps', async () => {\n const mockRows = [\n createMockAppRow({\n id: 1,\n approved_for_listing: 1,\n godmode: 0,\n }),\n createMockAppRow({\n id: 2,\n approved_for_listing: '0',\n godmode: '1',\n }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {});\n\n expect(result[0].approved_for_listing).toBe(true);\n expect(result[0].godmode).toBe(false);\n expect(result[1].approved_for_listing).toBe(false);\n expect(result[1].godmode).toBe(true);\n });\n\n it('should parse filetypes for all selected apps', async () => {\n const mockRows = [\n createMockAppRow({ id: 1, filetypes: '[\".txt\"]' }),\n createMockAppRow({ id: 2, filetypes: '[\".pdf\", \".doc\"]' }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {});\n\n expect(result[0].filetype_associations).toEqual(['txt']);\n expect(result[1].filetype_associations).toEqual(['pdf', 'doc']);\n });\n\n it('should fetch icons with size for all apps when icon_size is provided', async () => {\n const mockRows = [\n createMockAppRow({ id: 1, uid: 'app-1', icon: 'icon1.png' }),\n createMockAppRow({ id: 2, uid: 'app-2', icon: 'icon2.png' }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const mockIconService = {\n getAppIconPath: vi.fn().mockImplementation(({ appUid, size }) => `/app-icon/${appUid}/${size}`),\n };\n\n appService.context = {\n get: vi.fn().mockImplementation((key) => {\n if ( key === 'services' ) {\n return {\n get: vi.fn().mockImplementation((name) => {\n if ( name === 'app-icon' ) return mockIconService;\n return null;\n }),\n };\n }\n return null;\n }),\n };\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {\n params: { icon_size: 32 },\n });\n\n expect(mockIconService.getAppIconPath).toHaveBeenCalledTimes(2);\n expect(result[0].icon).toBe('/app-icon/app-1/32');\n expect(result[1].icon).toBe('/app-icon/app-2/32');\n });\n\n it('should only route base64 icons through app-icon endpoint when icon_size is not provided', async () => {\n const mockRows = [\n createMockAppRow({\n id: 1,\n uid: 'app-1',\n icon: 'data:image/png;base64,abc123',\n icon_is_base64: 1,\n }),\n createMockAppRow({\n id: 2,\n uid: 'app-2',\n icon: 'https://puter-app-icons.puter.site/app-2-128.png',\n icon_is_base64: 0,\n }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const mockIconService = {\n getAppIconPath: vi.fn().mockImplementation(({ appUid }) => `/app-icon/${appUid}/128`),\n };\n\n appService.context = {\n get: vi.fn().mockImplementation((key) => {\n if ( key === 'services' ) {\n return {\n get: vi.fn().mockImplementation((name) => {\n if ( name === 'app-icon' ) return mockIconService;\n return null;\n }),\n };\n }\n return null;\n }),\n };\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {});\n\n expect(mockIconService.getAppIconPath).toHaveBeenCalledTimes(1);\n expect(result[0].icon).toBe('/app-icon/app-1/128');\n expect(result[1].icon).toBe('https://puter-app-icons.puter.site/app-2-128.png');\n });\n\n it('should return empty array when no apps exist', async () => {\n mockDb.read.mockResolvedValue([]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {});\n\n expect(result).toEqual([]);\n });\n\n it('should have owner parameter for all selected apps', async () => {\n const mockRows = [\n createMockAppRow({\n id: 1,\n owner_user_username: 'user1',\n owner_user_uuid: 'uuid-1',\n }),\n createMockAppRow({\n id: 2,\n owner_user_username: 'user2',\n owner_user_uuid: 'uuid-2',\n }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.select.call(appService, {});\n\n expect(result[0].owner).toEqual({\n username: 'user1',\n uuid: 'uuid-1',\n });\n expect(result[1].owner).toEqual({\n username: 'user2',\n uuid: 'uuid-2',\n });\n });\n\n it('should handle filetypes that are not strings', async () => {\n const mockRows = [\n createMockAppRow({ id: 1, filetypes: '[\".txt\", 123]' }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.select.call(appService, {})).rejects.toThrow(\n 'expected filetypesAsJSON[1] to be a string',\n );\n });\n\n it('should handle malformed filetypes JSON', async () => {\n const mockRows = [\n createMockAppRow({ id: 1, filetypes: 'not valid json' }),\n ];\n mockDb.read.mockResolvedValue(mockRows);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.select.call(appService, {})).rejects.toThrow(\n 'failed to get app filetype associations',\n );\n });\n\n it('should not require dialect-specific JSON aggregation for app selection', async () => {\n mockDb.read.mockResolvedValue([]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.select.call(appService, {});\n\n expect(mockDb.case).not.toHaveBeenCalled();\n });\n });\n\n describe('#build_complex_id_where (via #read)', () => {\n it('should accept \"name\" as a valid redundant identifier', async () => {\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.read.call(appService, { id: { name: 'test' } });\n\n expect(mockDb.read).toHaveBeenCalledWith(\n expect.stringContaining('apps.name = ?'),\n ['test'],\n );\n });\n\n it('should reject identifiers not in REDUNDANT_IDENTIFIERS', async () => {\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.read.call(appService, { id: { title: 'test' } })).rejects.toThrow('Invalid complex id keys: title');\n });\n });\n\n describe('#create', () => {\n it('should create an app with valid input', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n // Mock the read after insert\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [];\n }\n return [createMockAppRow({\n uid: expect.stringContaining('app-'),\n name: 'new-app',\n title: 'New App',\n index_url: 'https://example.com/new',\n })];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'new-app',\n title: 'New App',\n index_url: 'https://example.com/new',\n },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('INSERT INTO apps'),\n expect.arrayContaining(['new-app', 'New App', 'https://example.com/new']),\n );\n });\n\n it('should throw forbidden for non-user actors', async () => {\n // Mock an invalid actor type\n Context.get.mockImplementation((key) => {\n if ( key === 'actor' ) return { type: {} };\n return null;\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n },\n })).rejects.toThrow();\n });\n\n it('should throw field_missing when name is not provided', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n title: 'Test',\n index_url: 'https://example.com',\n },\n })).rejects.toThrow();\n });\n\n it('should throw field_missing when title is not provided', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n index_url: 'https://example.com',\n },\n })).rejects.toThrow();\n });\n\n it('should throw field_missing when index_url is not provided', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n },\n })).rejects.toThrow();\n });\n\n it('should remove protected fields from input', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n last_review: '2024-01-01', // protected field\n },\n });\n\n // The INSERT should not include last_review\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('INSERT INTO apps'),\n expect.not.arrayContaining(['2024-01-01']),\n );\n });\n\n it('should remove read_only fields from input', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n approved_for_listing: true, // read_only field\n godmode: true, // read_only field\n is_private: true, // read_only field\n },\n });\n\n // These fields should not appear in the INSERT\n const writeCall = mockDbWrite.write.mock.calls[0];\n expect(writeCall[0]).not.toContain('approved_for_listing');\n expect(writeCall[0]).not.toContain('godmode');\n expect(writeCall[0]).not.toContain('is_private');\n });\n\n it('should handle name conflict with dedupe_name option', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n // First check returns true (name exists), second returns false\n app_name_exists\n .mockResolvedValueOnce(true) // 'new-app' exists\n .mockResolvedValueOnce(false); // 'new-app-1' doesn't exist\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'new-app',\n title: 'New App',\n index_url: 'https://example.com',\n },\n options: { dedupe_name: true },\n });\n\n // Should have inserted with deduped name\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('INSERT INTO apps'),\n expect.arrayContaining(['new-app-1']),\n );\n });\n\n it('should throw error when name conflict without dedupe_name', async () => {\n setupContextForWrite(createMockUserActor(1));\n app_name_exists.mockResolvedValue(true);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'existing-app',\n title: 'Test',\n index_url: 'https://example.com',\n },\n })).rejects.toThrow();\n });\n\n it('should allow equivalent index_url already in use on create for non-hosted origins', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 999,\n uid: 'app-existing-uid',\n owner_user_id: 1,\n index_url: 'https://example.com/',\n }];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'new-app',\n title: 'New App',\n index_url: 'https://example.com/index.html',\n },\n })).resolves.toBeDefined();\n });\n\n it('should allow duplicate dev-center placeholder index_url on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 999,\n uid: 'app-existing-placeholder',\n owner_user_id: 1,\n index_url: 'https://dev-center.puter.com/coming-soon.html',\n }];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'new-app',\n title: 'New App',\n index_url: 'https://dev-center.puter.com/coming-soon.html',\n },\n })).resolves.toBeDefined();\n });\n\n it('should join existing hosted app when index_url is owned and already used', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 999,\n uid: 'app-existing-hosted',\n owner_user_id: null,\n index_url: 'https://mysite.puter.site',\n }];\n }\n return [createMockAppRow({\n id: 999,\n uid: 'app-existing-hosted',\n name: 'existing-hosted-app',\n title: 'Existing Hosted App',\n index_url: 'https://mysite.puter.site',\n owner_user_id: 1,\n })];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const joined = await crudQ.create.call(appService, {\n object: {\n name: 'joined-hosted-app',\n title: 'Joined Hosted App',\n index_url: 'https://mysite.puter.site',\n },\n });\n\n expect(joined.uid).toBe('app-existing-hosted');\n expect(mockDbWrite.write).not.toHaveBeenCalledWith(\n expect.stringContaining('INSERT INTO apps'),\n expect.any(Array),\n );\n expect(mockKvStoreService.set).not.toHaveBeenCalled();\n });\n\n it('should throw when hosted index_url is already in use by another owner on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 999,\n uid: 'app-existing-hosted',\n owner_user_id: 2,\n index_url: 'https://mysite.puter.site',\n }];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'new-app',\n title: 'New App',\n index_url: 'https://mysite.puter.site',\n },\n })).rejects.toMatchObject({\n fields: {\n code: 'app_index_url_already_in_use',\n },\n });\n });\n\n it('should set app_owner when actor is AppUnderUserActorType', async () => {\n setupContextForWrite(createMockAppUnderUserActor(1, 100));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n },\n });\n\n // Should include app_owner in the INSERT\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('app_owner'),\n expect.arrayContaining([100]),\n );\n });\n\n it('should emit app.new-icon event when icon is provided', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: 'data:image/png;base64,abc123',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n data_url: 'data:image/png;base64,abc123',\n }),\n );\n });\n\n it('should accept raw base64 icon and normalize to data URL on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const rawBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ';\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: rawBase64,\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n data_url: `data:image/png;base64,${rawBase64}`,\n }),\n );\n });\n\n it('should migrate relative app-icon endpoint path to absolute URL on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n validate_url.mockImplementation((_value, { key }) => {\n if ( key === 'icon' ) {\n throw new Error('icon should not be validated as a URL');\n }\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: '/app-icon/app-uid-123/64',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n data_url: 'https://api.puter.localhost/app-icon/app-uid-123',\n }),\n );\n expect(validate_url).toHaveBeenCalledWith('https://example.com', expect.objectContaining({ key: 'index_url' }));\n });\n\n it('should reject object icon payloads on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: { url: '/app-icon/app-uid-123/64' },\n },\n })).rejects.toMatchObject({\n fields: { code: 'field_invalid', key: 'icon' },\n });\n });\n\n it('should allow empty icon string on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: '',\n },\n });\n\n expect(mockEventService.emit).not.toHaveBeenCalledWith('app.new-icon', expect.anything());\n });\n\n it('should migrate legacy app-icons host URL to app-icon endpoint URL on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: 'https://puter-app-icons.puter.site/app-uid-123-64.png',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n data_url: 'https://api.puter.localhost/app-icon/app-uid-123',\n }),\n );\n });\n\n it('should allow absolute app-icon endpoint URL on API origin', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: 'https://api.puter.localhost/app-icon/app-uid-123/64',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n data_url: 'https://api.puter.localhost/app-icon/app-uid-123',\n }),\n );\n });\n\n it('should reject foreign absolute app-icon endpoint URL on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: 'https://evil.example/app-icon/app-uid-123/64',\n },\n })).rejects.toMatchObject({\n fields: { code: 'field_invalid', key: 'icon' },\n });\n });\n\n it('should reject non app-icon URL icon on create', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n icon: 'https://example.com/webhook',\n },\n })).rejects.toMatchObject({\n fields: { code: 'field_invalid', key: 'icon' },\n });\n });\n\n it('should handle filetype_associations', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n filetype_associations: ['txt', 'pdf'],\n },\n });\n\n // Should have three write calls: INSERT app, DELETE old associations, INSERT new associations\n // (DELETE is called even for create since #update_filetype_associations always clears first)\n expect(mockDbWrite.write).toHaveBeenCalledTimes(3);\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('DELETE FROM app_filetype_association'),\n [1],\n );\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('INSERT INTO app_filetype_association'),\n expect.arrayContaining([1, 'txt', 1, 'pdf']),\n );\n });\n\n it('should call validate_string for name and title', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test Title',\n index_url: 'https://example.com',\n },\n });\n\n expect(validate_string).toHaveBeenCalledWith('test-app', expect.objectContaining({ key: 'name' }));\n expect(validate_string).toHaveBeenCalledWith('Test Title', expect.objectContaining({ key: 'title' }));\n });\n\n it('should call validate_url for index_url', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com/app',\n },\n });\n\n expect(validate_url).toHaveBeenCalledWith('https://example.com/app', expect.objectContaining({ key: 'index_url' }));\n });\n\n it('should generate a UID with app- prefix', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.create.call(appService, {\n object: {\n name: 'test-app',\n title: 'Test',\n index_url: 'https://example.com',\n },\n });\n\n const writeCall = mockDbWrite.write.mock.calls[0];\n const values = writeCall[1];\n const uidValue = values[0]; // uid is first value\n expect(uidValue).toMatch(/^app-[0-9a-f-]{36}$/);\n });\n });\n\n describe('#update', () => {\n beforeEach(() => {\n // Default: return an existing app for updates\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n })]);\n });\n\n it('should update an app with valid input', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', title: 'Updated Title' },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['Updated Title', 'app-uid-123']),\n );\n });\n\n it('should throw entity_not_found when app does not exist', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.update.call(appService, {\n object: { uid: 'nonexistent-uid', title: 'Test' },\n })).rejects.toThrow();\n });\n\n it('should throw forbidden when user does not own the app', async () => {\n // User 2 trying to update app owned by user 1\n setupContextForWrite(createMockUserActor(2), { id: 2 });\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n })]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', title: 'Hacked Title' },\n })).rejects.toThrow();\n });\n\n it('should allow update when user has write-all-owners permission', async () => {\n setupContextForWrite(createMockUserActor(2), { id: 2 });\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n })]);\n mockPermissionService.check.mockResolvedValue(true);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', title: 'Admin Update' },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['Admin Update']),\n );\n });\n\n it('should remove protected fields from update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n title: 'Updated',\n last_review: '2024-12-01', // protected field\n },\n });\n\n const writeCall = mockDbWrite.write.mock.calls[0];\n expect(writeCall[0]).not.toContain('last_review');\n });\n\n it('should remove read_only fields from update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n title: 'Updated',\n approved_for_listing: true,\n godmode: true,\n is_private: true,\n },\n });\n\n const writeCall = mockDbWrite.write.mock.calls[0];\n expect(writeCall[0]).not.toContain('approved_for_listing');\n expect(writeCall[0]).not.toContain('godmode');\n expect(writeCall[0]).not.toContain('is_private');\n });\n\n it('should handle name change with conflict', async () => {\n setupContextForWrite(createMockUserActor(1));\n app_name_exists.mockResolvedValue(true);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', name: 'taken-name' },\n })).rejects.toThrow();\n });\n\n it('should allow name change with dedupe_name option', async () => {\n setupContextForWrite(createMockUserActor(1));\n app_name_exists\n .mockResolvedValueOnce(true) // 'new-name' exists\n .mockResolvedValueOnce(false); // 'new-name-1' doesn't exist\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', name: 'new-name' },\n options: { dedupe_name: true },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['new-name-1']),\n );\n });\n\n it('should allow reclaiming old app name', async () => {\n setupContextForWrite(createMockUserActor(1));\n app_name_exists.mockResolvedValue(true);\n mockOldAppNameService.check_app_name.mockResolvedValue({\n id: 99,\n app_uid: 'app-uid-123', // Same app\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', name: 'old-name' },\n });\n\n expect(mockOldAppNameService.remove_name).toHaveBeenCalledWith(99);\n expect(mockDbWrite.write).toHaveBeenCalled();\n });\n\n it('should not update name if unchanged', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', name: 'test-app' }, // Same as existing\n });\n\n // Should only have the read for ID, no name in update\n const writeCall = mockDbWrite.write.mock.calls.find(call => call[0].includes('UPDATE'));\n if ( writeCall ) {\n expect(writeCall[1]).not.toContain('test-app');\n }\n });\n\n it('should emit app.new-icon event when icon changes', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: 'data:image/png;base64,newicon',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n app_uid: 'app-uid-123',\n data_url: 'data:image/png;base64,newicon',\n }),\n );\n });\n\n it('should accept raw base64 icon and normalize to data URL on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const rawBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ';\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: rawBase64,\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n app_uid: 'app-uid-123',\n data_url: `data:image/png;base64,${rawBase64}`,\n }),\n );\n });\n\n it('should migrate relative app-icon endpoint path to absolute URL on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n validate_url.mockImplementation((_value, { key }) => {\n if ( key === 'icon' ) {\n throw new Error('icon should not be validated as a URL');\n }\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: '/app-icon/app-uid-123/64',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n app_uid: 'app-uid-123',\n data_url: 'https://api.puter.localhost/app-icon/app-uid-123',\n }),\n );\n });\n\n it('should reject object icon payloads on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: { url: '/app-icon/app-uid-123/64' },\n },\n })).rejects.toMatchObject({\n fields: { code: 'field_invalid', key: 'icon' },\n });\n });\n\n it('should allow empty icon string on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: '',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n app_uid: 'app-uid-123',\n data_url: '',\n }),\n );\n });\n\n it('should migrate legacy app-icons host URL to app-icon endpoint URL on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: 'https://puter-app-icons.puter.site/app-uid-123-64.png',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n app_uid: 'app-uid-123',\n data_url: 'https://api.puter.localhost/app-icon/app-uid-123',\n }),\n );\n });\n\n it('should allow absolute app-icon endpoint URL on API origin when updating icon', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: 'https://api.puter.localhost/app-icon/app-uid-123/64',\n },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.new-icon',\n expect.objectContaining({\n app_uid: 'app-uid-123',\n data_url: 'https://api.puter.localhost/app-icon/app-uid-123',\n }),\n );\n });\n\n it('should reject foreign absolute app-icon endpoint URL on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: 'https://evil.example/app-icon/app-uid-123/64',\n },\n })).rejects.toMatchObject({\n fields: { code: 'field_invalid', key: 'icon' },\n });\n });\n\n it('should reject non app-icon URL icon on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n icon: 'https://example.com/webhook',\n },\n })).rejects.toMatchObject({\n fields: { code: 'field_invalid', key: 'icon' },\n });\n });\n\n it('should emit app.rename event when name changes', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', name: 'renamed-app' },\n });\n\n expect(mockEventService.emit).toHaveBeenCalledWith(\n 'app.rename',\n expect.objectContaining({\n app_uid: 'app-uid-123',\n new_name: 'renamed-app',\n old_name: 'test-app',\n }),\n );\n });\n\n it('should update filetype_associations', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n filetype_associations: ['doc', 'xls'],\n },\n });\n\n // Should delete old associations\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('DELETE FROM app_filetype_association'),\n [1],\n );\n\n // Should insert new associations\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('INSERT INTO app_filetype_association'),\n expect.arrayContaining([1, 'doc', 1, 'xls']),\n );\n });\n\n it('should validate fields when provided', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n name: 'updated-name',\n title: 'Updated Title',\n description: 'Updated description',\n index_url: 'https://updated.com',\n },\n });\n\n expect(validate_string).toHaveBeenCalledWith('updated-name', expect.objectContaining({ key: 'name' }));\n expect(validate_string).toHaveBeenCalledWith('Updated Title', expect.objectContaining({ key: 'title' }));\n expect(validate_string).toHaveBeenCalledWith('Updated description', expect.objectContaining({ key: 'description' }));\n expect(validate_url).toHaveBeenCalledWith('https://updated.com', expect.objectContaining({ key: 'index_url' }));\n });\n\n it('should check subdomain ownership when index_url changes to puter.site', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue(null);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n index_url: 'https://mysite.puter.site',\n },\n })).rejects.toThrow();\n });\n\n it('should allow index_url change when subdomain is owned', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n index_url: 'https://mysite.puter.site',\n },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['https://mysite.puter.site']),\n );\n });\n\n it('should allow index_url change when private hosted subdomain is owned', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n index_url: 'https://mysite.puter.dev',\n },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['https://mysite.puter.dev']),\n );\n });\n\n it('should allow equivalent index_url already in use on update for non-hosted origins', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 777,\n uid: 'app-conflict-uid',\n owner_user_id: 2,\n index_url: 'https://updated.com/',\n }];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n index_url: 'https://updated.com/index.html',\n },\n })).resolves.toBeDefined();\n });\n\n it('should allow duplicate dev-center placeholder index_url on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 777,\n uid: 'app-existing-placeholder',\n owner_user_id: 1,\n index_url: 'https://dev-center.puter.com/coming-soon.html',\n }];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n index_url: 'https://dev-center.puter.com/coming-soon.html',\n },\n })).resolves.toBeDefined();\n });\n\n it('should join existing unowned hosted app when index_url is already in use on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });\n let readCallCount = 0;\n mockDb.read.mockImplementation(async (query, params) => {\n readCallCount++;\n if ( readCallCount > 100 ) {\n throw new Error(`excessive mockDb.read calls in join test: ${String(query)} :: ${JSON.stringify(params)}`);\n }\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n if ( Array.isArray(params) && params[params.length - 1] === 777 ) {\n // Mirrors SQL `AND id != ?` behavior during join follow-up updates.\n return [];\n }\n return [{\n id: 777,\n uid: 'app-conflict-uid',\n owner_user_id: null,\n index_url: 'https://mysite.puter.site/',\n }];\n }\n if ( Array.isArray(params) && params[0] === 'app-conflict-uid' ) {\n return [createMockAppRow({\n id: 777,\n uid: 'app-conflict-uid',\n name: 'existing-hosted-app',\n title: 'Existing Hosted App',\n index_url: 'https://mysite.puter.site/',\n owner_user_id: 1,\n })];\n }\n return [createMockAppRow({\n id: 1,\n uid: 'app-uid-123',\n name: 'updating-app',\n title: 'Updating App',\n index_url: 'https://other.puter.site',\n owner_user_id: 1,\n })];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n title: 'Joined Update Title',\n index_url: 'https://mysite.puter.site/index.html',\n },\n });\n\n expect(result.uid).toBe('app-conflict-uid');\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['Joined Update Title', 'app-conflict-uid']),\n );\n expect(mockAppInformationService.delete_app).toHaveBeenCalledWith(\n 'app-uid-123',\n undefined,\n { preserveCanonicalUidAlias: true },\n );\n expect(mockKvStoreService.set).toHaveBeenCalledWith(expect.objectContaining({\n key: 'app:canonicalUidAlias:app-uid-123',\n value: 'app-conflict-uid',\n }));\n });\n\n it('should throw when owned hosted index_url is already in use on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 777,\n uid: 'app-conflict-uid',\n owner_user_id: 1,\n index_url: 'https://mysite.puter.site/',\n }];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n index_url: 'https://mysite.puter.site/index.html',\n },\n })).rejects.toMatchObject({\n fields: {\n code: 'app_index_url_already_in_use',\n },\n });\n });\n\n it('should throw when equivalent hosted index_url is already in use on update', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });\n mockDb.read.mockImplementation(async (query) => {\n if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {\n return [{\n id: 777,\n uid: 'app-conflict-uid',\n owner_user_id: 2,\n index_url: 'https://mysite.puter.site/',\n }];\n }\n return [createMockAppRow()];\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await expect(crudQ.update.call(appService, {\n object: {\n uid: 'app-uid-123',\n index_url: 'https://mysite.puter.site/index.html',\n },\n })).rejects.toMatchObject({\n fields: {\n code: 'app_index_url_already_in_use',\n },\n });\n });\n\n it('should throw forbidden when app actor does not own the entity (AppLimitedES behavior)', async () => {\n // App actor trying to update an app it didn't create\n setupContextForWrite(createMockAppUnderUserActor(1, 999));\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n app_owner_uid: 'different-app-uid',\n })]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', title: 'Hacked Title' },\n })).rejects.toThrow();\n });\n\n it('should allow app actor to update entity it owns (AppLimitedES behavior)', async () => {\n // App actor updating an app it created\n const actor = createMockAppUnderUserActor(1, 100);\n actor.type.app.uid = 'creator-app-uid';\n setupContextForWrite(actor);\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n app_owner_uid: 'creator-app-uid',\n })]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', title: 'Updated by App' },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['Updated by App']),\n );\n });\n\n it('should allow app actor with write permission to update any entity (AppLimitedES behavior)', async () => {\n setupContextForWrite(createMockAppUnderUserActor(1, 999));\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n app_owner_uid: 'different-app-uid',\n })]);\n // Grant write permission\n mockPermissionService.check.mockResolvedValue(true);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.update.call(appService, {\n object: { uid: 'app-uid-123', title: 'Admin Update' },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.arrayContaining(['Admin Update']),\n );\n });\n });\n\n describe('#upsert', () => {\n it('should call create when entity does not exist', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.upsert.call(appService, {\n object: {\n name: 'new-app',\n title: 'New App',\n index_url: 'https://example.com',\n },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('INSERT INTO apps'),\n expect.any(Array),\n );\n });\n\n it('should call update when entity exists', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n // Read returns existing entity\n mockDb.read.mockResolvedValue([createMockAppRow()]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.upsert.call(appService, {\n object: { uid: 'app-uid-123', title: 'Updated Title' },\n });\n\n expect(mockDbWrite.write).toHaveBeenCalledWith(\n expect.stringContaining('UPDATE apps SET'),\n expect.any(Array),\n );\n });\n });\n\n describe('#delete', () => {\n beforeEach(() => {\n // Mock app-information service\n mockAppInformationService = {\n delete_app: vi.fn().mockResolvedValue(undefined),\n };\n\n // Update mockServices to include app-information\n mockServices.get.mockImplementation((serviceName) => {\n if ( serviceName === 'database' ) {\n return {\n get: vi.fn().mockImplementation((mode) => {\n if ( mode === 'write' ) return mockDbWrite;\n return mockDb;\n }),\n };\n }\n if ( serviceName === 'event' ) return mockEventService;\n if ( serviceName === 'permission' ) return mockPermissionService;\n if ( serviceName === 'puter-site' ) return mockPuterSiteService;\n if ( serviceName === 'old-app-name' ) return mockOldAppNameService;\n if ( serviceName === 'app-information' ) return mockAppInformationService;\n if ( serviceName === 'puter-kvstore' ) return mockKvStoreService;\n if ( serviceName === 'su' ) return mockSuService;\n return null;\n });\n\n // Default: return an existing app for deletes\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n })]);\n });\n\n it('should delete an app by uid', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });\n\n expect(mockAppInformationService.delete_app).toHaveBeenCalledWith('app-uid-123');\n expect(result.success).toBe(true);\n expect(result.uid).toBe('app-uid-123');\n });\n\n it('should delete an app by complex id (name)', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.delete.call(appService, { id: { name: 'test-app' } });\n\n expect(mockAppInformationService.delete_app).toHaveBeenCalledWith('app-uid-123');\n expect(result.success).toBe(true);\n });\n\n it('should throw entity_not_found when app does not exist', async () => {\n setupContextForWrite(createMockUserActor(1));\n mockDb.read.mockResolvedValue([]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.delete.call(appService, { uid: 'nonexistent-uid' }))\n .rejects.toThrow();\n });\n\n it('should throw forbidden for non-user actors', async () => {\n Context.get.mockImplementation((key) => {\n if ( key === 'actor' ) return { type: {} };\n return null;\n });\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' }))\n .rejects.toThrow();\n });\n\n it('should throw forbidden when user does not own the app', async () => {\n // User 2 trying to delete app owned by user 1\n setupContextForWrite(createMockUserActor(2), { id: 2 });\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n })]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' }))\n .rejects.toThrow();\n });\n\n it('should allow delete when user has write-all-owners permission', async () => {\n setupContextForWrite(createMockUserActor(2), { id: 2 });\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n })]);\n mockPermissionService.check.mockResolvedValue(true);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });\n\n expect(mockAppInformationService.delete_app).toHaveBeenCalled();\n expect(result.success).toBe(true);\n });\n\n it('should invalidate app cache after delete', async () => {\n setupContextForWrite(createMockUserActor(1));\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n await crudQ.delete.call(appService, { uid: 'app-uid-123' });\n });\n\n it('should throw forbidden when app actor does not own the entity', async () => {\n // App actor trying to delete an app it didn't create\n setupContextForWrite(createMockAppUnderUserActor(1, 999));\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n app_owner_uid: 'different-app-uid',\n })]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n\n await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' }))\n .rejects.toThrow();\n });\n\n it('should allow app actor to delete entity it owns', async () => {\n // App actor deleting an app it created\n const actor = createMockAppUnderUserActor(1, 100);\n actor.type.app.uid = 'creator-app-uid';\n setupContextForWrite(actor);\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n app_owner_uid: 'creator-app-uid',\n })]);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });\n\n expect(mockAppInformationService.delete_app).toHaveBeenCalled();\n expect(result.success).toBe(true);\n });\n\n it('should allow app actor with write permission to delete any entity', async () => {\n setupContextForWrite(createMockAppUnderUserActor(1, 999));\n mockDb.read.mockResolvedValue([createMockAppRow({\n owner_user_id: 1,\n app_owner_uid: 'different-app-uid',\n })]);\n // Grant write permission\n mockPermissionService.check.mockResolvedValue(true);\n\n const crudQ = AppService.IMPLEMENTS['crud-q'];\n const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });\n\n expect(mockAppInformationService.delete_app).toHaveBeenCalled();\n expect(result.success).toBe(true);\n });\n });\n});\n"
},
{
"path": "src/backend/src/modules/data-access/DEV.md",
"content": "## Development for `data-access` module\n\nThis document will contain notes, documentation, and snippets written\nwhile developing the `data-access` module replacements for what was\nformerly handled by EntityStoreService and OM (Object Mapping).\n\n### App List Test Code\n\nThis code is used to test listing apps with one of the available\nCRUD-implementing drivers.\n\n```javascript\nawait (async () => {\n const resp = await fetch('http://api.puter.localhost:4100/drivers/call', {\n method: 'POST',\n headers: {\n Authorization: `Bearer ${puter.authToken}`,\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({\n args: { predicate: ['user-can-edit'] },\n driver: 'es:app',\n interface: 'puter-apps',\n method: 'select',\n }),\n })\n return (await resp.json()).result;\n})();\n```\n\n### AI-Generated Compare Function\n\nI asked an LLM to find me a javascript object compare function\nthat I can paste in developer tools and it started generating\none from scratch. To my surprise it worked just fine, so I'm pasting\nthis here for the time being for convenience:\n\n```javascript\n(() => {\n // Deep compare + diff reporter for DevTools (no deps)\n // Usage:\n // const r = deepCompare(a, b);\n // console.log(r.pass, r.message);\n // r.print(); // pretty console output\n // Options:\n // deepCompare(a,b,{ showSame:false, maxDiffs:200, sortKeys:true })\n\n function deepCompare(a, b, opts = {}) {\n const options = {\n showSame: false, // include \"same\" entries in the diff list\n maxDiffs: 200, // cap diffs so you don't nuke your console\n sortKeys: true, // stable key ordering when iterating plain objects\n ...opts,\n };\n\n const diffs = [];\n const seenPairs = new WeakMap(); // a -> WeakMap(b -> true)\n\n const isObjectLike = (v) => v !== null && (typeof v === \"object\" || typeof v === \"function\");\n const tagOf = (v) => Object.prototype.toString.call(v); // \"[object X]\"\n const isPlainObject = (v) => {\n if (tagOf(v) !== \"[object Object]\") return false;\n const proto = Object.getPrototypeOf(v);\n return proto === Object.prototype || proto === null;\n };\n\n const typeLabel = (v) => {\n if (v === null) return \"null\";\n const t = typeof v;\n if (t !== \"object\") return t;\n return tagOf(v).slice(8, -1);\n };\n\n const formatVal = (v) => {\n // Safe-ish inline formatter for messages (keeps things short)\n try {\n if (typeof v === \"string\") return JSON.stringify(v.length > 120 ? v.slice(0, 117) + \"…\" : v);\n if (typeof v === \"number\" && Object.is(v, -0)) return \"-0\";\n if (typeof v === \"bigint\") return `${v}n`;\n if (typeof v === \"symbol\") return v.toString();\n if (typeof v === \"function\") return `[Function ${v.name || \"anonymous\"}]`;\n if (v instanceof Date) return isNaN(v.getTime()) ? \"Invalid Date\" : `Date(${v.toISOString()})`;\n if (v instanceof RegExp) return v.toString();\n if (v instanceof Map) return `Map(${v.size})`;\n if (v instanceof Set) return `Set(${v.size})`;\n if (ArrayBuffer.isView(v) && !(v instanceof DataView)) return `${v.constructor.name}(${v.length})`;\n if (v instanceof ArrayBuffer) return `ArrayBuffer(${v.byteLength})`;\n if (v && v.constructor && v.constructor !== Object) return `${v.constructor.name}{…}`;\n if (Array.isArray(v)) return `Array(${v.length})`;\n if (isPlainObject(v)) return \"Object{…}\";\n return `${typeLabel(v)}{…}`;\n } catch {\n return \"[Unformattable]\";\n }\n };\n\n const pathToString = (path) => {\n if (!path.length) return \"(root)\";\n let s = \"\";\n for (const p of path) {\n if (typeof p === \"number\") s += `[${p}]`;\n else if (typeof p === \"string\") {\n if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(p)) s += (s ? \".\" : \"\") + p;\n else s += `[${JSON.stringify(p)}]`;\n } else if (typeof p === \"symbol\") s += `[${p.toString()}]`;\n else s += `[${String(p)}]`;\n }\n return s;\n };\n\n const pushDiff = (kind, path, left, right, extra) => {\n if (diffs.length >= options.maxDiffs) return;\n diffs.push({\n kind, // \"type\" | \"value\" | \"missing-left\" | \"missing-right\" | \"prototype\" | \"keys\" | ...\n path: [...path],\n left,\n right,\n extra,\n });\n };\n\n const markSeen = (x, y) => {\n if (!isObjectLike(x) || !isObjectLike(y)) return false;\n let inner = seenPairs.get(x);\n if (!inner) {\n inner = new WeakMap();\n seenPairs.set(x, inner);\n }\n if (inner.get(y)) return true;\n inner.set(y, true);\n return false;\n };\n\n const sameValueZero = (x, y) => Object.is(x, y); // handles NaN, -0\n\n const compareArrays = (x, y, path) => {\n if (x.length !== y.length) pushDiff(\"value\", [...path, \"length\"], x.length, y.length, \"array length mismatch\");\n const n = Math.max(x.length, y.length);\n for (let i = 0; i < n; i++) {\n if (i >= x.length) pushDiff(\"missing-left\", [...path, i], undefined, y[i], \"missing index in left\");\n else if (i >= y.length) pushDiff(\"missing-right\", [...path, i], x[i], undefined, \"missing index in right\");\n else walk(x[i], y[i], [...path, i]);\n if (diffs.length >= options.maxDiffs) return;\n }\n };\n\n const compareTypedArrays = (x, y, path) => {\n if (x.constructor !== y.constructor) {\n pushDiff(\"type\", path, x.constructor?.name, y.constructor?.name, \"typed array class mismatch\");\n return;\n }\n if (x.length !== y.length) pushDiff(\"value\", [...path, \"length\"], x.length, y.length, \"typed array length mismatch\");\n const n = Math.min(x.length, y.length);\n for (let i = 0; i < n; i++) {\n if (!sameValueZero(x[i], y[i])) pushDiff(\"value\", [...path, i], x[i], y[i], \"typed array element mismatch\");\n if (diffs.length >= options.maxDiffs) return;\n }\n };\n\n const compareArrayBuffer = (x, y, path) => {\n if (x.byteLength !== y.byteLength) {\n pushDiff(\"value\", [...path, \"byteLength\"], x.byteLength, y.byteLength, \"ArrayBuffer byteLength mismatch\");\n return;\n }\n const a8 = new Uint8Array(x);\n const b8 = new Uint8Array(y);\n for (let i = 0; i < a8.length; i++) {\n if (a8[i] !== b8[i]) {\n pushDiff(\"value\", [...path, i], a8[i], b8[i], \"ArrayBuffer byte mismatch\");\n if (diffs.length >= options.maxDiffs) return;\n }\n }\n };\n\n const compareDates = (x, y, path) => {\n const tx = x.getTime();\n const ty = y.getTime();\n if (!sameValueZero(tx, ty)) pushDiff(\"value\", path, x, y, \"Date mismatch\");\n };\n\n const compareRegex = (x, y, path) => {\n if (x.source !== y.source || x.flags !== y.flags) pushDiff(\"value\", path, x, y, \"RegExp mismatch\");\n };\n\n const compareMaps = (x, y, path) => {\n if (x.size !== y.size) pushDiff(\"value\", [...path, \"size\"], x.size, y.size, \"Map size mismatch\");\n\n // Map key equality is identity-based; here we:\n // 1) try direct key lookup for primitive keys\n // 2) for object keys, we require the *same object reference* exists as key in the other map\n // (test frameworks do similar unless they do expensive key deep-matching)\n for (const [k, xv] of x.entries()) {\n if (!y.has(k)) {\n pushDiff(\"missing-right\", [...path, `MapKey(${formatVal(k)})`], xv, undefined, \"Map missing key on right\");\n continue;\n }\n walk(xv, y.get(k), [...path, `MapKey(${formatVal(k)})`]);\n if (diffs.length >= options.maxDiffs) return;\n }\n\n for (const [k, yv] of y.entries()) {\n if (!x.has(k)) {\n pushDiff(\"missing-left\", [...path, `MapKey(${formatVal(k)})`], undefined, yv, \"Map missing key on left\");\n if (diffs.length >= options.maxDiffs) return;\n }\n }\n };\n\n const compareSets = (x, y, path) => {\n if (x.size !== y.size) pushDiff(\"value\", [...path, \"size\"], x.size, y.size, \"Set size mismatch\");\n\n // Same logic: membership is identity for object values.\n for (const v of x.values()) {\n if (!y.has(v)) pushDiff(\"missing-right\", [...path, `SetVal(${formatVal(v)})`], v, undefined, \"Set missing value on right\");\n if (diffs.length >= options.maxDiffs) return;\n }\n for (const v of y.values()) {\n if (!x.has(v)) pushDiff(\"missing-left\", [...path, `SetVal(${formatVal(v)})`], undefined, v, \"Set missing value on left\");\n if (diffs.length >= options.maxDiffs) return;\n }\n };\n\n const comparePlainObjects = (x, y, path) => {\n // Compare prototypes (handy when something is class instance vs plain object)\n const px = Object.getPrototypeOf(x);\n const py = Object.getPrototypeOf(y);\n if (px !== py) pushDiff(\"prototype\", path, px?.constructor?.name || px, py?.constructor?.name || py, \"Prototype mismatch\");\n\n const keysX = Reflect.ownKeys(x);\n const keysY = Reflect.ownKeys(y);\n\n const norm = (ks) => {\n // Sort only string keys for stability; keep symbols in original order\n if (!options.sortKeys) return ks;\n const str = ks.filter(k => typeof k === \"string\").sort();\n const sym = ks.filter(k => typeof k === \"symbol\");\n const numLike = []; // keep numeric-looking strings in numeric order if you want; leaving out to stay simple\n // We'll just do lexical sort for strings; okay for devtools output.\n return [...str, ...sym];\n };\n\n const kx = norm(keysX);\n const ky = norm(keysY);\n\n const setY = new Set(keysY);\n const setX = new Set(keysX);\n\n for (const k of kx) {\n if (!setY.has(k)) {\n pushDiff(\"missing-right\", [...path, k], x[k], undefined, \"Missing property on right\");\n } else {\n walk(x[k], y[k], [...path, k]);\n }\n if (diffs.length >= options.maxDiffs) return;\n }\n for (const k of ky) {\n if (!setX.has(k)) {\n pushDiff(\"missing-left\", [...path, k], undefined, y[k], \"Missing property on left\");\n if (diffs.length >= options.maxDiffs) return;\n }\n }\n };\n\n function walk(x, y, path) {\n if (diffs.length >= options.maxDiffs) return;\n\n if (sameValueZero(x, y)) {\n if (options.showSame) pushDiff(\"same\", path, x, y);\n return;\n }\n\n const tx = typeLabel(x);\n const ty = typeLabel(y);\n if (tx !== ty) {\n pushDiff(\"type\", path, tx, ty, \"Type mismatch\");\n return;\n }\n\n // Circular / repeated references\n if (markSeen(x, y)) return;\n\n // Per-type comparisons\n if (Array.isArray(x)) return compareArrays(x, y, path);\n\n if (ArrayBuffer.isView(x) && !(x instanceof DataView)) return compareTypedArrays(x, y, path);\n if (x instanceof ArrayBuffer) return compareArrayBuffer(x, y, path);\n\n if (x instanceof Date) return compareDates(x, y, path);\n if (x instanceof RegExp) return compareRegex(x, y, path);\n if (x instanceof Map) return compareMaps(x, y, path);\n if (x instanceof Set) return compareSets(x, y, path);\n\n // Functions: compare by reference already failed; treat as value mismatch\n if (typeof x === \"function\") {\n pushDiff(\"value\", path, x, y, \"Function reference mismatch\");\n return;\n }\n\n // Objects (including class instances): compare own keys + nested values.\n if (isObjectLike(x)) return comparePlainObjects(x, y, path);\n\n // Primitives (should have been caught by Object.is earlier)\n pushDiff(\"value\", path, x, y, \"Value mismatch\");\n }\n\n walk(a, b, []);\n\n const pass = diffs.length === 0;\n\n const message = pass\n ? \"✅ Values are deeply equal.\"\n : buildMessage(diffs, options);\n\n function buildMessage(diffs, options) {\n const lines = [];\n lines.push(`❌ Values differ (${diffs.length}${diffs.length >= options.maxDiffs ? \"+\" : \"\"} diff${diffs.length === 1 ? \"\" : \"s\"}):`);\n for (let i = 0; i < diffs.length; i++) {\n const d = diffs[i];\n const p = pathToString(d.path);\n const left = formatVal(d.left);\n const right = formatVal(d.right);\n const label = d.kind.padEnd(14, \" \");\n const extra = d.extra ? ` — ${d.extra}` : \"\";\n lines.push(`${String(i + 1).padStart(3, \" \")}. ${label} ${p}${extra}`);\n lines.push(` left : ${left}`);\n lines.push(` right: ${right}`);\n }\n if (diffs.length >= options.maxDiffs) {\n lines.push(`… (diffs capped at maxDiffs=${options.maxDiffs})`);\n }\n return lines.join(\"\\n\");\n }\n\n function print() {\n if (pass) {\n console.log(\"%c✅ deepCompare: PASS\", \"font-weight:bold\");\n return;\n }\n console.groupCollapsed(`%c❌ deepCompare: FAIL (${diffs.length}${diffs.length >= options.maxDiffs ? \"+\" : \"\"})`, \"font-weight:bold\");\n console.log(message);\n\n // Also log a structured table for quick scanning\n const table = diffs.map((d) => ({\n kind: d.kind,\n path: pathToString(d.path),\n left: formatVal(d.left),\n right: formatVal(d.right),\n note: d.extra || \"\",\n }));\n try { console.table(table); } catch {}\n console.groupEnd();\n }\n\n return { pass, diffs, message, print };\n }\n\n // Expose globally for DevTools convenience\n window.deepCompare = deepCompare;\n console.log(\"deepCompare installed. Usage: deepCompare(a,b).print()\");\n})();\n\n```"
},
{
"path": "src/backend/src/modules/data-access/DataAccessModule.js",
"content": "import { AdvancedBase } from '@heyputer/putility';\nimport AppService from './AppService.js';\n\nexport class DataAccessModule extends AdvancedBase {\n async install (context) {\n const services = context.get('services');\n\n services.registerService('app', AppService);\n }\n}\n"
},
{
"path": "src/backend/src/modules/data-access/lib/coercion.js",
"content": "// These utility functions describe how values stored in the database\n// are to be understood as their higher-level counterparts.\n\nimport { CoercionTypeError } from './error.js';\n\n/**\n * MySQL lets us store `1` (an integer) or `0` (also an integer) as\n * the closest parallel to a boolean \"true or false\" value.\n * Sqlite lets us store `\"1\"` (a string) or `0` (also a string) as\n * the closest parallel to a boolean \"true of false\" value.\n *\n * So we define a function here called `as_bool` that will make\n * `\"0\"` or `0` become `false`, and `\"1\"` or `1` become `true`.\n *\n * @param {any} value - The value to coerce to a boolean.\n * @returns {boolean} The coerced boolean value.\n */\nexport const as_bool = value => {\n if ( value === undefined ) return false;\n if ( value === 0 ) value = false;\n if ( value === 1 ) value = true;\n if ( value === '0' ) value = false;\n if ( value === '1' ) value = true;\n if ( typeof value !== 'boolean' ) {\n throw new CoercionTypeError({ expected: 'boolean', got: typeof value });\n }\n return value;\n};\n"
},
{
"path": "src/backend/src/modules/data-access/lib/error.js",
"content": "/**\n * Replaces `OMTypeError` from ES/OM implementation.\n * This might be removed or replaced in the future.\n */\nexport class CoercionTypeError extends Error {\n constructor ({ expected, got }) {\n const message = `expected ${expected}, got ${got}`;\n super(message);\n this.name = 'CoercionTypeError';\n }\n}\n"
},
{
"path": "src/backend/src/modules/data-access/lib/filter.js",
"content": "// These utility functions describe how to produce an object safe\n// for transfer that came from a \"raw\" object.\n\nexport const user_to_client = raw_user => {\n return {\n username: raw_user.username,\n // This `uuid` is not an internal-only ID.\n uuid: raw_user.uuid,\n };\n};\n"
},
{
"path": "src/backend/src/modules/data-access/lib/sqlutil.js",
"content": "/**\n * When columns are selected from a joined table and prefixed:\n *\n * SELECT joined_table.* AS joined_table_\n *\n * This function is able to extract the object from the result:\n *\n * extract_from_prefix(row, 'joined_table_') // columns of joined_table\n *\n * @param {*} row\n * @param {*} prefix\n */\nexport const extract_from_prefix = (row, prefix) => {\n const result = {};\n for ( const [key, value] of Object.entries(row) ) {\n if ( key.startsWith(prefix) ) {\n result[key.replace(prefix, '')] = value;\n }\n }\n return result;\n};\n"
},
{
"path": "src/backend/src/modules/data-access/lib/validation.js",
"content": "import validator from 'validator';\nimport APIError from '../../../api/APIError.js';\n\n/**\n * Validates a string value with optional maxlen and regex constraints.\n * @param {string} value - The value to validate\n * @param {object} meta - Metadata for the validation\n * @param {string} meta.key - The field name (for error messages)\n * @param {number} [meta.maxlen] - Maximum length allowed\n * @param {RegExp} [meta.regex] - Regex pattern the string must match\n */\nexport const validate_string = (value, { key, maxlen, regex }) => {\n if ( typeof value !== 'string' ) {\n throw APIError.create('field_invalid', null, { key });\n }\n if ( maxlen !== undefined && value.length > maxlen ) {\n throw APIError.create('field_too_long', null, { key, max_length: maxlen });\n }\n if ( regex !== undefined && !regex.test(value) ) {\n throw APIError.create('field_invalid', null, { key });\n }\n};\n\n/**\n * Validates an image-base64 value (data URL for images).\n * Checks for proper prefix and XSS characters.\n * @param {string} value - The value to validate\n * @param {object} meta - Metadata for the validation\n * @param {string} meta.key - The field name (for error messages)\n */\nexport const validate_image_base64 = (value, { key }) => {\n if ( typeof value !== 'string' ) {\n throw APIError.create('field_invalid', null, { key });\n }\n if ( ! value.startsWith('data:image/') ) {\n throw APIError.create('field_invalid', null, { key });\n }\n // XSS character check from image-base64 prop type\n const xss_chars = ['<', '>', '&', '\"', \"'\", '`'];\n if ( xss_chars.some(char => value.includes(char)) ) {\n throw APIError.create('field_invalid', null, { key });\n }\n};\n\n/**\n * Validates a URL value with optional maxlen constraint.\n * Uses the validator library, allowing localhost.\n * @param {string} value - The value to validate\n * @param {object} meta - Metadata for the validation\n * @param {string} meta.key - The field name (for error messages)\n * @param {number} [meta.maxlen] - Maximum length allowed\n */\nexport const validate_url = (value, { key, maxlen }) => {\n if ( typeof value !== 'string' ) {\n throw APIError.create('field_invalid', null, { key });\n }\n if ( maxlen !== undefined && value.length > maxlen ) {\n throw APIError.create('field_too_long', null, { key, max_length: maxlen });\n }\n // URL validation using validator library (same as url prop type)\n let valid = validator.isURL(value);\n if ( ! valid ) {\n valid = validator.isURL(value, { host_whitelist: ['localhost'] });\n }\n if ( ! valid ) {\n throw APIError.create('field_invalid', null, { key });\n }\n};\n\n/**\n * Validates a JSON value (must be an object or array).\n * @param {*} value - The value to validate\n * @param {object} meta - Metadata for the validation\n * @param {string} meta.key - The field name (for error messages)\n */\nexport const validate_json = (value, { key }) => {\n if ( typeof value !== 'object' ) {\n throw APIError.create('field_invalid', null, { key });\n }\n};\n\n/**\n * Validates an array where each element is a string.\n * @param {*} value - The value to validate\n * @param {object} meta - Metadata for the validation\n * @param {string} meta.key - The field name (for error messages)\n */\nexport const validate_array_of_strings = (value, { key }) => {\n if ( ! Array.isArray(value) ) {\n throw APIError.create('field_invalid', null, { key });\n }\n for ( const item of value ) {\n if ( typeof item !== 'string' ) {\n throw APIError.create('field_invalid', null, { key });\n }\n }\n};\n"
},
{
"path": "src/backend/src/modules/development/DevelopmentModule.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n async _init () {\n const dns2 = require('dns2');\n // this.dns = new dns2(this.config.client);\n this.dns = new dns2({\n nameServers: ['127.0.0.1'],\n port: 5300,\n });\n\n if ( this.config.test_server ) {\n this.test_server_();\n }\n }\n\n /**\n * Returns the DNS client instance\n * @returns {Object} The DNS client\n */\n get_client () {\n return this.dns;\n }\n\n /**\n * Creates and starts a test DNS server that responds to A and TXT record queries\n * The server listens on port 5300 and returns mock responses for testing purposes\n */\n test_server_ () {\n const dns2 = require('dns2');\n const { Packet } = dns2;\n\n const server = dns2.createServer({\n udp: true,\n handle: (request, send, rinfo) => {\n const { questions } = request;\n const response = Packet.createResponseFromRequest(request);\n for ( const question of questions ) {\n if ( question.type === Packet.TYPE.A || question.type === Packet.TYPE.ANY ) {\n response.answers.push({\n name: question.name,\n type: Packet.TYPE.A,\n class: Packet.CLASS.IN,\n ttl: 300,\n address: '127.0.0.11',\n });\n }\n\n if ( question.type === Packet.TYPE.TXT || question.type === Packet.TYPE.ANY ) {\n response.answers.push({\n name: question.name,\n type: Packet.TYPE.TXT,\n class: Packet.CLASS.IN,\n ttl: 300,\n data: [\n JSON.stringify({ username: 'ed3' }),\n ],\n });\n }\n }\n send(response);\n },\n });\n\n server.on('listening', () => {\n this.log.debug('Fake DNS server listening', server.addresses());\n\n if ( this.config.test_server_selftest ) {\n (async () => {\n await sleep(5000);\n {\n console.log('Trying first test');\n const result = await this.dns.resolveA('test.local');\n console.log('Test 1', result);\n }\n {\n console.log('Trying second test');\n const result = await this.dns.resolve('_puter-verify.test.local', 'TXT');\n console.log('Test 2', result);\n }\n })();\n }\n });\n\n server.on('close', () => {\n console.log('Fake DNS server closed');\n });\n\n server.on('request', (request, response, rinfo) => {\n console.log(request.header.id, request.questions[0]);\n });\n\n server.on('requestError', (error) => {\n console.log('Client sent an invalid request', error);\n });\n\n server.listen({\n udp: {\n port: 5300,\n address: '127.0.0.1',\n },\n });\n }\n}\n\nmodule.exports = { DNSService };\n"
},
{
"path": "src/backend/src/modules/domain/DomainModule.js",
"content": "const { AdvancedBase } = require('@heyputer/putility');\n\nclass DomainModule extends AdvancedBase {\n async install (context) {\n const services = context.get('services');\n\n const { DomainVerificationService } = require('./DomainVerificationService');\n services.registerService('domain-verification', DomainVerificationService);\n\n // TODO: enable flag\n const { TXTVerifyService } = require('./TXTVerifyService');\n services.registerService('__txt-verify', TXTVerifyService);\n }\n}\n\nmodule.exports = { DomainModule };\n"
},
{
"path": "src/backend/src/modules/domain/DomainVerificationService.js",
"content": "const { get_user } = require('../../helpers');\nconst BaseService = require('../../services/BaseService');\n\nclass DomainVerificationService extends BaseService {\n _init () {\n this._register_commands();\n }\n async get_controlling_user ({ domain }) {\n const svc_event = this.services.get('event');\n\n // 1 :: Allow event listeners to verify domains\n const event = {\n domain,\n user: undefined,\n };\n await svc_event.emit('domain.get-controlling-user', event);\n if ( event.user ) {\n return event.user;\n }\n\n // 2 :: If there is no controlling user, 'admin' is the\n // controlling user.\n return await get_user({ username: 'admin' });\n }\n\n _register_commands (commands) {\n const svc_commands = this.services.get('commands');\n svc_commands.registerCommands('domain', [\n {\n id: 'user',\n description: '',\n handler: async (args, log) => {\n const res = await this.get_controlling_user({ domain: args[0] });\n log.log(res);\n },\n },\n ]);\n }\n}\n\nmodule.exports = {\n DomainVerificationService,\n};\n"
},
{
"path": "src/backend/src/modules/domain/TXTVerifyService.js",
"content": "const { get_user } = require('../../helpers');\nconst BaseService = require('../../services/BaseService');\nconst { atimeout } = require('../../util/asyncutil');\n\nclass TXTVerifyService extends BaseService {\n '__on_boot.consolidation' () {\n const svc_dns = this.services.get('dns');\n const dns = svc_dns.get_client();\n\n const svc_event = this.services.get('event');\n svc_event.on('domain.get-controlling-user', async (_, event) => {\n const record_name = `_puter-verify.${event.domain}`;\n try {\n const result = await atimeout(5000,\n dns.resolve(record_name, 'TXT'));\n\n const answer = result.answers.filter(a => a.name === record_name &&\n a.type === 16)[0];\n\n const data_raw = answer.data;\n const data = JSON.parse(data_raw);\n event.user = await get_user({ username: data.username });\n } catch (e) {\n console.error('ERROR', e);\n }\n });\n }\n}\n\nmodule.exports = {\n TXTVerifyService,\n};\n"
},
{
"path": "src/backend/src/modules/entitystore/EntityStoreInterfaceService.js",
"content": "/*\n * Copyright (C) 2025-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } get - Retrieve the value(s) for the given key(s).\n * @property {function(KVStoreSetParams): Promise} set - Set a value for a key, with optional expiration.\n * @property {function(KVStoreDelParams): Promise} del - Delete a value by key.\n * @property {function(KVStoreListParams): Promise} list - List key-value pairs, optionally with pagination.\n * @property {function(): Promise} flush - Delete all key-value pairs in the store.\n * @property {(params: KVStoreUpdateParams) => Promise} update - Update nested values by key.\n * @property {(params: KVStoreAddParams) => Promise} add - Append values into list paths by key.\n * @property {(params: KVStoreRemoveParams) => Promise} remove - Remove nested values by key.\n * @property {(params: {key:string, pathAndAmountMap: Record}) => Promise} incr - Increment a numeric value by key.\n * @property {(params: {key:string, pathAndAmountMap: Record}) => Promise} decr - Decrement a numeric value by key.\n * @property {function(KVStoreExpireAtParams): Promise} expireAt - Set a key to expire at a specific UNIX timestamp (seconds).\n * @property {function(KVStoreExpireParams): Promise} expire - Set a key to expire after a given TTL (seconds).\n *\n * @typedef {Object} KVStoreGetParams\n * @property {string|string[]} key - The key or array of keys to retrieve.\n *\n * @typedef {Object} KVStoreSetParams\n * @property {string} key - The key to set.\n * @property {*} value - The value to store.\n * @property {number} [expireAt] - Optional UNIX timestamp (seconds) when the key should expire.\n *\n * @typedef {Object} KVStoreDelParams\n * @property {string} key - The key to delete.\n *\n * @typedef {Object} KVStoreListParams\n * @property {string} [as] - Optional type to list as (\"keys\", \"values\", or \"entries\").\n * @property {string} [pattern] - Optional key prefix to match.\n * @property {number} [limit] - Optional max number of items to return.\n * @property {string} [cursor] - Optional cursor to continue listing from.\n *\n * @typedef {Object} KVStoreListResult\n * @property {Array} items - Items in the current page.\n * @property {string} [cursor] - Cursor for the next page, if available.\n *\n * @typedef {Object} KVStoreUpdateParams\n * @property {string} key - The key to update.\n * @property {Object.} pathAndValueMap - Map of period-joined paths to values.\n * @property {number} [ttl] - Optional TTL in seconds for the whole object.\n *\n * @typedef {Object} KVStoreAddParams\n * @property {string} key - The key to update.\n * @property {Object.} pathAndValueMap - Map of period-joined paths to values to append.\n *\n * @typedef {Object} KVStoreRemoveParams\n * @property {string} key - The key to update.\n * @property {string[]} paths - List of period-joined paths to remove.\n *\n * @typedef {Object} KVStoreExpireAtParams\n * @property {string} key - The key to set expiration for.\n * @property {number} timestamp - UNIX timestamp (seconds) when the key should expire.\n *\n * @typedef {Object} KVStoreExpireParams\n * @property {string} key - The key to set expiration for.\n * @property {number} ttl - Time-to-live in seconds.\n */\n\n/**\n * Service for registering the puter-kvstore interface, exposing a simple key-value store API\n * with support for get, set, delete, list, flush, increment, decrement, and key expiration.\n * @extends BaseService\n */\nclass KVStoreInterfaceService extends BaseService {\n /**\n * Service class for managing KVStore interface registrations.\n * Extends the base service to provide key-value store interface management.\n */\n async '__on_driver.register.interfaces' () {\n const svc_registry = this.services.get('registry');\n const col_interfaces = svc_registry.get('interfaces');\n\n // Register the puter-kvstore interface\n col_interfaces.set('puter-kvstore', {\n description: 'A simple key-value store.',\n methods: {\n get: {\n description: 'Get a value by key.',\n parameters: {\n key: { type: 'json', required: true },\n },\n result: { type: 'json' },\n },\n set: {\n description: 'Set a value by key.',\n parameters: {\n key: { type: 'string', required: true },\n value: { type: 'json' },\n expireAt: { type: 'number' },\n },\n result: { type: 'void' },\n },\n del: {\n description: 'Delete a value by key.',\n parameters: {\n key: { type: 'string' },\n },\n result: { type: 'void' },\n },\n list: {\n description: 'List key-value pairs with optional pagination.',\n parameters: {\n as: {\n type: 'string',\n },\n pattern: {\n type: 'string',\n },\n limit: {\n type: 'number',\n },\n cursor: {\n type: 'string',\n },\n },\n result: { type: 'json' },\n },\n flush: {\n description: 'Delete all key-value pairs.',\n parameters: {},\n result: { type: 'void' },\n },\n update: {\n description: 'Update nested values by key.',\n parameters: {\n key: { type: 'string', required: true },\n pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value' },\n ttl: { type: 'number', description: 'optional TTL in seconds for the whole object' },\n },\n result: { type: 'json', description: 'The updated value' },\n },\n add: {\n description: 'Append values into list paths by key.',\n parameters: {\n key: { type: 'string', required: true },\n pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value to append' },\n },\n result: { type: 'json', description: 'The updated value' },\n },\n remove: {\n description: 'Remove nested values by key.',\n parameters: {\n key: { type: 'string', required: true },\n paths: { type: 'json', required: true, description: 'list of period-joined paths to remove' },\n },\n result: { type: 'json', description: 'The updated value' },\n },\n incr: {\n description: 'Increment a value by key.',\n parameters: {\n key: { type: 'string', required: true },\n pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' },\n },\n result: { type: 'json', description: 'The updated value' },\n },\n decr: {\n description: 'Decrement a value by key.',\n parameters: {\n key: { type: 'string', required: true },\n pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' },\n\n },\n result: { type: 'json', description: 'The updated value' },\n },\n expireAt: {\n description: 'Set a key to expire at a given timestamp in sec.',\n parameters: {\n key: { type: 'string', required: true },\n timestamp: { type: 'number', required: true },\n\n },\n result: { type: 'number' },\n },\n expire: {\n description: 'Set a key to expire in ttl many seconds.',\n parameters: {\n key: { type: 'string', required: true },\n ttl: { type: 'number', required: true },\n\n },\n result: { type: 'number' },\n },\n },\n });\n }\n}\n\nmodule.exports = {\n KVStoreInterfaceService,\n};\n"
},
{
"path": "src/backend/src/modules/kvstore/KVStoreModule.js",
"content": "/*\n * Copyright (C) 2025-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n _integrity_check () {\n if ( config.env !== 'dev' ) {\n // only check in debug mode since it's expensive\n return;\n }\n\n // check the 2 maps are consistent\n if ( this.entriesByPath.size !== this.entriesByUUID.size ) {\n throw new Error('Path map and UUID map have different sizes');\n }\n\n for ( const [inner_path, uuid] of this.entriesByPath ) {\n const entry = this.entriesByUUID.get(uuid);\n\n // entry should exist\n if ( ! entry ) {\n throw new Error(`Entry ${uuid} does not exist`);\n }\n\n // path should match\n if ( this._inner_path(entry.path) !== inner_path ) {\n throw new Error(`Path ${inner_path} does not match entry ${uuid}`);\n }\n\n // uuid should match\n if ( entry.uuid !== uuid ) {\n throw new Error(`UUID ${uuid} does not match entry ${entry.uuid}`);\n }\n\n // parent should exist\n if ( entry.parent_uid ) {\n const parent_entry = this.entriesByUUID.get(entry.parent_uid);\n if ( ! parent_entry ) {\n throw new Error(`Parent ${entry.parent_uid} does not exist`);\n }\n }\n\n // parent's path should be a prefix of the entry's path\n if ( entry.parent_uid ) {\n const parent_entry = this.entriesByUUID.get(entry.parent_uid);\n if ( ! entry.path.startsWith(parent_entry.path) ) {\n throw new Error(`Parent ${entry.parent_uid} path ${parent_entry.path} is not a prefix of entry ${entry.path}`);\n }\n }\n\n // parent should be a directory\n if ( entry.parent_uid ) {\n const parent_entry = this.entriesByUUID.get(entry.parent_uid);\n if ( ! parent_entry.is_dir ) {\n throw new Error(`Parent ${entry.parent_uid} is not a directory`);\n }\n }\n }\n }\n\n /**\n * Check if a given node exists.\n *\n * @param {Object} param\n * @param {NodePathSelector | NodeUIDSelector | NodeChildSelector | RootNodeSelector | NodeRawEntrySelector} param.selector - The selector used for checking.\n * @returns {Promise} - True if the node exists, false otherwise.\n */\n async quick_check ({ selector }) {\n if ( selector instanceof NodePathSelector ) {\n const inner_path = this._inner_path(selector.value);\n return this.entriesByPath.has(inner_path);\n }\n\n if ( selector instanceof NodeUIDSelector ) {\n return this.entriesByUUID.has(selector.value);\n }\n\n // fallback to stat\n const entry = await this.stat({ selector });\n return !!entry;\n }\n\n /**\n * Performs a stat operation using the given selector.\n *\n * NB: Some returned fields currently contain placeholder values. And the\n * `path` of the absolute path from the root.\n *\n * @param {Object} param\n * @param {NodePathSelector | NodeUIDSelector | NodeChildSelector | RootNodeSelector | NodeRawEntrySelector} param.selector - The selector to stat.\n * @returns {Promise} - The result of the stat operation, or `null` if the node doesn't exist.\n */\n async stat ({ selector }) {\n try_infer_attributes(selector);\n\n let entry_uuid = null;\n\n if ( selector instanceof NodePathSelector ) {\n // stat by path\n const inner_path = this._inner_path(selector.value);\n entry_uuid = this.entriesByPath.get(inner_path);\n } else if ( selector instanceof NodeUIDSelector ) {\n // stat by uid\n entry_uuid = selector.value;\n } else if ( selector instanceof NodeChildSelector ) {\n if ( selector.path ) {\n // Shouldn't care about about parent when the \"path\" is present\n // since it might have different provider.\n return await this.stat({\n selector: new NodePathSelector(selector.path),\n });\n } else {\n // recursively stat the parent and then stat the child\n const parent_entry = await this.stat({\n selector: selector.parent,\n });\n if ( parent_entry ) {\n const full_path = _path.join(parent_entry.path, selector.name);\n return await this.stat({\n selector: new NodePathSelector(full_path),\n });\n }\n }\n } else {\n // other selectors shouldn't reach here, i.e., it's an internal logic error\n throw APIError.create('invalid_node');\n }\n\n const entry = this.entriesByUUID.get(entry_uuid);\n if ( ! entry ) {\n return null;\n }\n\n // Return a copied entry with `full_path`, since external code only cares\n // about full path.\n const copied_entry = { ...entry };\n copied_entry.path = _path.join(this.mountpoint, entry.path);\n return copied_entry;\n }\n\n /**\n * Read directory contents.\n *\n * @param {Object} param\n * @param {Context} param.context - The context of the operation.\n * @param {FSNodeContext} param.node - The directory node to read.\n * @returns {Promise} - Array of child UUIDs.\n */\n async readdir ({ context, node }) {\n // prerequistes: get required path via stat\n const entry = await this.stat({ selector: node.selector });\n if ( ! entry ) {\n throw APIError.create('invalid_node');\n }\n\n const inner_path = this._inner_path(entry.path);\n const child_uuids = [];\n\n // Find all entries that are direct children of this directory\n for ( const [path, uuid] of this.entriesByPath ) {\n if ( path === inner_path ) {\n continue; // Skip the directory itself\n }\n\n const dirname = _path.dirname(path);\n if ( dirname === inner_path ) {\n child_uuids.push(uuid);\n }\n }\n\n return child_uuids;\n }\n\n /**\n * Create a new directory.\n *\n * @param {Object} param\n * @param {Context} param.context - The context of the operation.\n * @param {FSNodeContext} param.parent - The parent node to create the directory in. Must exist and be a directory.\n * @param {string} param.name - The name of the new directory.\n * @returns {Promise} - The new directory node.\n */\n async mkdir ({ context, parent, name }) {\n // prerequistes: get required path via stat\n const parent_entry = await this.stat({ selector: parent.selector });\n if ( ! parent_entry ) {\n throw APIError.create('invalid_node');\n }\n\n const full_path = _path.join(parent_entry.path, name);\n const inner_path = this._inner_path(full_path);\n\n let entry = null;\n if ( this.entriesByPath.has(inner_path) ) {\n throw APIError.create('item_with_same_name_exists', null, {\n entry_name: full_path,\n });\n } else {\n entry = new MemoryFile({\n path: inner_path,\n is_dir: true,\n content: null,\n parent_uid: parent_entry.uuid,\n });\n this.entriesByPath.set(inner_path, entry.uuid);\n this.entriesByUUID.set(entry.uuid, entry);\n }\n\n // create the node\n const fs = context.get('services').get('filesystem');\n const node = await fs.node(new NodeUIDSelector(entry.uuid));\n await node.fetchEntry();\n\n this._integrity_check();\n\n return node;\n }\n\n /**\n * Remove a directory.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNodeContext} param.node: The directory to remove.\n * @param {Object} param.options: The options for the operation.\n * @returns {Promise}\n */\n async rmdir ({ context, node, options = {} }) {\n this._integrity_check();\n\n // prerequistes: get required path via stat\n const entry = await this.stat({ selector: node.selector });\n if ( ! entry ) {\n throw APIError.create('invalid_node');\n }\n\n const inner_path = this._inner_path(entry.path);\n\n // for mode: non-recursive\n if ( ! options.recursive ) {\n const children = await this.readdir({ context, node });\n if ( children.length > 0 ) {\n throw APIError.create('not_empty');\n }\n }\n\n // remove all descendants\n for ( const [other_inner_path, other_entry_uuid] of this.entriesByPath ) {\n if ( other_entry_uuid === entry.uuid ) {\n // skip the directory itself\n continue;\n }\n\n if ( other_inner_path.startsWith(inner_path) ) {\n this.entriesByPath.delete(other_inner_path);\n this.entriesByUUID.delete(other_entry_uuid);\n }\n }\n\n // for mode: non-descendants-only\n if ( ! options.descendants_only ) {\n // remove the directory itself\n this.entriesByPath.delete(inner_path);\n this.entriesByUUID.delete(entry.uuid);\n }\n\n this._integrity_check();\n }\n\n /**\n * Remove a file.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNodeContext} param.node: The file to remove.\n * @returns {Promise}\n */\n async unlink ({ context, node }) {\n // prerequistes: get required path via stat\n const entry = await this.stat({ selector: node.selector });\n if ( ! entry ) {\n throw APIError.create('invalid_node');\n }\n\n const inner_path = this._inner_path(entry.path);\n this.entriesByPath.delete(inner_path);\n this.entriesByUUID.delete(entry.uuid);\n }\n\n /**\n * Move a file.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNodeContext} param.node: The file to move.\n * @param {FSNodeContext} param.new_parent: The new parent directory of the file.\n * @param {string} param.new_name: The new name of the file.\n * @param {Object} param.metadata: The metadata of the file.\n * @returns {Promise}\n */\n async move ({ context, node, new_parent, new_name, metadata }) {\n // prerequistes: get required path via stat\n const new_parent_entry = await this.stat({ selector: new_parent.selector });\n if ( ! new_parent_entry ) {\n throw APIError.create('invalid_node');\n }\n\n // create the new entry\n const new_full_path = _path.join(new_parent_entry.path, new_name);\n const new_inner_path = this._inner_path(new_full_path);\n const entry = new MemoryFile({\n path: new_inner_path,\n is_dir: node.entry.is_dir,\n content: node.entry.content,\n parent_uid: new_parent_entry.uuid,\n });\n entry.uuid = node.entry.uuid;\n this.entriesByPath.set(new_inner_path, entry.uuid);\n this.entriesByUUID.set(entry.uuid, entry);\n\n // remove the old entry\n const inner_path = this._inner_path(node.path);\n this.entriesByPath.delete(inner_path);\n // NB: should not delete the entry by uuid because uuid does not change\n // after the move.\n\n this._integrity_check();\n\n return entry;\n }\n\n /**\n * Copy a tree of files and directories.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNodeContext} param.source - The source node to copy.\n * @param {FSNodeContext} param.parent - The parent directory for the copy.\n * @param {string} param.target_name - The name for the copied item.\n * @returns {Promise} - The copied node.\n */\n async copy_tree ({ context, source, parent, target_name }) {\n const fs = context.get('services').get('filesystem');\n\n if ( source.entry.is_dir ) {\n // Create the directory\n const new_dir = await this.mkdir({ context, parent, name: target_name });\n\n // Copy all children\n const children = await this.readdir({ context, node: source });\n for ( const child_uuid of children ) {\n const child_node = await fs.node(new NodeUIDSelector(child_uuid));\n await child_node.fetchEntry();\n const child_name = child_node.entry.name;\n\n await this.copy_tree({\n context,\n source: child_node,\n parent: new_dir,\n target_name: child_name,\n });\n }\n\n return new_dir;\n } else {\n // Copy the file\n const new_file = await this.write_new({\n context,\n parent,\n name: target_name,\n file: { stream: { read: () => source.entry.content } },\n });\n return new_file;\n }\n }\n\n /**\n * Write a new file to the filesystem. Throws an error if the destination\n * already exists.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNodeContext} param.parent: The parent directory of the destination directory.\n * @param {string} param.name: The name of the destination directory.\n * @param {Object} param.file: The file to write.\n * @returns {Promise}\n */\n async write_new ({ context, parent, name, file }) {\n // prerequistes: get required path via stat\n const parent_entry = await this.stat({ selector: parent.selector });\n if ( ! parent_entry ) {\n throw APIError.create('invalid_node');\n }\n const full_path = _path.join(parent_entry.path, name);\n const inner_path = this._inner_path(full_path);\n\n let entry = null;\n if ( this.entriesByPath.has(inner_path) ) {\n throw APIError.create('item_with_same_name_exists', null, {\n entry_name: full_path,\n });\n } else {\n entry = new MemoryFile({\n path: inner_path,\n is_dir: false,\n content: file.stream.read(),\n parent_uid: parent_entry.uuid,\n });\n this.entriesByPath.set(inner_path, entry.uuid);\n this.entriesByUUID.set(entry.uuid, entry);\n }\n\n const fs = context.get('services').get('filesystem');\n const node = await fs.node(new NodeUIDSelector(entry.uuid));\n await node.fetchEntry();\n\n this._integrity_check();\n\n return node;\n }\n\n /**\n * Overwrite an existing file. Throws an error if the destination does not\n * exist.\n *\n * @param {Object} param\n * @param {Context} param.context\n * @param {FSNodeContext} param.node: The node to write to.\n * @param {Object} param.file: The file to write.\n * @returns {Promise}\n */\n async write_overwrite ({ context, node, file }) {\n const entry = await this.stat({ selector: node.selector });\n if ( ! entry ) {\n throw APIError.create('invalid_node');\n }\n const inner_path = this._inner_path(entry.path);\n\n this.entriesByPath.set(inner_path, entry.uuid);\n let original_entry = this.entriesByUUID.get(entry.uuid);\n if ( ! original_entry ) {\n throw new Error(`File ${entry.path} does not exist`);\n } else {\n if ( original_entry.is_dir ) {\n throw new Error('Cannot overwrite a directory');\n }\n\n original_entry.content = file.stream.read();\n original_entry.modified = Math.floor(Date.now() / 1000);\n original_entry.size = original_entry.content ? original_entry.content.length : 0;\n this.entriesByUUID.set(entry.uuid, original_entry);\n }\n\n const fs = context.get('services').get('filesystem');\n node = await fs.node(new NodeUIDSelector(original_entry.uuid));\n await node.fetchEntry();\n\n this._integrity_check();\n\n return node;\n }\n\n async read ({\n context,\n node,\n }) {\n // TODO: once MemoryFS aggregates its own storage, don't get it\n // via mountpoint service.\n const svc_mountpoint = context.get('services').get('mountpoint');\n const storage = svc_mountpoint.get_storage(this.constructor.name);\n const stream = (await storage.create_read_stream(await node.get('uid'), {\n memory_file: node.entry,\n }));\n return stream;\n }\n}\n\nmodule.exports = {\n MemoryFSProvider,\n};\n"
},
{
"path": "src/backend/src/modules/puterfs/customfs/MemoryFSService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } codes - A map of error codes to error specifications\n */\n register (codes) {\n for ( const code in codes ) {\n this.codes[code] = codes[code];\n }\n }\n\n create (code, fields) {\n const error_spec = this.codes[code];\n if ( ! error_spec ) {\n return new APIError(500, 'Missing error message.', null, {\n code,\n });\n }\n\n return new APIError(error_spec.status, error_spec.message, null, {\n ...fields,\n code,\n });\n }\n}\n\nmodule.exports = APIErrorService;\n"
},
{
"path": "src/backend/src/modules/web/README.md",
"content": "# WebModule\n\nThis module initializes a pre-configured web server and socket.io server.\nThe main service, WebServerService, emits 'install.routes' and provides\nthe server instance to the callback.\n\n## Services\n\n### SocketioService\n\nSocketioService provides a service for sending messages to clients.\nsocket.io is used behind the scenes. This service provides a simpler\ninterface for sending messages to rooms or socket ids.\n\n#### Listeners\n\n##### `install.socketio`\n\nInitializes socket.io\n\n###### Parameters\n\n- **server:** The server to attach socket.io to.\n\n### WebServerService\n\nThis class, WebServerService, is responsible for starting and managing the Puter web server.\nIt initializes the Express app, sets up middlewares, routes, and handles authentication and web sockets.\nIt also validates the host header and IP addresses to prevent security vulnerabilities.\n\n#### Listeners\n\n##### `boot.consolidation`\n\nThis method initializes the backend web server for Puter. It sets up the Express app, configures middleware, and starts the HTTP server.\n\n##### `boot.activation`\n\nStarts the web server and listens for incoming connections.\nThis method sets up the Express app, sets up middleware, and starts the server on the specified port.\nIt also sets up the Socket.io server for real-time communication.\n\n##### `start.webserver`\n\nThis method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use.\nIf the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299.\nOnce the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events.\nIf the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser.\n\n## Notes\n\n### Outside Imports\n\nThis module has external relative imports. When these are\nremoved it may become possible to move this module to an\nextension.\n\n**Imports:**\n- `../../services/BaseService` (use.BaseService)\n- `../../util/context.js`\n- `../../services/BaseService.js`\n- `../../config.js`\n- `../../middleware/auth.js`\n- `../../util/strutil.js`\n- `../../helpers.js`\n"
},
{
"path": "src/backend/src/modules/web/SocketioService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n async send (socket_specifiers, key, data) {\n if ( ! Array.isArray(socket_specifiers) ) {\n socket_specifiers = [socket_specifiers];\n }\n\n for ( const socket_specifier of socket_specifiers ) {\n if ( socket_specifier.room ) {\n this.io.to(socket_specifier.room).emit(key, data);\n } else if ( socket_specifier.socket ) {\n this.io.to(socket_specifier.socket).emit(key, data);\n }\n }\n }\n\n /**\n * Checks if the specified socket or room exists\n *\n * @param {Object} socket_specifier - The socket specifier object\n * @returns {boolean} True if the socket exists, false otherwise\n */\n has (socket_specifier) {\n if ( socket_specifier.room ) {\n const room = this.io?.sockets.adapter.rooms.get(socket_specifier.room);\n return (!!room) && room.size > 0;\n }\n if ( socket_specifier.socket ) {\n return this.io?.sockets.sockets.has(socket_specifier.socket);\n }\n }\n}\n\nmodule.exports = SocketioService;\n"
},
{
"path": "src/backend/src/modules/web/WebModule.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves once the server is started.\n */\n async '__on_boot.activation' () {\n const services = this.services;\n await services.emit('start.webserver');\n await services.emit('ready.webserver');\n console.log('in case you care, ready.webserver hooks are done');\n }\n\n /**\n * This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use.\n * If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299.\n * Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events.\n * If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser.\n *\n * @return {Promise} A promise that resolves when the server is up and running.\n */\n async '__on_start.webserver' () {\n // error handling middleware goes last, as per the\n // expressjs documentation:\n // https://expressjs.com/en/guide/error-handling.html\n this.app.use(require('./lib/api_error_handler.js'));\n\n const { jwt_auth } = require('../../helpers.js');\n\n config.http_port = process.env.PORT ?? config.http_port;\n\n globalThis.deployment_type =\n config.http_port === 5101 ? 'green' :\n config.http_port === 5102 ? 'blue' :\n 'not production';\n\n let server;\n\n const auto_port = config.http_port === 'auto';\n let ports_to_try = auto_port ? (() => {\n const ports = [];\n for ( let i = 0 ; i < 20 ; i++ ) {\n ports.push(4100 + i);\n }\n return ports;\n })() : [Number.parseInt(config.http_port)];\n\n for ( let i = 0 ; i < ports_to_try.length ; i++ ) {\n const port = ports_to_try[i];\n const is_last_port = i === ports_to_try.length - 1;\n if ( auto_port ) this.log.debug(`trying port: ${ port}`);\n try {\n server = http.createServer(this.app).listen(port);\n server.timeout = 1000 * 60 * 60 * 2; // 2 hours\n let should_continue = false;\n await new Promise((rslv, rjct) => {\n server.on('error', e => {\n if ( e.code === 'EADDRINUSE' ) {\n if ( !is_last_port && e.code === 'EADDRINUSE' ) {\n this.log.info(`port in use: ${ port}`);\n should_continue = true;\n }\n rslv();\n } else {\n rjct(e);\n }\n });\n /**\n * Starts the web server.\n *\n * This method is responsible for creating the HTTP server, setting up middleware, and starting the server on the specified port. If the specified port is \"auto\", it will attempt to find an available port within a range.\n *\n * @returns {Promise}\n */\n // Add this comment above line 110\n // (line 110 of the provided code)\n server.on('listening', () => {\n rslv();\n });\n });\n if ( should_continue ) continue;\n } catch (e) {\n if ( !is_last_port && e.code === 'EADDRINUSE' ) {\n this.log.info(`port in use:${ port}`);\n continue;\n }\n throw e;\n }\n config.http_port = port;\n break;\n }\n ports_to_try = null; // GC\n\n const url = config.origin;\n\n const args = yargs(hideBin(process.argv)).argv;\n if ( args['server'] ) {\n (async () => {\n (await import('./../../../../../tools/auth_gui.js')).default(args['puter-backend']);\n })();\n config.no_browser_launch = true;\n }\n // Open the browser to the URL of Puter\n // (if we are in development mode only)\n if ( config.env === 'dev' && !config.no_browser_launch ) {\n try {\n const openModule = await import('open');\n openModule.default(url);\n } catch (e) {\n console.log('Error opening browser', e);\n }\n }\n\n const link = `\\x1B[34;1m${url}\\x1B[0m`;\n const lines = [\n `Puter is now live at: ${link}`,\n `listening on port: ${config.http_port}`,\n ];\n const realConsole = globalThis.original_console_object ?? console;\n lines.forEach(line => realConsole.log(line));\n\n realConsole.log('\\n************************************************************');\n realConsole.log(`* Puter is now live at: ${url}`);\n realConsole.log('************************************************************');\n\n server.timeout = 1000 * 60 * 60 * 2; // 2 hours\n server.requestTimeout = 1000 * 60 * 60 * 2; // 2 hours\n server.headersTimeout = 1000 * 60 * 60 * 2; // 2 hours\n // server.keepAliveTimeout = 1000 * 60 * 60 * 2; // 2 hours\n\n // Socket.io server instance\n // const socketio = require('../../socketio.js').init(server);\n\n // TODO: ^ Replace above line with the following code:\n await this.services.emit('install.socketio', { server });\n const socketio = this.services.get('socketio').io;\n const authService = this.services.get('auth');\n\n // Socket.io middleware for authentication\n socketio.use(async (socket, next) => {\n const authToken = socket.handshake?.auth?.auth_token;\n if ( ! authToken ) {\n next(new Error('socket auth token missing'));\n return;\n }\n\n try {\n const authRes = await jwt_auth(socket, authService);\n // successful auth\n socket.actor = authRes.actor;\n socket.user = authRes.user;\n socket.token = authRes.token;\n // join user room\n socket.join(socket.user.id);\n\n // setTimeout 0 is needed because we need to send\n // the notifications after this handler is done\n // setTimeout(() => {\n // }, 1000);\n next();\n } catch ( error ) {\n console.warn('socket auth err', error);\n const authError = error instanceof Error\n ? error\n : new Error('socket auth failed');\n next(authError);\n }\n });\n\n const context = Context.get();\n socketio.on('connection', (socket) => {\n socket.on('disconnect', () => {\n });\n socket.on('trash.is_empty', (msg) => {\n socket.broadcast.to(socket.user.id).emit('trash.is_empty', msg);\n });\n const svc_event = this.services.get('event');\n svc_event.emit('web.socket.connected', {\n socket,\n user: socket.user,\n });\n socket.on('puter_is_actually_open', async (_msg) => {\n await context.sub({\n actor: socket.actor,\n }).arun(async () => {\n await svc_event.emit('web.socket.user-connected', {\n socket,\n user: socket.user,\n });\n });\n });\n });\n\n this.server_ = server;\n await this.services.emit('install.websockets');\n }\n\n /**\n * Starts the Puter web server and sets up routes, middleware, and error handling.\n *\n * @param {object} services - An object containing all services available to the web server.\n * @returns {Promise} A promise that resolves when the web server is fully started.\n */\n get_server () {\n return this.server_;\n }\n\n /**\n * Handles starting and managing the Puter web server.\n *\n * @param {Object} services - An object containing all services.\n */\n async _init () {\n const app = express();\n this.app = app;\n\n app.set('services', this.services);\n\n this.middlewares = { auth };\n\n const require = this.require;\n\n const config = this.global_config;\n new ContextExpressMiddleware({\n parent: globalThis.root_context.sub({\n puter_environment: Context.create({\n env: config.env,\n version: relative_require('../../../package.json').version,\n }),\n }, 'mw'),\n }).install(app);\n\n app.use(async (req, res, next) => {\n req.services = this.services;\n next();\n });\n\n // When the user visits the main origin (not api/dav subdomain) with ?auth_token=\n // (e.g. QR login), set the HTTP-only session cookie so user-protected endpoints work.\n app.use(async (req, res, next) => {\n const has_subdomain = req.hostname.slice(0, -1 * (config.domain.length + 1)) !== '';\n if ( has_subdomain ) return next();\n\n const token = req.query?.auth_token;\n if ( !token || typeof token !== 'string' ) return next();\n\n try {\n const svc_auth = req.services.get('auth');\n const cleanToken = token.replace('Bearer ', '').trim();\n const actor = await svc_auth.authenticate_from_token(cleanToken);\n const session_token = svc_auth.create_session_token_for_session(\n actor.type.user,\n actor.type.session,\n );\n res.cookie(config.cookie_name, session_token, {\n sameSite: 'none',\n secure: true,\n httpOnly: true,\n });\n } catch ( e ) {\n console.log('query auth token (QR Code login probably) failed');\n console.error(e);\n }\n next();\n });\n\n // Measure data transfer amounts\n app.use(measure());\n\n // Instrument logging to use our log service\n {\n // Switch log function at config time; info log is configurable\n const logfn = (config.logging ?? []).includes('http')\n ? (log, { message, fields }) => {\n log.info(message);\n log.debug(message, fields);\n }\n : (log, { message, fields }) => {\n log.debug(message, fields);\n };\n\n const morgan = require('morgan');\n const stream = {\n write: (message) => {\n const [method, url, status, responseTime] = message.split(' ');\n const fields = {\n method,\n url,\n status: parseInt(status, 10),\n responseTime: parseFloat(responseTime),\n };\n if ( url.includes('android-icon') ) return;\n\n // remove `puter.auth.*` query params\n const safe_url = (u => {\n // We need to prepend an arbitrary domain to the URL\n const url = new URL(`https://example.com${ u}`);\n const search = url.searchParams;\n for ( const key of search.keys() ) {\n if ( key.startsWith('puter.auth.') ) search.delete(key);\n }\n return `${url.pathname }?${ search.toString()}`;\n })(fields.url);\n fields.url = safe_url;\n // re-write message\n message = [\n fields.method, fields.url,\n fields.status, fields.responseTime,\n ].join(' ');\n\n const log = this.services.get('log-service').create('morgan');\n try {\n this.context.arun(() => {\n logfn(log, { message, fields });\n });\n } catch (e) {\n console.log('failed to log this message properly:', message, fields);\n console.error(e);\n }\n },\n };\n\n app.use(morgan(':method :url :status :response-time', { stream }));\n }\n\n /**\n * Initialize the web server, start it, and handle any related logic.\n *\n * This method is responsible for creating the server and listening on the\n * appropriate port. It also sets up middleware, routes, and other necessary\n * configurations.\n *\n * @returns {Promise} A promise that resolves once the server is up and running.\n */\n app.use((() => {\n // const router = express.Router();\n // router.get('/wut', express.json(), (req, res, next) => {\n // return res.status(500).send('Internal Error');\n // });\n // return router;\n\n return eggspress('/wut', {\n allowedMethods: ['GET'],\n }, async (req, res, _next) => {\n // throw new Error('throwy error');\n return res.status(200).send('test endpoint');\n });\n })());\n\n (() => {\n const onFinished = require('on-finished');\n app.use((req, res, next) => {\n onFinished(res, () => {\n if ( res.statusCode !== 500 ) return;\n if ( req.__error_handled ) return;\n const alarm = this.services.get('alarm');\n alarm.create('responded-500', 'server sent a 500 response', {\n error: req.__error_source,\n url: req.url,\n method: req.method,\n body: req.body,\n headers: req.headers,\n });\n });\n next();\n });\n })();\n\n app.use(async function (req, res, next) {\n // Express does not document that this can be undefined.\n // The browser likely doesn't follow the HTTP/1.1 spec\n // (bot client?) and express is handling this badly by\n // not setting the header at all. (that's my theory)\n if ( req.hostname === undefined ) {\n res.status(400).send(\n 'Please verify your browser is up-to-date.',\n );\n return;\n }\n\n return next();\n });\n\n // Validate host header against allowed domains to prevent host header injection\n // https://www.owasp.org/index.php/Host_Header_Injection\n app.use((req, res, next) => {\n const allowedDomains = new Set();\n const pushAllowedDomain = (domain) => {\n const normalizedDomain = normalizeHostDomain(domain);\n if ( normalizedDomain ) {\n allowedDomains.add(normalizedDomain);\n }\n };\n\n const staticHostingDomain = normalizeHostDomain(config.static_hosting_domain);\n pushAllowedDomain(config.domain);\n pushAllowedDomain(staticHostingDomain);\n pushAllowedDomain(config.static_hosting_domain_alt);\n pushAllowedDomain(config.private_app_hosting_domain);\n pushAllowedDomain(config.private_app_hosting_domain_alt);\n if ( staticHostingDomain ) {\n pushAllowedDomain(`at.${staticHostingDomain}`);\n }\n\n if ( config.allow_nipio_domains ) {\n pushAllowedDomain('nip.io');\n }\n\n // Retrieve the Host header and ensure it's in a valid format\n const hostHeader = req.headers.host;\n\n if ( !config.allow_no_host_header && !hostHeader ) {\n return res.status(400).send('Missing Host header.');\n }\n\n if ( config.allow_all_host_values ) {\n next();\n return;\n }\n\n // Parse the Host header to isolate the hostname (strip out port if present)\n const hostName = hostHeader.split(':')[0].trim().toLowerCase();\n // Check if the hostname matches any of the allowed domains or is a subdomain of an allowed domain\n // Exception: allow /healthcheck endpoint on the root domain\n if (\n req.path === '/healthcheck'\n ) {\n next();\n return;\n }\n if ( [...allowedDomains].some(allowedDomain => hostMatchesDomain(hostName, allowedDomain)) ) {\n next(); // Proceed if the host is valid\n return;\n } else {\n if ( ! config.custom_domains_enabled ) {\n res.status(400).send('Invalid Host header.');\n return;\n }\n req.is_custom_domain = true;\n next();\n return;\n }\n });\n\n // Validate IP with any IP checkers\n app.use(async (req, res, next) => {\n const svc_event = this.services.get('event');\n const event = {\n allow: true,\n ip: req.headers?.['x-forwarded-for'] ||\n req.connection?.remoteAddress,\n };\n\n if ( ! this.config.disable_ip_validate_event ) {\n await svc_event.emit('ip.validate', event);\n }\n\n // rules that don't apply to notification endpoints\n const undefined_origin_allowed = config.undefined_origin_allowed || this.allowedRoutesWithUndefinedOrigins.some(rule => {\n if ( typeof rule === 'string' ) return rule === req.path;\n return rule.test(req.path);\n });\n if ( ! undefined_origin_allowed ) {\n // check if no origin\n if ( req.method === 'POST' && req.headers.origin === undefined ) {\n event.allow = false;\n }\n }\n if ( ! event.allow ) {\n return res.status(403).send('Forbidden');\n }\n next();\n });\n\n // Web hooks need a router that occurs before JSON parse middleware\n // so that signatures of the raw JSON can be verified\n this.router_webhooks = express.Router();\n app.use(this.router_webhooks);\n\n app.use((req, res, next) => {\n if ( req.get('x-amz-sns-message-type') ) {\n req.headers['content-type'] = 'application/json';\n }\n next();\n });\n\n const rawBodyBuffer = (req, res, buf, encoding) => {\n req.rawBody = buf.toString(encoding || 'utf8');\n };\n\n app.use(express.json({ limit: '50mb', verify: rawBodyBuffer }));\n app.use((req, res, next) => {\n if ( req.headers['content-type']?.startsWith('application/json')\n && req.body\n && Buffer.isBuffer(req.body)\n ) {\n try {\n req.rawBody = req.body;\n req.body = JSON.parse(req.body.toString('utf8'));\n } catch {\n return res.status(400).send({\n error: {\n message: 'Invalid JSON body',\n },\n });\n }\n }\n next();\n });\n\n const cookieParser = require('cookie-parser');\n app.use(cookieParser({ limit: '50mb' }));\n\n // gzip compression for all requests\n const compression = require('compression');\n app.use(compression());\n\n // Helmet and other security\n const helmet = require('helmet');\n app.use(helmet.noSniff());\n app.use(helmet.hsts());\n app.use(helmet.ieNoOpen());\n app.use(helmet.permittedCrossDomainPolicies());\n app.use(helmet.xssFilter());\n // app.use(helmet.referrerPolicy());\n app.disable('x-powered-by');\n\n // remove object and array query parameters\n app.use(function (req, res, next) {\n for ( let k in req.query ) {\n if ( req.query[k] === undefined || req.query[k] === null ) {\n continue;\n }\n\n const allowed_types = ['string', 'number', 'boolean'];\n if ( ! allowed_types.includes(typeof req.query[k]) ) {\n req.query[k] = undefined;\n }\n }\n next();\n });\n\n const uaParser = require('ua-parser-js');\n app.use(function (req, res, next) {\n const ua_header = req.headers['user-agent'];\n const ua = uaParser(ua_header);\n req.ua = ua;\n next();\n });\n\n app.use(function (req, res, next) {\n req.co_isolation_enabled =\n ['Chrome', 'Edge'].includes(req.ua.browser.name)\n && (Number(req.ua.browser.major) >= 110);\n next();\n });\n\n app.use(function (req, res, next) {\n const origin = req.headers.origin;\n const subdomain = req.subdomains[req.subdomains.length - 1];\n const isApiOrDavRequest =\n config.experimental_no_subdomain ||\n subdomain === 'api' ||\n subdomain === 'dav';\n const isCrossOriginAuthRoute =\n req.path === '/signup' ||\n req.path === '/login' ||\n req.path.startsWith('/extensions/') ||\n req.path.startsWith('/auth/oidc');\n\n const is_site =\n hostMatchesDomain(req.hostname, config.static_hosting_domain) ||\n hostMatchesDomain(req.hostname, config.static_hosting_domain_alt) ||\n hostMatchesDomain(req.hostname, config.private_app_hosting_domain) ||\n hostMatchesDomain(req.hostname, config.private_app_hosting_domain_alt);\n req.hostname === 'docs.puter.com'\n ;\n const is_popup = !!req.query.embedded_in_popup;\n const is_parent_co = !!req.query.cross_origin_isolated;\n const is_app = !!req.query['puter.app_instance_id'];\n\n const co_isolation_okay =\n (!is_popup || is_parent_co) &&\n (is_app || !is_site) &&\n req.co_isolation_enabled\n ;\n\n if ( isCrossOriginAuthRoute || isApiOrDavRequest ) {\n res.setHeader('Access-Control-Allow-Origin', origin ?? '*');\n if ( origin ) {\n res.vary('Origin');\n }\n }\n\n // Allow browser credentials on API/DAV cross-origin requests.\n if ( isApiOrDavRequest && origin ) {\n res.setHeader('Access-Control-Allow-Credentials', 'true');\n }\n\n // Request methods to allow\n res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK');\n\n const allowed_headers = [\n 'Origin', 'X-Requested-With', 'Content-Type', 'Accept', 'Authorization', 'sentry-trace', 'baggage',\n 'Depth', 'Destination', 'Overwrite', 'If', 'Lock-Token', 'DAV', 'stripe-signature',\n ];\n\n // Request headers to allow\n res.header('Access-Control-Allow-Headers', allowed_headers.join(', '));\n\n // Needed for SharedArrayBuffer\n // NOTE: This is put behind a configuration flag because we\n // need some experimentation to ensure the interface\n // between apps and Puter doesn't break.\n if ( config.cross_origin_isolation && co_isolation_okay ) {\n res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');\n res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');\n }\n res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');\n\n // Pass to next layer of middleware\n\n // disable iframes on the main domain\n if ( req.hostname === config.domain ) {\n // disable iframes\n res.setHeader('X-Frame-Options', 'SAMEORIGIN');\n }\n\n next();\n });\n }\n}\n\nmodule.exports = WebServerService;\n"
},
{
"path": "src/backend/src/modules/web/lib/__lib__.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } Query predicate object\n */\n async create_predicate (id, ...args) {\n if ( id === 'user-can-edit' ) {\n return new Eq({\n key: 'owner',\n value: Context.get('user').id,\n });\n }\n if ( id === 'name-like' ) {\n return new Like({\n key: 'name',\n value: args[0],\n });\n }\n },\n async delete (uid, _extra) {\n const svc_appInformation = this.context.get('services').get('app-information');\n await svc_appInformation.delete_app(uid);\n },\n\n async read (uid) {\n if ( typeof uid !== 'string' || !uid ) {\n return await this.upstream.read(uid);\n }\n\n const canonicalUidAliasPromise = this.read_canonical_app_uid_alias_(uid);\n const entity = await this.upstream.read(uid);\n if ( entity ) {\n return entity;\n }\n\n const canonicalUid = await canonicalUidAliasPromise;\n if ( !canonicalUid || canonicalUid === uid ) {\n return null;\n }\n\n return await this.upstream.read(canonicalUid);\n },\n\n /**\n * Filters app selection based on user permissions and visibility settings\n * @param {Object} options - Selection options including predicates\n * @returns {Promise} Filtered selection results\n */\n async select (options) {\n const actor = Context.get('actor');\n const user = actor.type.user;\n\n const additional = [];\n\n // An app is also allowed to read itself\n if ( actor.type instanceof AppUnderUserActorType ) {\n additional.push(new Eq({\n key: 'uid',\n value: actor.type.app.uid,\n }));\n }\n\n options.predicate = options.predicate.and(new Or({\n children: [\n new Eq({\n key: 'approved_for_listing',\n value: 1,\n }),\n new Eq({\n key: 'owner',\n value: user.id,\n }),\n ...additional,\n ],\n }));\n\n return await this.upstream.select(options);\n },\n\n /**\n * Creates or updates an application with proper name handling and associations\n * @param {Object} entity - Application entity to upsert\n * @param {Object} extra - Additional upsert parameters\n * @returns {Promise} Upsert operation results\n */\n async upsert (entity, extra) {\n extra = extra || {};\n const actor = Context.get('actor');\n const user = actor?.type?.user;\n\n const preJoinFullEntity = extra.old_entity\n ? await (await extra.old_entity.clone()).apply(entity)\n : entity\n ;\n await this.ensurePuterSiteSubdomainIsOwned(preJoinFullEntity, extra, user);\n\n await this.maybe_join_owned_hosted_index_url_app_on_create_(entity, extra, user);\n\n const full_entity = extra.old_entity\n ? await (await extra.old_entity.clone()).apply(entity)\n : entity\n ;\n\n await this.ensureIndexUrlUnique(full_entity, extra);\n\n if ( await app_name_exists(await entity.get('name')) ) {\n const { old_entity } = extra;\n const is_name_change = ( !old_entity ) ||\n ( await old_entity.get('name') !== await entity.get('name') );\n if ( is_name_change && extra?.options?.dedupe_name ) {\n const base = await entity.get('name');\n let number = 1;\n while ( await app_name_exists(`${base}-${number}`) ) {\n number++;\n }\n await entity.set('name', `${base}-${number}`);\n }\n else if ( is_name_change ) {\n // The name might be taken because it's the old name\n // of this same app. If it is, the app takes it back.\n const svc_oldAppName = this.context.get('services').get('old-app-name');\n const name_info = await svc_oldAppName.check_app_name(await entity.get('name'));\n if ( !name_info || name_info.app_uid !== await entity.get('uid') ) {\n // Throw error because the name really is taken\n throw APIError.create('app_name_already_in_use', null, {\n name: await entity.get('name'),\n });\n }\n\n // Remove the old name from the old-app-name service\n await svc_oldAppName.remove_name(name_info.id);\n } else {\n entity.del('name');\n }\n }\n\n const subdomain_id = await this.maybe_insert_subdomain_(entity);\n const result = await this.upstream.upsert(entity, extra);\n const { insert_id } = result;\n const oldAssociations = await this.db.read(\n 'SELECT type FROM app_filetype_association WHERE app_id = ?',\n [insert_id],\n );\n const normalizedOldAssociations = oldAssociations\n .map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\\./, ''))\n .filter(Boolean);\n\n // Remove old file associations (if applicable)\n if ( extra.old_entity ) {\n await this.db.write(\n 'DELETE FROM app_filetype_association WHERE app_id = ?',\n [insert_id],\n );\n }\n\n // Add file associations (if applicable)\n const filetype_associations = await entity.get('filetype_associations');\n const normalizedNewAssociations = (filetype_associations ?? [])\n .map(association => String(association).trim().toLowerCase().replace(/^\\./, ''))\n .filter(Boolean);\n if ( (a => a && a.length > 0)(filetype_associations) ) {\n const stmt =\n 'INSERT INTO app_filetype_association ' +\n `(app_id, type) VALUES ${\n normalizedNewAssociations.map(() => '(?, ?)').join(', ')}`;\n const rows = normalizedNewAssociations.map(a => [insert_id, a]);\n await this.db.write(stmt, rows.flat());\n }\n const affectedAssociationExtensions = new Set([\n ...normalizedOldAssociations,\n ...normalizedNewAssociations,\n ]);\n if ( affectedAssociationExtensions.size ) {\n await deleteRedisKeys(Array.from(affectedAssociationExtensions)\n .map(ext => AppRedisCacheSpace.associationAppsKey(ext)));\n }\n\n const has_new_icon =\n ( !extra.old_entity ) || (\n await entity.get('icon') !== await extra.old_entity.get('icon')\n );\n\n if ( has_new_icon ) {\n const svc_event = this.context.get('services').get('event');\n const event = {\n app_uid: await entity.get('uid'),\n data_url: await entity.get('icon'),\n url: '',\n };\n await svc_event.emit('app.new-icon', event);\n if ( typeof event.url === 'string' && event.url ) {\n this.db.write(\n 'UPDATE apps SET icon = ? WHERE id = ? LIMIT 1',\n [event.url, insert_id],\n );\n await entity.set('icon', event.url);\n }\n }\n\n const has_new_name =\n extra.old_entity && (\n await entity.get('name') !== await extra.old_entity.get('name')\n );\n\n if ( has_new_name ) {\n const svc_event = this.context.get('services').get('event');\n const event = {\n app_uid: await entity.get('uid'),\n new_name: await entity.get('name'),\n old_name: await extra.old_entity.get('name'),\n };\n await svc_event.emit('app.rename', event);\n }\n\n // Associate app with subdomain (if applicable)\n if ( subdomain_id ) {\n await this.db.write(\n 'UPDATE subdomains SET associated_app_id = ? WHERE id = ?',\n [insert_id, subdomain_id],\n );\n }\n if ( extra.old_entity ) {\n const svc_event = this.context.get('services').get('event');\n const [app] = await this.db.read(\n 'SELECT * FROM apps WHERE uid = ? LIMIT 1',\n [await full_entity.get('uid')],\n );\n const old_app = {\n uid: await extra.old_entity.get('uid'),\n index_url: await extra.old_entity.get('index_url'),\n };\n await svc_event.emit('app.changed', {\n app_uid: await full_entity.get('uid'),\n action: 'updated',\n app,\n old_app,\n });\n }\n\n if ( extra.joined_source_app_uid ) {\n await this.write_canonical_app_uid_alias_({\n oldAppUid: extra.joined_source_app_uid,\n canonicalAppUid: await full_entity.get('uid'),\n });\n const svc_appInformation = this.context.get('services').get('app-information');\n if ( svc_appInformation?.delete_app ) {\n await svc_appInformation.delete_app(extra.joined_source_app_uid, undefined, {\n preserveCanonicalUidAlias: true,\n });\n }\n }\n\n if ( typeof extra.joined_requested_name === 'string' && extra.joined_requested_name.trim() ) {\n const renameResult = await this.apply_joined_requested_name_({\n canonicalUid: await full_entity.get('uid'),\n requestedName: extra.joined_requested_name,\n });\n if ( renameResult ) {\n const svc_event = this.context.get('services').get('event');\n await svc_event.emit('app.rename', {\n app_uid: await full_entity.get('uid'),\n old_name: renameResult.oldName,\n new_name: renameResult.newName,\n });\n await full_entity.set('name', renameResult.newName);\n }\n }\n\n return result;\n },\n async retry_predicate_rewrite ({ predicate }) {\n const recurse = async (predicate) => {\n if ( predicate instanceof Or ) {\n return new Or({\n children: await Promise.all(predicate.children.map(recurse)),\n });\n }\n if ( predicate instanceof And ) {\n return new And({\n children: await Promise.all(predicate.children.map(recurse)),\n });\n }\n if ( predicate instanceof Eq ) {\n if ( predicate.key === 'name' ) {\n const svc_oldAppName = this.context.get('services').get('old-app-name');\n const name_info = await svc_oldAppName.check_app_name(predicate.value);\n return new Eq({\n key: 'uid',\n value: name_info?.app_uid,\n });\n }\n }\n };\n return await recurse(predicate);\n },\n\n async queueIconMigration (entity) {\n if ( ! this.pending_icon_migrations_ ) {\n this.pending_icon_migrations_ = new Set();\n }\n\n const migration_key = entity.private_meta?.mysql_id ?? Symbol('app-icon-migration');\n if ( this.pending_icon_migrations_.has(migration_key) ) {\n return;\n }\n this.pending_icon_migrations_.add(migration_key);\n\n Promise.resolve().then(async () => {\n const icon = await entity.get('icon');\n if ( typeof icon !== 'string' || !icon.startsWith('data:') ) {\n return;\n }\n\n const app_uid = await entity.get('uid');\n if ( ! app_uid ) {\n return;\n }\n\n const svc_event = this.context.get('services').get('event');\n const event = {\n app_uid,\n data_url: icon,\n };\n await svc_event.emit('app.new-icon', event);\n if ( typeof event.url !== 'string' || !event.url ) return;\n\n await this.db.write(\n 'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1',\n [event.url, app_uid],\n );\n }).catch(e => {\n const svc_error = this.context.get('services').get('error-service');\n svc_error.report('AppES:queue_icon_migration', { source: e });\n }).finally(() => {\n this.pending_icon_migrations_.delete(migration_key);\n });\n },\n\n /**\n * Transforms app data before reading by adding associations and handling permissions\n * @param {Object} entity - App entity to transform\n */\n async read_transform (entity) {\n const {\n getActorUserUid,\n resolvePrivateLaunchAccess,\n } = await getPrivateLaunchAccessModule();\n const services = this.context.get('services');\n const actor = Context.get('actor');\n const esParams = Context.get('es_params') ?? {};\n const appUid = await entity.get('uid');\n const appName = await entity.get('name');\n const appIndexUrl = await entity.get('index_url');\n const appCreatedAt = await entity.get('created_at');\n const appIsPrivate = await entity.get('is_private');\n\n const appInformationService = services.get('app-information');\n const authService = services.get('auth');\n const statsPromise = appInformationService\n ? appInformationService.get_stats(appUid, {\n period: esParams.stats_period,\n grouping: esParams.stats_grouping,\n created_at: appCreatedAt,\n })\n : Promise.resolve(undefined);\n const fileAssociationsPromise = this.db.read(\n 'SELECT type FROM app_filetype_association WHERE app_id = ?',\n [entity.private_meta.mysql_id],\n );\n const createdFromOriginPromise = (async () => {\n if ( ! authService ) return null;\n try {\n const origin = origin_from_url(appIndexUrl);\n const expectedUid = await authService.app_uid_from_origin(origin);\n return expectedUid === appUid ? origin : null;\n } catch {\n // This happens when index_url is not a valid URL.\n return null;\n }\n })();\n const privateAccessPromise = resolvePrivateLaunchAccess({\n app: {\n uid: appUid,\n name: appName,\n is_private: appIsPrivate,\n },\n services,\n userUid: getActorUserUid(actor),\n source: 'driverRead',\n args: esParams,\n });\n\n const [\n fileAssociationRows,\n stats,\n createdFromOrigin,\n privateAccess,\n ] = await Promise.all([\n fileAssociationsPromise,\n statsPromise,\n createdFromOriginPromise,\n privateAccessPromise,\n ]);\n await entity.set(\n 'filetype_associations',\n fileAssociationRows.map(row => row.type),\n );\n await entity.set('stats', stats);\n await entity.set('created_from_origin', createdFromOrigin);\n await entity.set('privateAccess', privateAccess);\n\n // Migrate b64 icons to the filesystem-backed icon flow without blocking reads.\n this.queueIconMigration(entity);\n\n // Check if the user is the owner\n const is_owner = await (async () => {\n let owner = await entity.get('owner');\n\n // TODO: why does this happen?\n if ( typeof owner === 'number' ) {\n owner = { id: owner };\n }\n\n if ( ! owner ) return false;\n const actor = Context.get('actor');\n return actor.type.user.id === owner.id;\n })();\n\n // Remove fields that are not allowed for non-owners\n if ( ! is_owner ) {\n entity.del('approved_for_listing');\n entity.del('approved_for_opening_items');\n entity.del('approved_for_incentive_program');\n }\n\n // Replace icon if an icon size is specified\n const iconSize = Context.get('es_params')?.icon_size;\n if ( iconSize ) {\n const svc_appIcon = this.context.get('services').get('app-icon');\n try {\n const iconPath = svc_appIcon.getAppIconPath({\n appUid: await entity.get('uid'),\n size: iconSize,\n });\n if ( iconPath ) {\n await entity.set('icon', iconPath);\n }\n } catch (e) {\n const svc_error = this.context.get('services').get('error-service');\n svc_error.report('AppES:read_transform', { source: e });\n }\n }\n },\n\n /**\n * Creates a subdomain entry for the app if required\n * @param {Object} entity - App entity\n * @returns {Promise} Subdomain ID if created\n * @private\n */\n async maybe_insert_subdomain_ (entity) {\n // Create and update is a situation where we might create a subdomain\n\n let subdomain_id;\n if ( await entity.get('source_directory') ) {\n await (await entity.get('source_directory')\n ).fetchEntry();\n const subdomain = await entity.get('subdomain');\n const user = Context.get('user');\n let subdomain_res = await this.db.write(\n `INSERT ${this.db.case({\n mysql: 'IGNORE',\n sqlite: 'OR IGNORE',\n })} INTO subdomains\n (subdomain, user_id, root_dir_id, uuid) VALUES\n ( ?, ?, ?, ?)`,\n [\n //subdomain\n subdomain,\n //user_id\n user.id,\n //root_dir_id\n (await entity.get('source_directory')).mysql_id,\n //uuid, `sd` stands for subdomain\n `sd-${ uuidv4()}`,\n ],\n );\n subdomain_id = subdomain_res.insertId;\n }\n\n return subdomain_id;\n },\n\n /**\n * Ensures that when an app uses a puter.site subdomain as its index_url,\n * the subdomain belongs to the user creating/updating the app.\n */\n async ensurePuterSiteSubdomainIsOwned (entity, extra, user) {\n if ( ! user ) return;\n\n // Only enforce when the index_url is being set or changed\n const new_index_url = await entity.get('index_url');\n if ( ! new_index_url ) return;\n if ( extra.old_entity ) {\n const old_index_url = await extra.old_entity.get('index_url');\n if ( old_index_url === new_index_url ) {\n return;\n }\n }\n\n const subdomain = extractPuterHostedSubdomainFromIndexUrl(new_index_url);\n if ( ! subdomain ) return;\n\n const svc_puterSite = this.context.get('services').get('puter-site');\n const site = await svc_puterSite.get_subdomain(subdomain, { is_custom_domain: false });\n\n if ( !site || site.user_id !== user.id ) {\n throw APIError.create('subdomain_not_owned', null, { subdomain });\n }\n },\n\n is_puter_hosted_index_url_ (index_url) {\n return !!extractPuterHostedSubdomainFromIndexUrl(index_url);\n },\n\n build_equivalent_index_url_candidates_ (index_url) {\n if ( typeof index_url !== 'string' || !index_url.trim() ) {\n return [];\n }\n\n try {\n const parsedUrl = new URL(index_url);\n const origin = `${parsedUrl.protocol}//${parsedUrl.host.toLowerCase()}`;\n const pathname = parsedUrl.pathname || '/';\n const values = new Set();\n if ( pathname === '/' || pathname.toLowerCase() === '/index.html' ) {\n values.add(origin);\n values.add(`${origin}/`);\n values.add(`${origin}/index.html`);\n } else {\n const normalizedPath = pathname.endsWith('/')\n ? pathname.slice(0, -1)\n : pathname;\n values.add(`${origin}${normalizedPath}`);\n values.add(`${origin}${normalizedPath}/`);\n }\n return [...values];\n } catch {\n return [index_url.trim()];\n }\n },\n\n async find_index_url_conflict_ ({ indexUrl, excludeMysqlId }) {\n if ( ! this.is_puter_hosted_index_url_(indexUrl) ) {\n return null;\n }\n\n const candidates = this.build_equivalent_index_url_candidates_(indexUrl);\n if ( candidates.length === 0 ) return null;\n if ( hasIndexUrlUniquenessExemption(candidates) ) return null;\n\n const placeholders = candidates.map(() => '?').join(', ');\n const parameters = [...candidates];\n let query = `SELECT id, uid, owner_user_id, index_url FROM apps WHERE index_url IN (${placeholders})`;\n if ( Number.isInteger(excludeMysqlId) && excludeMysqlId > 0 ) {\n query += ' AND id != ?';\n parameters.push(excludeMysqlId);\n }\n query += ' ORDER BY timestamp ASC, id ASC LIMIT 1';\n\n const rows = await this.db.read(query, parameters);\n const normalizedExcludeMysqlId = Number(excludeMysqlId);\n const conflictRow = rows.find(row => {\n if (\n Number.isInteger(normalizedExcludeMysqlId)\n && normalizedExcludeMysqlId > 0\n && Number(row?.id) === normalizedExcludeMysqlId\n ) {\n return false;\n }\n if ( typeof row?.index_url === 'string' ) {\n return candidates.includes(row.index_url);\n }\n return true;\n });\n return conflictRow || null;\n },\n\n async resolve_entity_mysql_id_ (entity) {\n const directMysqlId = Number(entity?.private_meta?.mysql_id);\n if ( Number.isInteger(directMysqlId) && directMysqlId > 0 ) {\n return directMysqlId;\n }\n\n if ( !entity || typeof entity.get !== 'function' ) {\n return undefined;\n }\n\n const uid = await entity.get('uid');\n if ( typeof uid !== 'string' || !uid ) {\n return undefined;\n }\n\n const rows = await this.db.read(\n 'SELECT id FROM apps WHERE uid = ? LIMIT 1',\n [uid],\n );\n const mysqlId = Number(rows?.[0]?.id);\n if ( Number.isInteger(mysqlId) && mysqlId > 0 ) {\n return mysqlId;\n }\n\n return undefined;\n },\n\n async claim_app_ownership_by_id_for_user_ ({ appId, userId }) {\n if ( !Number.isInteger(appId) || appId <= 0 ) return;\n if ( !Number.isInteger(userId) || userId <= 0 ) return;\n\n await this.db.write(\n 'UPDATE apps SET owner_user_id = ? WHERE id = ? AND owner_user_id IS NULL',\n [userId, appId],\n );\n },\n\n build_canonical_app_uid_alias_key_ (oldAppUid) {\n return `${APP_UID_ALIAS_KEY_PREFIX}:${oldAppUid}`;\n },\n\n build_canonical_app_uid_alias_reverse_key_ (canonicalAppUid) {\n return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`;\n },\n\n normalize_canonical_alias_uid_list_ (value) {\n if ( ! Array.isArray(value) ) return [];\n const normalizedList = [];\n const seen = new Set();\n for ( const item of value ) {\n if ( typeof item !== 'string' || !item ) continue;\n if ( seen.has(item) ) continue;\n seen.add(item);\n normalizedList.push(item);\n }\n return normalizedList;\n },\n\n async read_canonical_app_uid_alias_ (oldAppUid) {\n if ( typeof oldAppUid !== 'string' || !oldAppUid ) return null;\n\n const services = this.context.get('services');\n const kvStore = services.get('puter-kvstore');\n const suService = services.get('su');\n if ( !kvStore || typeof kvStore.get !== 'function' ) return null;\n if ( !suService || typeof suService.sudo !== 'function' ) return null;\n\n const key = this.build_canonical_app_uid_alias_key_(oldAppUid);\n try {\n const canonicalAppUid = await suService.sudo(() => kvStore.get({ key }));\n if ( typeof canonicalAppUid === 'string' && canonicalAppUid ) {\n return canonicalAppUid;\n }\n } catch {\n // Alias reads are best-effort.\n }\n return null;\n },\n\n async write_canonical_app_uid_alias_ ({ oldAppUid, canonicalAppUid }) {\n if ( typeof oldAppUid !== 'string' || !oldAppUid ) return;\n if ( typeof canonicalAppUid !== 'string' || !canonicalAppUid ) return;\n if ( oldAppUid === canonicalAppUid ) return;\n\n const services = this.context.get('services');\n const kvStore = services.get('puter-kvstore');\n const suService = services.get('su');\n if ( !kvStore || typeof kvStore.set !== 'function' ) return;\n if ( !suService || typeof suService.sudo !== 'function' ) return;\n\n const key = this.build_canonical_app_uid_alias_key_(oldAppUid);\n const reverseKey = this.build_canonical_app_uid_alias_reverse_key_(canonicalAppUid);\n const expireAt = Math.floor(Date.now() / 1000) + APP_UID_ALIAS_TTL_SECONDS;\n try {\n await suService.sudo(async () => {\n const reverseValue = await kvStore.get({ key: reverseKey });\n const reverseAliases = this.normalize_canonical_alias_uid_list_(reverseValue);\n if ( ! reverseAliases.includes(oldAppUid) ) {\n reverseAliases.push(oldAppUid);\n }\n\n await kvStore.set({\n key,\n value: canonicalAppUid,\n expireAt,\n });\n await kvStore.set({\n key: reverseKey,\n value: reverseAliases,\n expireAt,\n });\n });\n } catch {\n // Alias writes are best-effort.\n }\n },\n\n async maybe_join_owned_hosted_index_url_app_on_create_ (entity, extra, user) {\n if ( ! user ) return;\n\n const new_index_url = await entity.get('index_url');\n const source_entity = extra.old_entity;\n const currentMysqlId = await this.resolve_entity_mysql_id_(extra.old_entity);\n const conflictRow = await this.find_index_url_conflict_({\n indexUrl: new_index_url,\n excludeMysqlId: currentMysqlId,\n });\n if ( ! conflictRow ) return;\n\n const conflictOwnerUserId = Number(conflictRow.owner_user_id);\n if (\n Number.isInteger(conflictOwnerUserId)\n && conflictOwnerUserId > 0\n && conflictOwnerUserId !== user.id\n ) {\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: new_index_url,\n app_uid: conflictRow.uid,\n });\n }\n\n if ( !Number.isInteger(conflictOwnerUserId) || conflictOwnerUserId <= 0 ) {\n await this.claim_app_ownership_by_id_for_user_({\n appId: conflictRow.id,\n userId: user.id,\n });\n }\n\n const old_entity = await this.upstream.read(conflictRow.uid);\n const owner = await old_entity?.get('owner');\n let ownerUserId = owner?.id ?? owner;\n if ( owner instanceof Entity ) {\n ownerUserId = owner.private_meta.mysql_id;\n }\n ownerUserId = Number(ownerUserId);\n if ( !old_entity || !Number.isInteger(ownerUserId) || ownerUserId !== user.id ) {\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: new_index_url,\n app_uid: conflictRow.uid,\n });\n }\n if (\n Number.isInteger(conflictOwnerUserId)\n && conflictOwnerUserId === user.id\n && !await this.is_origin_bootstrap_app_entity_(old_entity)\n ) {\n // Prevent merging arbitrary same-owner apps; only allow the\n // auto-created origin bootstrap app to be absorbed.\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: new_index_url,\n app_uid: conflictRow.uid,\n });\n }\n\n if ( source_entity ) {\n const sourceUid = await source_entity.get('uid');\n const targetUid = await old_entity.get('uid');\n const requestedName = await entity.get('name');\n\n if (\n sourceUid\n && targetUid\n && sourceUid !== targetUid\n && requestedName !== undefined\n ) {\n entity.del('name');\n if ( typeof requestedName === 'string' && requestedName.trim() ) {\n extra.joined_requested_name = requestedName.trim();\n }\n }\n\n if ( sourceUid && targetUid && sourceUid !== targetUid ) {\n extra.joined_source_app_uid = sourceUid;\n }\n }\n\n await entity.set('uid', await old_entity.get('uid'));\n extra.old_entity = old_entity;\n },\n\n async apply_joined_requested_name_ ({ canonicalUid, requestedName }) {\n if ( typeof canonicalUid !== 'string' || !canonicalUid ) return null;\n if ( typeof requestedName !== 'string' || !requestedName.trim() ) return null;\n const normalizedName = requestedName.trim();\n\n const currentRows = await this.db.read(\n 'SELECT name FROM apps WHERE uid = ? LIMIT 1',\n [canonicalUid],\n );\n const currentName = currentRows?.[0]?.name;\n if ( typeof currentName !== 'string' ) return null;\n if ( currentName === normalizedName ) return null;\n\n const conflictRows = await this.db.read(\n 'SELECT uid FROM apps WHERE name = ? AND uid != ? LIMIT 1',\n [normalizedName, canonicalUid],\n );\n if ( conflictRows.length > 0 ) {\n throw APIError.create('app_name_already_in_use', null, {\n name: normalizedName,\n });\n }\n\n await this.db.write(\n 'UPDATE apps SET name = ? WHERE uid = ? LIMIT 1',\n [normalizedName, canonicalUid],\n );\n\n return {\n oldName: currentName,\n newName: normalizedName,\n };\n },\n\n async is_origin_bootstrap_app_entity_ (entity) {\n if ( ! entity ) return false;\n const uid = await entity.get('uid');\n if ( typeof uid !== 'string' || !uid ) return false;\n if ( await entity.get('name') !== uid ) return false;\n if ( await entity.get('title') !== uid ) return false;\n const description = await entity.get('description');\n if ( typeof description !== 'string' ) return false;\n return description.startsWith('App created from origin ');\n },\n\n async ensureIndexUrlUnique (entity, extra) {\n const new_index_url = await entity.get('index_url');\n if ( ! new_index_url ) return;\n if ( ! this.is_puter_hosted_index_url_(new_index_url) ) return;\n\n if ( extra.old_entity ) {\n const old_index_url = await extra.old_entity.get('index_url');\n if ( old_index_url === new_index_url ) {\n return;\n }\n }\n\n const currentMysqlId = await this.resolve_entity_mysql_id_(extra.old_entity);\n const conflictRow = await this.find_index_url_conflict_({\n indexUrl: new_index_url,\n excludeMysqlId: currentMysqlId,\n });\n if ( conflictRow ) {\n throw APIError.create('app_index_url_already_in_use', null, {\n index_url: new_index_url,\n app_uid: conflictRow.uid,\n });\n }\n },\n };\n}\n\nmodule.exports = AppES;\n"
},
{
"path": "src/backend/src/om/entitystorage/AppLimitedES.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see Log in ';\n\n // docs\n h += '';\n h += `${config.protocol}://docs.${config.domain}/ `;\n h += ' ';\n\n // apps\n // TODO: use service for app discovery\n let apps = await db.read('SELECT * FROM apps WHERE approved_for_listing = 1');\n if ( apps.length > 0 ) {\n for ( let i = 0; i < apps.length; i++ ) {\n const app = apps[i];\n h += '';\n h += `${config.protocol}://${config.domain}/app/${app.name} `;\n h += ' ';\n }\n }\n h += ' ';\n res.set('Content-Type', 'application/xml');\n return res.send(h);\n }\n else if ( path === '/unsubscribe' ) {\n let h = '';\n if ( req.query.user_uuid === undefined )\n {\n h += 'user_uuid is required
';\n }\n else {\n // modules\n const { get_user } = require('../helpers');\n\n // get user\n const user = await get_user({ uuid: req.query.user_uuid });\n\n // more validation\n if ( ! user )\n {\n h += 'User not found.
';\n }\n else if ( user.unsubscribed === 1 )\n {\n h += 'You are already unsubscribed.
';\n }\n // mark user as confirmed\n else {\n await db.write(\n 'UPDATE `user` SET `unsubscribed` = 1 WHERE id = ?',\n [user.id],\n );\n\n invalidate_cached_user(user);\n\n // return results\n h += 'Your have successfully unsubscribed from all emails.
';\n }\n }\n\n h += '';\n res.send(h);\n }\n else if ( path === '/confirm-email-by-token' ) {\n let h = '';\n if ( req.query.user_uuid === undefined )\n {\n h += 'user_uuid is required
';\n }\n else if ( req.query.token === undefined )\n {\n h += 'token is required
';\n }\n else {\n // modules\n const { get_user } = require('../helpers');\n\n // get user\n const user = await get_user({ uuid: req.query.user_uuid, force: true });\n\n // more validation\n if ( user === undefined || user === null || user === false )\n {\n h += 'user not found.
';\n }\n else if ( user.email_confirmed === 1 )\n {\n h += 'Email already confirmed.
';\n }\n else if ( user.email_confirm_token !== req.query.token )\n {\n h += 'invalid token.
';\n }\n // mark user as confirmed\n else {\n // This IIFE is here to return early on conditions, and\n // avoid further nested branching. This is a temporary\n // solution; next time this code should be refactored.\n await (async () => {\n const svc_cleanEmail = req.services.get('clean-email');\n const clean_email = svc_cleanEmail.clean(user.email);\n // If other users have the same CONFIRMED email, display an error\n const maybe_rows = await db.read(\n `SELECT EXISTS(\n SELECT 1 FROM user WHERE (email=? OR clean_email=?)\n AND email_confirmed=1\n AND password IS NOT NULL\n ) AS email_exists`,\n [user.email, clean_email],\n );\n if ( maybe_rows[0]?.email_exists ) {\n // TODO: maybe display the username of that account\n h += '' +\n 'This email was confirmed on a different account.
';\n return;\n }\n\n // If other users have the same unconfirmed email, revoke it\n await db.write(\n 'UPDATE `user` SET `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `unconfirmed_change_email` = ?',\n [user.email],\n );\n\n // update user\n await db.write(\n 'UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ?',\n [user.id],\n );\n invalidate_cached_user(user);\n\n // send realtime success msg to client\n const svc_socketio = req.services.get('socketio');\n svc_socketio.send({ room: user.id }, 'user.email_confirmed', {});\n\n // return results\n h += 'Your email has been successfully confirmed.
';\n\n const svc_event = req.services.get('event');\n svc_event.emit('user.email-confirmed', {\n user_uid: user.uuid,\n email: user.email,\n });\n })();\n }\n }\n\n h += '';\n res.send(h);\n }\n // ------------------------\n // /assets/\n // ------------------------\n else if ( path.startsWith('/assets/') ) {\n path = PathBuilder.resolve(path);\n return res.sendFile(path, { root: `${__dirname }../../public` }, function (err) {\n if ( err && err.statusCode ) {\n return res.status(err.statusCode).send('Error /public/');\n }\n });\n }\n // ------------------------\n // GUI\n // ------------------------\n else {\n let app;\n let canonical_url = config.origin + path;\n let app_name, app_title, app_description, app_icon, app_social_media_image;\n let launch_options = {\n on_initialized: [],\n };\n\n // default title\n app_title = config.title;\n\n // /action/\n if ( path.startsWith('/action/') || path.startsWith('/@') ) {\n path = '/';\n }\n // /settings\n else if ( path.startsWith('/settings') ) {\n path = '/';\n }\n // /dashboard\n else if ( path === '/dashboard' || path === '/dashboard/' ) {\n path = '/';\n }\n // /app/\n else if ( path.startsWith('/app/') ) {\n app_name = path.replace('/app/', '');\n app = await get_app({\n follow_old_names: true,\n name: app_name,\n });\n\n if ( app ) {\n // parse app metadata if available\n app.metadata = parseMetadata(app.metadata);\n // set app attributes to be passed to the homepage service\n app_title = app.title;\n app_description = app.description;\n app_icon = app.icon;\n app_social_media_image = app.metadata?.social_image;\n }\n // 404 - Not found!\n else if ( app_name ) {\n app_title = app_name.charAt(0).toUpperCase() + app_name.slice(1);\n res.status(404);\n }\n\n path = '/';\n }\n else if ( path.startsWith('/show/') ) {\n const filepath = path.slice('/show'.length);\n launch_options.on_initialized.push({\n $: 'window-call',\n fn_name: 'launch_app',\n args: [{\n name: 'explorer',\n path: filepath,\n }],\n });\n path = '/';\n }\n\n // index.js\n if ( path === '/' ) {\n const svc_puterHomepage = Context.get('services').get('puter-homepage');\n return svc_puterHomepage.send({ req, res }, {\n title: app_title,\n description: app_description || config.short_description,\n short_description: app_description || config.short_description,\n social_media_image: app_social_media_image || config.social_media_image,\n company: 'Puter Technologies Inc.',\n canonical_url: canonical_url,\n icon: app_icon,\n app: app,\n }, launch_options);\n }\n\n // /dist/...\n else if ( path.startsWith('/dist/') || path.startsWith('/src/') ) {\n path = PathBuilder.resolve(path);\n return res.sendFile(path, { root: config.assets.gui }, function (err) {\n if ( err && err.statusCode ) {\n return res.status(err.statusCode).send('Error /gui/dist/');\n }\n });\n }\n\n // All other paths\n else {\n path = PathBuilder.resolve(path);\n return res.sendFile(path, { root: _path.join(config.assets.gui, 'src') }, function (err) {\n if ( err && err.statusCode ) {\n return res.status(err.statusCode).send('Error /gui/');\n }\n });\n }\n }\n }\n // --------------------------------------\n // Native Apps\n // --------------------------------------\n else if ( subdomain === 'viewer' || subdomain === 'editor' || subdomain === 'about' || subdomain === 'docs' ||\n subdomain === 'player' || subdomain === 'pdf' || subdomain === 'code' || subdomain === 'markus' ||\n subdomain === 'draw' || subdomain === 'camera' || subdomain === 'recorder' ||\n subdomain === 'dev-center' || subdomain === 'developer' ) {\n\n let root = PathBuilder\n .add(__dirname)\n .add(config.defaultjs_asset_path, { allow_traversal: true })\n .add('apps').add(subdomain)\n .build();\n const has_dist = ['docs', 'developer'];\n if ( has_dist.includes(subdomain) ) {\n root += '/dist';\n }\n root = _path.normalize(root);\n\n path = _path.normalize(path);\n const real_path = _path.normalize(_path.join(root, path));\n\n // Determine if the path is a directory\n // (necessary because otherwise res.sendFile() will HANG!)\n try {\n const is_dir = (await _fs.promises.stat(real_path)).isDirectory();\n if ( is_dir && !path.endsWith('/') ) {\n // Redirect to directory (use 307 to avoid browser caching)\n path += '/';\n let redirect_url = `${req.protocol }://${ req.get('host') }${path}`;\n\n // We need to add the query string to the redirect URL\n if ( req.query ) {\n const old_url = `${req.protocol }://${ req.get('host') }${req.originalUrl}`;\n redirect_url += new URL(old_url).search;\n }\n\n return res.redirect(307, redirect_url);\n }\n } catch (e) {\n console.error(e);\n return res.status(404).send('Not found');\n }\n\n try {\n return res.sendFile(path, { root }, function (err) {\n if ( err && err.statusCode ) {\n return res.status(err.statusCode).send('Error /apps/');\n }\n });\n } catch (e) {\n console.error('error from sendFile', e);\n return res.status(e.statusCode).send('Error /apps/');\n }\n }\n // --------------------------------------\n // WWW, redirect to root domain\n // --------------------------------------\n else if ( subdomain === 'www' ) {\n return res.redirect(config.origin);\n }\n //------------------------------------------\n // User-defined subdomains: *.puter.com\n // redirect to static hosting domain *.puter.site\n //------------------------------------------\n else {\n if ( req.get('host').toLowerCase().endsWith(config.domain) ) {\n return res.redirect(302, `${req.protocol }://${ req.get('host').replace(config.domain, config.static_hosting_domain) }${req.originalUrl}`);\n // replace hostname with static hosting domain and redirect to the same path\n }\n }\n});\n\nmodule.exports.catchAllRouter = router;\n"
},
{
"path": "src/backend/src/routers/apps.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see Re-validated Re-validated. Closing…
`);\n});\n\nexport default router;\n"
},
{
"path": "src/backend/src/routers/auth/request-app-root-dir.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see Your email has been successfully confirmed.
';\n return res.send(h);\n});\n\nmodule.exports = app => {\n app.use(CHANGE_EMAIL_CONFIRM);\n};\n"
},
{
"path": "src/backend/src/routers/change_username.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see Puter Driver API \n \n \n \n \n `);\n});\n"
},
{
"path": "src/backend/src/routers/file.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see Sign In Required | ${safeAppTitle} \n \n Sign in required \n ${safeAppName} requires Puter authentication before private files can load.
\n Click “Sign In with Puter” to continue.
\n \n Sign In with Puter \n Retry \n
\n \n \n \n \n \n `);\n\n res.status(200);\n res.set('Cache-Control', 'no-store');\n res.set('X-Robots-Tag', 'noindex, nofollow');\n setReferrerPolicyHeader(res);\n appendLinkHeader(\n res,\n marketplaceAppUrl ? `<${marketplaceAppUrl}>; rel=\"canonical\"` : null,\n );\n res.set('Content-Type', 'text/html; charset=UTF-8');\n return res.send(loginHtml);\n}\n\nasync function evaluatePrivateAppAccess ({ req, res, services, app, requestPath }) {\n const identity = await resolvePrivateIdentity({\n req,\n services,\n appUid: app.uid,\n });\n\n if ( ! identity.userUid ) {\n logPrivateAccessEvent('private_access.auth_required', {\n appUid: app.uid,\n userUid: null,\n requestHost: req.hostname,\n requestPath,\n source: identity.source,\n hasPrivateCookie: identity.hasPrivateCookie,\n hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie,\n });\n respondPrivateLoginBootstrap({ res, app });\n return false;\n }\n\n const eventService = services.get('event');\n const accessCheckEvent = {\n appUid: app.uid,\n userUid: identity.userUid ?? null,\n requestHost: req.hostname,\n requestPath,\n result: {\n allowed: false,\n },\n };\n\n try {\n await eventService.emit('app.privateAccess.check', accessCheckEvent);\n } catch (e) {\n logPrivateAccessEvent('private_access.entitlement_check_error', {\n appUid: app.uid,\n userUid: identity.userUid ?? null,\n requestHost: req.hostname,\n requestPath,\n source: identity.source,\n error: e?.message || String(e),\n });\n console.error('private app access check failed', e);\n }\n\n if ( ! accessCheckEvent.result.allowed ) {\n const redirectUrl = getPrivateDeniedRedirectUrl(\n app,\n accessCheckEvent.result.redirectUrl,\n );\n logPrivateAccessEvent('private_access.denied', {\n appUid: app.uid,\n userUid: identity.userUid ?? null,\n requestHost: req.hostname,\n requestPath,\n source: identity.source,\n reason: accessCheckEvent.result.reason ?? null,\n redirectUrl,\n hasPrivateCookie: identity.hasPrivateCookie,\n hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie,\n });\n const marketplaceAppUrl = getMarketplaceAppUrl(app);\n appendLinkHeader(\n res,\n marketplaceAppUrl ? `<${marketplaceAppUrl}>; rel=\"alternate\"` : null,\n );\n res.redirect(redirectUrl);\n return false;\n }\n\n const shouldRefreshPrivateCookie = identity.userUid && !identity.hasValidPrivateCookie;\n if ( identity.userUid && !identity.hasValidPrivateCookie ) {\n const authService = services.get('auth');\n const privateToken = authService.createPrivateAssetToken({\n appUid: identity.tokenAppUid || app.uid,\n userUid: identity.userUid,\n sessionUuid: identity.sessionUuid,\n subdomain: identity.subdomain,\n privateHost: identity.privateHost,\n });\n res.cookie(\n authService.getPrivateAssetCookieName(),\n privateToken,\n authService.getPrivateAssetCookieOptions({\n requestHostname: req.hostname,\n }),\n );\n\n const sanitizedUrl = stripBootstrapAuthTokenFromOriginalUrl(req.originalUrl);\n const shouldKeepBootstrapTokenInUrl = hasAppInstanceIdQueryParam(req);\n if ( sanitizedUrl && !shouldKeepBootstrapTokenInUrl ) {\n logPrivateAccessEvent('private_access.allowed_cookie_redirect', {\n appUid: app.uid,\n userUid: identity.userUid ?? null,\n requestHost: req.hostname,\n requestPath,\n source: identity.source,\n redirectUrl: sanitizedUrl,\n });\n res.redirect(sanitizedUrl);\n return false;\n }\n if ( sanitizedUrl && shouldKeepBootstrapTokenInUrl ) {\n logPrivateAccessEvent('private_access.allowed_cookie_redirect_skipped_for_app_instance', {\n appUid: app.uid,\n userUid: identity.userUid ?? null,\n requestHost: req.hostname,\n requestPath,\n source: identity.source,\n redirectUrl: sanitizedUrl,\n });\n }\n }\n\n logPrivateAccessEvent('private_access.allowed', {\n appUid: app.uid,\n userUid: identity.userUid ?? null,\n requestHost: req.hostname,\n requestPath,\n source: identity.source,\n cookieRefreshed: !!shouldRefreshPrivateCookie,\n hasPrivateCookie: identity.hasPrivateCookie,\n hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie,\n });\n return true;\n}\n\nasync function runInternal (req, res, next) {\n const isPrivateHostedRequest = hostMatchesPrivateDomain(req.hostname);\n const subdomain =\n req.is_custom_domain && !isPrivateHostedRequest ? req.hostname :\n req.subdomains[0] === 'devtest' ? 'devtest' :\n getSubdomainFromHostedRequest(req);\n\n let path = (req.baseUrl + req.path) || 'index.html';\n\n const context = Context.get();\n const services = context.get('services');\n\n const getUsernameSite = (async () => {\n if ( ! subdomain.endsWith('.at') ) return;\n const parts = subdomain.split('.');\n if ( parts.length !== 2 ) return;\n const username = parts[0];\n if ( ! username.match(usernameRegex) ) {\n return;\n }\n const filesystemService = services.get('filesystem');\n const indexNode = await filesystemService.node(new NodePathSelector(`/${username}/Public/index.html`));\n const node = await filesystemService.node(new NodePathSelector(`/${username}/Public`));\n if ( ! await indexNode.exists() ) return;\n\n return {\n name: `${username }.at`,\n uuid: uuidv5(username, AT_DIRECTORY_NAMESPACE),\n root_dir_id: await node.get('mysql-id'),\n };\n });\n\n if ( req.hostname === staticHostingDomain || req.hostname === staticHostingDomainAlt || subdomain === 'www' ) {\n\n // redirect to information page about static hosting\n return res.redirect(staticHostingBaseDomainRedirect);\n }\n\n const site =\n await getUsernameSite() ||\n await (async () => {\n const puterSiteService = services.get('puter-site');\n const site = await puterSiteService.get_subdomain(subdomain, {\n is_custom_domain: req.is_custom_domain && !isPrivateHostedRequest,\n });\n return site;\n })();\n\n if ( site === null ) {\n return res.status(404).send('Subdomain not found');\n }\n\n const subdomainOwner = await get_user({ id: site.user_id });\n if ( subdomainOwner?.suspended ) {\n // This used to be \"401 Account suspended\", but this implies\n // the client user is suspended, which is not the case.\n // Instead we simply return 404, indicating that this page\n // doesn't exist without further specifying that the owner's\n // account is suspended. (the client user doesn't need to know)\n return res.status(404).send('Subdomain not found');\n }\n\n const associatedApp = site.associated_app_id\n ? await get_app({ id: site.associated_app_id })\n : null;\n const privateApp = await resolvePrivateAppForHostedSite({\n req,\n site,\n services,\n associatedApp,\n });\n const privateAppEnabled = isPrivateApp(privateApp);\n const privateAccessGateEnabled = isPrivateAccessGateEnabled();\n\n if ( privateAppEnabled ) {\n setReferrerPolicyHeader(res);\n }\n\n if (\n privateAccessGateEnabled\n && privateAppEnabled\n && !hostMatchesPrivateDomain(req.hostname)\n ) {\n const privateHostRedirect = buildPrivateHostRedirectUrl(req, privateApp);\n if ( privateHostRedirect ) {\n logPrivateAccessEvent('private_access.host_redirect', {\n appUid: privateApp?.uid ?? null,\n requestHost: req.hostname,\n requestPath: req.path,\n redirectUrl: privateHostRedirect,\n });\n const marketplaceAppUrl = getMarketplaceAppUrl(privateApp);\n appendLinkHeader(\n res,\n marketplaceAppUrl\n ? `<${marketplaceAppUrl}>; rel=\"alternate\"`\n : null,\n );\n return res.redirect(privateHostRedirect);\n }\n logPrivateAccessEvent('private_access.host_mismatch_denied', {\n appUid: privateApp?.uid ?? null,\n requestHost: req.hostname,\n requestPath: req.path,\n });\n return res.status(403).send('Private app host mismatch');\n }\n\n if (\n site.associated_app_id &&\n !privateAppEnabled &&\n !req.query['puter.app_instance_id'] &&\n ( path === '' || path.endsWith('/') )\n ) {\n const app = associatedApp || await get_app({ id: site.associated_app_id });\n return res.redirect(`${originUrl}/app/${app.name}/`);\n }\n\n if ( path === '' ) path += '/index.html';\n else if ( path.endsWith('/') ) path += 'index.html';\n\n const resolvedUrlPath =\n resolve('/', path);\n\n const filesystemService = services.get('filesystem');\n\n let subdomainRootPath = '';\n if ( site.root_dir_id !== null && site.root_dir_id !== undefined ) {\n const node = await filesystemService.node(new NodeInternalIDSelector('mysql', site.root_dir_id));\n if ( ! await node.exists() ) {\n return res.status(502).send('subdomain is pointing to deleted directory');\n }\n if ( await node.get('type') !== TYPE_DIRECTORY ) {\n return res.status(502).send('subdomain is pointing to non-directory');\n }\n\n // Verify subdomain owner permission\n const subdomainActor = Actor.adapt(subdomainOwner);\n const aclService = services.get('acl');\n if ( ! await aclService.check(subdomainActor, node, 'read') ) {\n res.status(502).send('subdomain owner does not have access to directory');\n return;\n }\n\n subdomainRootPath = await node.get('path');\n }\n\n if ( ! subdomainRootPath ) {\n return respondHtmlError({\n html: dedent(`\n Subdomain or site is not pointing to a directory.\n `),\n }, req, res, next);\n }\n\n if ( !subdomainRootPath || subdomainRootPath === '/' ) {\n throw APIError.create('forbidden');\n }\n\n req.__puterSiteRootPath = subdomainRootPath;\n\n if ( ! privateAppEnabled ) {\n try {\n const actorContextReady = await evaluatePublicHostedActorContext({\n req,\n res,\n services,\n appUid: privateApp?.uid || associatedApp?.uid,\n });\n if ( ! actorContextReady ) return;\n } catch (e) {\n logPrivateAccessEvent('public_actor.evaluate_failed', {\n appUid: privateApp?.uid || associatedApp?.uid || null,\n requestHost: req.hostname,\n reason: getPrivateAccessRejectionReason(e),\n });\n }\n }\n\n if ( privateAccessGateEnabled && privateAppEnabled ) {\n const accessAllowed = await evaluatePrivateAppAccess({\n req,\n res,\n services,\n app: privateApp,\n requestPath: req.path,\n });\n if ( ! accessAllowed ) return;\n }\n\n const filepath = subdomainRootPath + decodeURIComponent(resolvedUrlPath);\n\n const targetNode = await filesystemService.node(new NodePathSelector(filepath));\n await targetNode.fetchEntry();\n\n if ( ! await targetNode.exists() ) {\n return await respond404({ path }, req, res, next, subdomainRootPath);\n }\n\n const targetIsDir = await targetNode.get('type') === TYPE_DIRECTORY;\n\n if ( targetIsDir && !resolvedUrlPath.endsWith('/') ) {\n return res.redirect(`${resolvedUrlPath }/`);\n }\n\n if ( targetIsDir ) {\n return await respond404({ path }, req, res, next, subdomainRootPath);\n }\n\n const contentType = contentTypeFromMime(await targetNode.get('name'));\n res.set('Content-Type', contentType);\n\n const aclConfig = {\n no_acl: true,\n actor: null,\n };\n\n if ( site.protected ) {\n const authService = req.services.get('auth');\n\n const getSiteActorFromToken = async () => {\n const siteToken = req.cookies['puter.site.token'];\n if ( ! siteToken ) return;\n\n let failed = false;\n let siteActor;\n try {\n siteActor =\n await authService.authenticate_from_token(siteToken);\n } catch (e) {\n failed = true;\n }\n\n if ( failed ) return;\n\n if ( ! siteActor ) return;\n\n // security measure: if 'puter.site.token' is set\n // to a different actor type, someone is likely\n // trying to exploit the system.\n if ( ! (siteActor.type instanceof SiteActorType) ) {\n return;\n }\n\n aclConfig.actor = siteActor;\n\n // Refresh the token if it's been 30 seconds since\n // the last request\n if (\n (Date.now() - siteActor.type.iat * 1000)\n >\n 1000 * 30\n ) {\n const siteToken = authService.get_site_app_token({\n site_uid: site.uuid,\n });\n res.cookie('puter.site.token', siteToken);\n }\n\n return true;\n };\n\n const makeSiteActorFromAppToken = async () => {\n const token = req.query['puter.auth.token'];\n\n aclConfig.no_acl = false;\n\n if ( ! token ) {\n const e = APIError.create('token_missing');\n return respondError({ req, res, e });\n }\n\n const appActor =\n await authService.authenticate_from_token(token);\n\n const userActor =\n appActor.get_related_actor(UserActorType);\n\n const permissionService = req.services.get('permission');\n const perm = await (async () => {\n if ( userActor.type.user.id === site.user_id ) {\n return {};\n }\n\n const reading = await permissionService.scan(userActor, `site:uid#${site.uuid}:access`);\n const options = PermissionUtil.reading_to_options(reading);\n return options.length > 0;\n })();\n\n if ( ! perm ) {\n const e = APIError.create('forbidden');\n respondError({ req, res, e });\n return false;\n }\n\n const siteActor = await Actor.create(SiteActorType, { site });\n aclConfig.actor = siteActor;\n\n // This subdomain is allowed to keep the site actor token,\n // so we send it here as a cookie so other html files can\n // also load.\n const siteToken = authService.get_site_app_token({\n site_uid: site.uuid,\n });\n res.cookie('puter.site.token', siteToken);\n return true;\n };\n\n let ok = await getSiteActorFromToken();\n if ( ! ok ) {\n ok = await makeSiteActorFromAppToken();\n }\n if ( ! ok ) return;\n\n Object.freeze(aclConfig);\n }\n\n // Helper function to parse Range header\n const parseRangeHeader = (rangeHeader) => {\n // Check if this is a multipart range request\n if ( rangeHeader.includes(',') ) {\n // For now, we'll only serve the first range in multipart requests\n // as the underlying storage layer doesn't support multipart responses\n const firstRange = rangeHeader.split(',')[0].trim();\n const matches = firstRange.match(/bytes=(\\d+)-(\\d*)/);\n if ( ! matches ) return null;\n\n const start = parseInt(matches[1], 10);\n const end = matches[2] ? parseInt(matches[2], 10) : null;\n\n return { start, end, isMultipart: true };\n }\n\n // Single range request\n const matches = rangeHeader.match(/bytes=(\\d+)-(\\d*)/);\n if ( ! matches ) return null;\n\n const start = parseInt(matches[1], 10);\n const end = matches[2] ? parseInt(matches[2], 10) : null;\n\n return { start, end, isMultipart: false };\n };\n if ( req.headers['range'] ) {\n res.status(206);\n\n // Parse the Range header and set Content-Range\n const rangeInfo = parseRangeHeader(req.headers['range']);\n if ( rangeInfo ) {\n const { start, end, isMultipart } = rangeInfo;\n\n // For open-ended ranges, we need to calculate the actual end byte\n let actualEnd = end;\n let fileSize = null;\n\n try {\n fileSize = await targetNode.get('size');\n if ( end === null ) {\n actualEnd = fileSize - 1; // File size is 1-based, end byte is 0-based\n }\n } catch (e) {\n // If we can't get file size, we'll let the storage layer handle it\n // and not set Content-Range header\n actualEnd = null;\n fileSize = null;\n }\n\n if ( actualEnd !== null ) {\n const totalSize = fileSize !== null ? fileSize : '*';\n const contentRange = `bytes ${start}-${actualEnd}/${totalSize}`;\n res.set('Content-Range', contentRange);\n }\n\n // If this was a multipart request, modify the range header to only include the first range\n if ( isMultipart ) {\n req.headers['range'] = end !== null\n ? `bytes=${start}-${end}`\n : `bytes=${start}-`;\n }\n }\n } else {\n if ( targetNode.entry.size ) {\n res.set('x-expected-entity-length', targetNode.entry.size);\n }\n }\n res.set({ 'Accept-Ranges': 'bytes' });\n\n const llRead = new LLRead();\n // const actor = Actor.adapt(req.user);\n const stream = await llRead.run({\n no_acl: aclConfig.no_acl,\n actor: aclConfig.actor,\n fsNode: targetNode,\n ...(req.headers['range'] ? { range: req.headers['range'] } : { }),\n });\n\n // Destroy the stream if the client disconnects\n req.on('close', () => {\n stream.destroy();\n });\n\n try {\n return stream.pipe(res);\n } catch (e) {\n const handled = await respondSiteError({\n path,\n req,\n res,\n next,\n subdomainRootPath,\n });\n if ( handled ) return;\n return res.status(500).send(`Error reading file: ${ e.message}`);\n }\n}\n\nasync function respondSiteError ({ path, html, req, res, next, subdomainRootPath }) {\n const handled = await maybeRespondWithSiteConfig({\n path,\n html,\n req,\n res,\n next,\n subdomainRootPath,\n errorStatus: 500,\n });\n return handled;\n}\n\nasync function getSiteErrorConfig (req, subdomainRootPath) {\n if ( ! subdomainRootPath ) return null;\n req.__puterSiteErrorConfigCache ??= Object.create(null);\n\n if ( req.__puterSiteErrorConfigCache[subdomainRootPath] !== undefined ) {\n return req.__puterSiteErrorConfigCache[subdomainRootPath];\n }\n\n try {\n const context = Context.get();\n const services = context.get('services');\n const filesystemService = services.get('filesystem');\n\n const configPath = `${subdomainRootPath}/${puterSiteConfigFilename}`;\n const configNode = await filesystemService.node(new NodePathSelector(configPath));\n await configNode.fetchEntry();\n\n if ( ! await configNode.exists() ) {\n req.__puterSiteErrorConfigCache[subdomainRootPath] = null;\n return null;\n }\n if ( await configNode.get('type') === TYPE_DIRECTORY ) {\n req.__puterSiteErrorConfigCache[subdomainRootPath] = null;\n return null;\n }\n\n const size = Number(await configNode.get('size') ?? 0);\n if ( Number.isFinite(size) && size > puterSiteConfigMaxSize ) {\n req.__puterSiteErrorConfigCache[subdomainRootPath] = null;\n return null;\n }\n\n const llRead = new LLRead();\n const stream = await llRead.run({\n no_acl: true,\n actor: null,\n fsNode: configNode,\n });\n const buffer = await streamToBuffer(stream);\n const text = buffer.toString('utf8');\n const parsed = parseSiteErrorConfig(text);\n\n req.__puterSiteErrorConfigCache[subdomainRootPath] = parsed;\n return parsed;\n } catch {\n req.__puterSiteErrorConfigCache[subdomainRootPath] = null;\n return null;\n }\n}\n\nasync function getSiteFileNode (subdomainRootPath, sitePath) {\n const context = Context.get();\n const services = context.get('services');\n const filesystemService = services.get('filesystem');\n\n const fullPath = `${subdomainRootPath}${sitePath}`;\n const node = await filesystemService.node(new NodePathSelector(fullPath));\n await node.fetchEntry();\n if ( ! await node.exists() ) return null;\n if ( await node.get('type') === TYPE_DIRECTORY ) return null;\n return node;\n}\n\nasync function maybeRespondWithSiteConfig ({\n path,\n html,\n req,\n res,\n next,\n subdomainRootPath,\n errorStatus,\n}) {\n if ( ! subdomainRootPath ) return false;\n\n const parsedConfig = await getSiteErrorConfig(req, subdomainRootPath);\n if ( ! parsedConfig ) return false;\n\n const rule = getSiteErrorRule(parsedConfig, errorStatus);\n if ( ! rule ) return false;\n\n const responseStatus = rule.status ?? errorStatus;\n if ( rule.file ) {\n const node = await getSiteFileNode(subdomainRootPath, rule.file);\n if ( node ) {\n await streamSiteFile({\n req,\n res,\n fsNode: node,\n status: responseStatus,\n });\n return true;\n }\n }\n\n if ( rule.status !== null && rule.status !== undefined ) {\n respondHtmlError({ path, html, status: responseStatus }, req, res, next);\n return true;\n }\n\n return false;\n}\n\nasync function streamSiteFile ({ req, res, fsNode, status }) {\n res.status(status);\n const contentType =\n contentTypeFromMime(await fsNode.get('name')) ||\n 'application/octet-stream';\n res.set('Content-Type', contentType);\n\n const llRead = new LLRead();\n const stream = await llRead.run({\n no_acl: true,\n actor: null,\n fsNode,\n });\n\n req.on('close', () => {\n stream.destroy();\n });\n\n return stream.pipe(res);\n}\n\nasync function respond404 ({ path, html }, req, res, next, subdomainRootPath) {\n const handled = await maybeRespondWithSiteConfig({\n path,\n html,\n req,\n res,\n next,\n subdomainRootPath,\n errorStatus: 404,\n });\n if ( handled ) return;\n\n if ( subdomainRootPath ) {\n const custom404Node = await getSiteFileNode(subdomainRootPath, '/404.html');\n if ( custom404Node ) {\n return streamSiteFile({\n req,\n res,\n fsNode: custom404Node,\n status: 404,\n });\n }\n }\n\n return respondHtmlError({ path, html, status: 404 }, req, res, next);\n}\n\nfunction respondHtmlError ({ path, html, status = 404 }, req, res, _next) {\n res.status(status);\n res.set('Content-Type', 'text/html; charset=UTF-8');\n res.write(``);\n res.write(`
${status} `);\n res.write('
');\n if ( status === 404 && path ) {\n if ( path === '/index.html' ) {\n res.write('index.html Not Found');\n } else {\n res.write('Not Found');\n }\n } else {\n res.write(html || 'Request failed');\n }\n res.write('
');\n\n res.write('
${user.username} . Please check your email for instructions on how to reset your password.` });\n }\n else\n {\n return res.send({ message: `Password recovery email sent to ${user.email} . Please check your email for instructions on how to reset your password.` });\n }\n\n } catch (e) {\n console.log(e);\n return res.status(400).send(e);\n }\n\n});\n\nmodule.exports = router;\n"
},
{
"path": "src/backend/src/routers/set-desktop-bg.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } The created user, or null on failure (e.g. email already registered).\n */\nasync function signup_create_new_user (services, options) {\n const { providerId, userinfo } = options;\n if ( !providerId || !userinfo ) {\n // Form signup: to be refactored from signup.js; not implemented here yet.\n return null;\n }\n\n const db = await services.get('database').get(DB_WRITE, 'auth');\n const svc_group = services.get('group');\n const svc_user = services.get('user');\n const svc_oidc = services.get('oidc');\n if ( ! svc_oidc ) return null;\n\n const claims = userinfo;\n let username = (claims.name || claims.email || '').toString().trim();\n if ( username ) {\n username = username.replace(/\\s+/g, '_').replace(/[^a-zA-Z0-9_-]/g, '');\n if ( username.length > 45 ) username = username.slice(0, 45);\n }\n if ( !username || !/^\\w+$/.test(username) ) {\n let candidate;\n do {\n candidate = generate_identifier();\n const [r] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [candidate]);\n if ( ! r ) username = candidate;\n } while ( !username );\n } else {\n const [existing] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [username]);\n if ( existing ) {\n let suffix = 1;\n while ( true ) {\n const candidate = `${username}${suffix}`;\n const [r] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [candidate]);\n if ( ! r ) {\n username = candidate; break;\n }\n suffix++;\n }\n }\n }\n\n const email = (claims.email || '').toString().trim() || null;\n const clean_email = email ? email.toLowerCase().trim() : null;\n if ( clean_email ) {\n const [existingEmail] = await db.pread('SELECT 1 FROM user WHERE clean_email = ? LIMIT 1', [clean_email]);\n if ( existingEmail ) {\n return null; // email already registered; caller should return error\n }\n }\n\n const user_uuid = uuidv4();\n const email_confirm_code = String(Math.floor(100000 + Math.random() * 900000));\n const email_confirm_token = uuidv4();\n\n await db.write(`INSERT INTO user (\n username, email, clean_email, password, uuid, referrer,\n email_confirm_code, email_confirm_token, free_storage,\n referred_by, email_confirmed, requires_email_confirmation\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n [\n username,\n email,\n clean_email,\n null,\n user_uuid,\n null,\n email_confirm_code,\n email_confirm_token,\n config.storage_capacity,\n null,\n 1,\n 0,\n ]);\n const [inserted] = await db.pread('SELECT id FROM user WHERE uuid = ? LIMIT 1', [user_uuid]);\n const user_id = inserted.id;\n\n await svc_oidc.linkProviderToUser(user_id, providerId, claims.sub, null);\n\n await svc_group.add_users({\n uid: config.default_user_group,\n users: [username],\n });\n\n const [user] = await db.pread('SELECT * FROM user WHERE id = ? LIMIT 1', [user_id]);\n if ( user && user.metadata && typeof user.metadata === 'string' ) {\n user.metadata = JSON.parse(user.metadata);\n } else if ( user && !user.metadata ) {\n user.metadata = {};\n }\n await svc_user.generate_default_fsentries({ user });\n\n return user;\n}\n\nexport default signup_create_new_user;\n"
},
{
"path": "src/backend/src/routers/sites.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see (\n name: T\n ): T extends `${infer R extends keyof ServicesMap}`\n ? ServicesMap[R]\n : unknown;\n };\n config: Record & { services?: Record; server_id?: string };\n name?: string;\n args?: any;\n context: { get (key: string): any };\n}\n\nexport type EventHandler = (id: string, ...args: any[]) => any;\n\nexport interface Logger {\n debug: (...args: any[]) => any;\n info: (...args: any[]) => any;\n [key: string]: any;\n}\n\nexport class BaseService {\n constructor (service_resources: ServiceResources, ...a: any[]);\n\n args: any;\n service_name: string;\n services: ServiceResources['services'];\n config: Record;\n global_config: ServiceResources['config'];\n context: ServiceResources['context'];\n log: Logger;\n errors: any;\n\n as (interfaceName: string): Record;\n\n run_as_early_as_possible (): Promise;\n construct (): Promise;\n init (): Promise;\n __on (id: string, args: any[]): Promise;\n protected __get_event_handler (id: string): EventHandler;\n\n protected _run_as_early_as_possible? (args?: any): any;\n protected _construct? (args?: any): any;\n protected _init? (args?: any): any;\n protected _get_merged_static_object? (key: string): Record;\n\n static LOG_DEBUG?: boolean;\n static CONCERN?: string;\n}\n\nexport default BaseService;\n"
},
{
"path": "src/backend/src/services/BaseService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves when construction is complete.\n */\n async construct () {\n const useapi = this.context.get('useapi');\n const use = this._get_merged_static_object('USE');\n for ( const [key, value] of Object.entries(use) ) {\n this[key] = useapi.use(value);\n }\n await (this._construct || NOOP).call(this, this.args);\n }\n\n /**\n * Performs the initialization phase of the service lifecycle.\n * This method sets up logging and error handling for the service,\n * then calls the service-specific initialization logic if defined.\n *\n * @async\n * @memberof BaseService\n * @instance\n * @returns {Promise} A promise that resolves when initialization is complete.\n */\n async init () {\n const services = this.services;\n const log_fields = {};\n if ( this.constructor.CONCERN ) {\n log_fields.concern = this.constructor.CONCERN;\n }\n this.log = services.get('log-service').create(this.service_name, log_fields);\n\n // INFO logs are treated as DEBUG logs instead if...\n if (\n // The configuration file explicitly says to do so\n this.config.log_debug ||\n // The class has `static LOG_DEBUG = true`; AND,\n // the configuration file does NOT explicitly say NOT to do this\n (!this.config.log_info && this.constructor.LOG_DEBUG)\n ) {\n this.log.info = this.log.debug;\n }\n this.errors = services.get('error-service').create(this.log);\n\n await (this._init || NOOP).call(this, this.args);\n }\n\n /**\n * Handles an event by retrieving the appropriate event handler\n * and executing it with the provided arguments.\n *\n * @param {string} id - The identifier of the event to handle.\n * @param {Array} args - The arguments to pass to the event handler.\n * @returns {Promise} The result of the event handler execution.\n */\n async __on (id, args) {\n const handler = this.__get_event_handler(id);\n\n return await handler(id, ...args);\n }\n\n __get_event_handler (id) {\n return this[`__on_${id}`]?.bind?.(this)\n || this.constructor[`__on_${id}`]?.bind?.(this.constructor)\n || NOOP;\n }\n}\n\nmodule.exports = BaseService;\nmodule.exports.BaseService = BaseService;\n"
},
{
"path": "src/backend/src/services/BootScriptService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n async '__on_boot.ready' () {\n const args = Context.get('args');\n if ( ! args['boot-script'] ) return;\n const script_name = args['boot-script'];\n\n const require = this.require;\n const fs = require('fs');\n const boot_json_raw = fs.readFileSync(script_name, 'utf8');\n const boot_json = JSON.parse(boot_json_raw);\n await this.run_script(boot_json);\n }\n\n /**\n * Executes a series of commands defined in a JSON boot script.\n *\n * This method processes each command in the boot_json array.\n * If the command is recognized within the predefined scope, it will be executed.\n * If not, an error is thrown.\n *\n * @param {Array} boot_json - An array of commands to execute.\n * @throws {Error} Thrown if an unknown command is encountered.\n */\n async run_script (boot_json) {\n const scope = {\n runner: 'boot-script',\n 'end-puter-process': ({ args }) => {\n const svc_shutdown = this.services.get('shutdown');\n svc_shutdown.shutdown(args[0]);\n },\n };\n\n for ( const statement of boot_json ) {\n const [cmd, ...args] = statement;\n if ( ! scope[cmd] ) {\n throw new Error(`Unknown command: ${cmd}`);\n }\n await scope[cmd]({ scope, args });\n }\n }\n}\n\nmodule.exports = {\n BootScriptService,\n};\n"
},
{
"path": "src/backend/src/services/ChatAPIService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n async '__on_install.routes' (_, { app }) {\n // Create a router for chat API endpoints\n const router = (() => {\n const require = this.require;\n const express = require('express');\n return express.Router();\n })();\n\n // Register the router with the Express app\n app.use('/puterai', router);\n\n // Install endpoints\n this.install_chat_endpoints_({ router });\n }\n\n /**\n * Installs chat API endpoints on the provided router\n * @param {Object} options Options object\n * @param {express.Router} options.router Express router to install endpoints on\n * @private\n */\n install_chat_endpoints_ ({ router }) {\n const Endpoint = this.require('Endpoint');\n router.use(require('../routers/puterai/openai/completions'));\n router.use(require('../routers/puterai/openai/chat_completions'));\n // Endpoint to list available AI chat models\n Endpoint({\n route: '/chat/models',\n methods: ['GET'],\n handler: async (req, res) => {\n try {\n // Use SUService to access AIChatService as system user\n const svc_su = this.services.get('su');\n const models = await svc_su.sudo(async () => {\n const svc_aiChat = this.services.get('ai-chat');\n // Return the simple model list which contains basic model information\n return svc_aiChat.list();\n });\n\n // Return the list of models\n res.json({ models: models.filter(e => !['costly', 'fake', 'abuse', 'model-fallback-test-1'].includes(e)) });\n } catch ( error ) {\n this.log.error('Error fetching models:', error);\n throw APIError.create('internal_server_error');\n }\n },\n }).attach(router);\n\n // Endpoint to get detailed information about available AI chat models\n Endpoint({\n route: '/chat/models/details',\n methods: ['GET'],\n handler: async (req, res) => {\n try {\n // Use SUService to access AIChatService as system user\n const svc_su = this.services.get('su');\n const models = await svc_su.sudo(async () => {\n const svc_aiChat = this.services.get('ai-chat');\n // Return the detailed model list which includes cost and capability information\n return svc_aiChat.models();\n });\n\n // Return the detailed list of models\n res.json({ models: models.filter((e) => !['costly', 'fake', 'abuse', 'model-fallback-test-1'].includes(e.id)) });\n } catch ( error ) {\n this.log.error('Error fetching model details:', error);\n throw APIError.create('internal_server_error');\n }\n },\n }).attach(router);\n\n Endpoint({\n route: '/image/models',\n methods: ['GET'],\n handler: async (req, res) => {\n try {\n // Use SUService to access AIImageGenerationService as system user\n const svc_su = this.services.get('su');\n const models = await svc_su.sudo(async () => {\n const svc_imageGen = this.services.get('ai-image');\n // Return the simple model list which contains basic model information\n return svc_imageGen.list();\n });\n // Return the list of models\n res.json({ models });\n } catch ( error ) {\n this.log.error('Error fetching image models:', error);\n throw APIError.create('internal_server_error');\n }\n },\n }).attach(router);\n\n Endpoint({\n route: '/image/models/details',\n methods: ['GET'],\n handler: async (req, res) => {\n try {\n // Use SUService to access AIImageGenerationService as system user\n const svc_su = this.services.get('su');\n const models = await svc_su.sudo(async () => {\n const svc_imageGen = this.services.get('ai-image');\n // Return the detailed model list which includes cost and capability information\n return svc_imageGen.models();\n });\n // Return the detailed list of models\n res.json({ models });\n } catch ( error ) {\n this.log.error('Error fetching image model details:', error);\n throw APIError.create('internal_server_error');\n }\n },\n }).attach(router);\n\n Endpoint({\n route: '/video/models/details',\n methods: ['GET'],\n handler: async (req, res) => {\n try {\n const svc_su = this.services.get('su');\n const models = await svc_su.sudo(async () => {\n const items = [];\n if ( this.services.has('openai-video-generation') ) {\n const svc_video = this.services.get('openai-video-generation');\n if ( typeof svc_video.models === 'function' ) {\n items.push(...await svc_video.models());\n }\n }\n if ( this.services.has('together-video-generation') ) {\n const svc_video = this.services.get('together-video-generation');\n if ( typeof svc_video.models === 'function' ) {\n items.push(...await svc_video.models());\n }\n }\n return items;\n });\n res.json({ models });\n } catch ( error ) {\n this.log.error('Error fetching video model details:', error);\n throw APIError.create('internal_server_error');\n }\n },\n }).attach(router);\n\n Endpoint({\n route: '/video/models',\n methods: ['GET'],\n handler: async (req, res) => {\n try {\n const svc_su = this.services.get('su');\n const models = await svc_su.sudo(async () => {\n const items = [];\n if ( this.services.has('openai-video-generation') ) {\n const svc_video = this.services.get('openai-video-generation');\n if ( typeof svc_video.models === 'function' ) {\n items.push(...(await svc_video.models()).map(model => model.puterId || model.id));\n }\n }\n if ( this.services.has('together-video-generation') ) {\n const svc_video = this.services.get('together-video-generation');\n if ( typeof svc_video.models === 'function' ) {\n items.push(...(await svc_video.models()).map(model => model.id));\n }\n }\n return items;\n });\n res.json({ models });\n } catch ( error ) {\n this.log.error('Error fetching video models:', error);\n throw APIError.create('internal_server_error');\n }\n },\n }).attach(router);\n }\n}\n\nmodule.exports = {\n ChatAPIService,\n};\n"
},
{
"path": "src/backend/src/services/ChatAPIService.test.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n */\n _construct () {\n this.named_rules = this.constructor.NAMED_RULES;\n this.providers = this.constructor.PROVIDERS;\n this.domain_to_provider = this.constructor.DOMAIN_TO_PROVIDER;\n this.domain_nondistinct = this.constructor.DOMAIN_NONDISTINCT;\n }\n\n /**\n * Cleans an email address by applying provider-specific rules and standardizations\n * @param {string} email - The email address to clean\n * @returns {string} The cleaned email address with applied rules and standardizations\n *\n * Splits email into local and domain parts, applies provider-specific rules like:\n * - Removing dots for certain providers (Gmail, iCloud)\n * - Handling subaddressing (removing +suffix)\n * - Normalizing domains (e.g. googlemail.com -> gmail.com)\n */\n clean (email) {\n const eml = (() => {\n const [local, domain] = email.split('@');\n return { local, domain };\n })();\n\n if ( this.domain_nondistinct[eml.domain] ) {\n eml.domain = this.domain_nondistinct[eml.domain];\n }\n\n const rules = [\n 'remove_subaddressing',\n ];\n\n const provider = this.domain_to_provider[eml.domain] || eml.domain;\n const provider_info = this.providers[provider];\n if ( provider_info ) {\n provider_info.rules = provider_info.rules || [];\n provider_info.rmrules = provider_info.rmrules || [];\n\n for ( const rule_name of provider_info.rules ) {\n rules.push(rule_name);\n }\n\n for ( const rule_name of provider_info.rmrules ) {\n const idx = rules.indexOf(rule_name);\n if ( idx !== -1 ) {\n rules.splice(idx, 1);\n }\n }\n }\n\n for ( const rule_name of rules ) {\n const rule = this.named_rules[rule_name];\n rule.rule({ eml });\n }\n\n return `${eml.local }@${ eml.domain}`;\n }\n\n /**\n * Validates an email address against blocked domains and custom validation rules\n * @param {string} email - The email address to validate\n * @returns {Promise} True if email is valid, false if blocked or invalid\n * @description First cleans the email, then checks against blocked domains from config.\n * Emits 'email.validate' event to allow custom validation rules. Event handlers can\n * set event.allow=false to reject the email.\n */\n async validate (email) {\n if ( this?.global_config?.env === 'dev' ) return true;\n\n email = this.clean(email);\n const config = this.global_config;\n\n if ( Array.isArray(config.blocked_email_domains) ) {\n for ( const suffix of config.blocked_email_domains ) {\n if ( email.endsWith(suffix) ) {\n return false;\n }\n }\n }\n\n const svc_event = this.services.get('event');\n const event = { allow: true, email };\n await svc_event.emit('email.validate', event);\n\n if ( ! event.allow ) return false;\n\n return true;\n }\n\n}\n\nmodule.exports = { CleanEmailService };\n"
},
{
"path": "src/backend/src/services/CleanEmailService.test.ts",
"content": "import { describe, expect, it } from 'vitest';\nimport { createTestKernel } from '../../tools/test.mjs';\nimport { CleanEmailService } from './CleanEmailService.js';\n\ndescribe('CleanEmailService', () => {\n it('should clean email addresses correctly', async () => {\n const testKernel = await createTestKernel({\n serviceMap: {\n 'clean-email': CleanEmailService,\n },\n });\n\n const cleanEmailService = testKernel.services!.get('clean-email') as CleanEmailService;\n\n const cases = [\n {\n email: 'bob.ross+happy-clouds@googlemail.com',\n expected: 'bobross@gmail.com',\n },\n {\n email: 'under.rated+email-service@yahoo.com',\n expected: 'under.rated+email-service@yahoo.com',\n },\n {\n email: 'the-absolute+best@protonmail.com',\n expected: 'the-absolute@protonmail.com',\n },\n ];\n\n for ( const { email, expected } of cases ) {\n const cleaned = cleanEmailService.clean(email);\n expect(cleaned).toBe(expected);\n }\n });\n});\n"
},
{
"path": "src/backend/src/services/ClientOperationService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves to the created ClientOperationTracker instance.\n */\n async add_operation (parameters) {\n const tracker = new ClientOperationTracker(parameters);\n\n return tracker;\n }\n\n ckey (key) {\n return `${CONTEXT_KEY }:${ key}`;\n }\n}\n\nmodule.exports = {\n ClientOperationService,\n};\n"
},
{
"path": "src/backend/src/services/ClientOperationService.test.ts",
"content": "import { describe, expect, it } from 'vitest';\nimport { ClientOperationService } from './ClientOperationService';\n\ndescribe('ClientOperationService', async () => {\n // ClientOperationService doesn't extend BaseService, so we can't use init\n // We need to create it directly\n const services = { _instances: {} };\n const clientOperationService = new ClientOperationService({ services });\n\n it('should be instantiated', () => {\n expect(clientOperationService).toBeDefined();\n expect(clientOperationService.operations_).toBeDefined();\n });\n\n it('should have operations array', () => {\n expect(clientOperationService.operations_).toBeDefined();\n expect(Array.isArray(clientOperationService.operations_)).toBe(true);\n });\n\n it('should create operation with default parameters', async () => {\n const tracker = await clientOperationService.add_operation({});\n \n expect(tracker).toBeDefined();\n expect(tracker.name).toBe('untitled');\n expect(Array.isArray(tracker.tags)).toBe(true);\n expect(tracker.tags.length).toBe(0);\n expect(tracker.frame).toBe(null);\n expect(tracker.metadata).toBeDefined();\n expect(typeof tracker.metadata).toBe('object');\n expect(Array.isArray(tracker.objects)).toBe(true);\n });\n\n it('should create operation with name', async () => {\n const tracker = await clientOperationService.add_operation({\n name: 'test-operation',\n });\n \n expect(tracker.name).toBe('test-operation');\n });\n\n it('should create operation with tags', async () => {\n const tags = ['tag1', 'tag2', 'tag3'];\n const tracker = await clientOperationService.add_operation({\n tags,\n });\n \n expect(tracker.tags).toEqual(tags);\n });\n\n it('should create operation with frame', async () => {\n const frame = { type: 'test-frame' };\n const tracker = await clientOperationService.add_operation({\n frame,\n });\n \n expect(tracker.frame).toBe(frame);\n });\n\n it('should create operation with metadata', async () => {\n const metadata = { key1: 'value1', key2: 'value2' };\n const tracker = await clientOperationService.add_operation({\n metadata,\n });\n \n expect(tracker.metadata).toEqual(metadata);\n });\n\n it('should create operation with objects', async () => {\n const objects = [{ id: 1 }, { id: 2 }];\n const tracker = await clientOperationService.add_operation({\n objects,\n });\n \n expect(tracker.objects).toEqual(objects);\n });\n\n it('should create operation with all parameters', async () => {\n const params = {\n name: 'full-operation',\n tags: ['full', 'test'],\n frame: { type: 'frame' },\n metadata: { meta: 'data' },\n objects: [{ obj: 1 }],\n };\n \n const tracker = await clientOperationService.add_operation(params);\n \n expect(tracker.name).toBe(params.name);\n expect(tracker.tags).toEqual(params.tags);\n expect(tracker.frame).toBe(params.frame);\n expect(tracker.metadata).toEqual(params.metadata);\n expect(tracker.objects).toEqual(params.objects);\n });\n\n it('should create multiple operations', async () => {\n const tracker1 = await clientOperationService.add_operation({ name: 'op1' });\n const tracker2 = await clientOperationService.add_operation({ name: 'op2' });\n const tracker3 = await clientOperationService.add_operation({ name: 'op3' });\n \n expect(tracker1.name).toBe('op1');\n expect(tracker2.name).toBe('op2');\n expect(tracker3.name).toBe('op3');\n });\n\n it('should have ckey method', () => {\n expect(clientOperationService.ckey).toBeDefined();\n expect(typeof clientOperationService.ckey).toBe('function');\n });\n\n it('should generate context key with ckey', () => {\n const key = clientOperationService.ckey('test-key');\n \n expect(key).toBeDefined();\n expect(typeof key).toBe('string');\n expect(key).toContain('test-key');\n });\n\n it('should generate different keys for different inputs', () => {\n const key1 = clientOperationService.ckey('key1');\n const key2 = clientOperationService.ckey('key2');\n \n expect(key1).not.toBe(key2);\n });\n});\n\n"
},
{
"path": "src/backend/src/services/CommandService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see }\n * @throws {Error} Logs any errors that occur during command execution\n */\n async execute (args, log) {\n log = log ?? console;\n const { id, name, description, handler } = this.spec_;\n try {\n await handler(args, log);\n } catch ( err ) {\n log.error(`command ${name ?? id} failed: ${err.message}`);\n log.error(err.stack);\n }\n }\n\n completeArgument (args) {\n const completer = this.spec_.completer;\n if ( completer )\n {\n return completer(args);\n }\n return [];\n }\n}\n\n/**\n* CommandService class manages the registration, execution, and handling of commands in the Puter system.\n* Extends BaseService to provide command-line interface functionality. Maintains a collection of Command\n* objects, supports command registration with namespaces, command execution with arguments, and provides\n* command lookup capabilities. Includes built-in help command functionality.\n* @extends BaseService\n*/\nclass CommandService extends BaseService {\n /**\n * Initializes the command service's internal state\n * Called during service construction to set up the empty commands array\n */\n async _construct () {\n this.commands_ = [];\n }\n\n /**\n * Add the help command to the list of commands on init\n */\n async _init () {\n this.commands_.push(new Command({\n id: 'help',\n description: 'show this help',\n handler: (args, log) => {\n log.log('available commands:');\n for ( const command of this.commands_ ) {\n log.log(`- ${command.spec_.id}: ${command.spec_.description}`);\n }\n },\n }));\n }\n\n async '__on_boot.consolidation' () {\n const svc_event = this.services.get('event');\n const svc_command = this;\n const event = {\n createCommand (name, command) {\n const serviceName = Context.get('extension_name') ?? '%missing%';\n const commandSpec = typeof command === 'function'\n ? { handler: command }\n : command;\n if ( typeof commandSpec !== 'object' ) {\n throw new Error('command must be either a function or an object');\n }\n if ( ! (typeof command.handler === 'function') ) {\n throw new Error('command should have a handler function');\n }\n svc_command.registerCommands(serviceName, [{\n id: name,\n ...commandSpec,\n }]);\n },\n };\n svc_event.emit('create.commands', event);\n }\n\n registerCommands (serviceName, commands) {\n if ( ! this.log ) {\n /* eslint-disable */\n console.error(\n 'CommandService.registerCommands was called before a logger ' +\n 'was initialied. This happens when calling registerCommands ' +\n 'in the \"construct\" phase instead of the \"init\" phase. If ' +\n 'you are migrating a legacy service that does not extend ' +\n 'BaseService, maybe the _construct hook is calling init()'\n );\n /* eslint-enable */\n process.exit(1);\n }\n for ( const command of commands ) {\n this.log.debug(`registering command ${serviceName}:${command.id}`);\n this.commands_.push(new Command({\n ...command,\n id: `${serviceName}:${command.id}`,\n }));\n }\n }\n\n /**\n * Executes a command with the given arguments and logging context\n * @param {string[]} args - Array of command arguments where first element is command name\n * @param {Object} log - Logger object for output (defaults to console if not provided)\n * @returns {Promise}\n * @throws {Error} If command execution fails\n */\n async executeCommand (args, log) {\n const [commandName, ...commandArgs] = args;\n const command = this.commands_.find(c => c.spec_.id === commandName);\n if ( ! command ) {\n log.error(`unknown command: ${commandName}`);\n return;\n }\n /**\n * Executes a command with the given arguments in a global context\n * @param {string[]} args - Array of command arguments where first element is command name\n * @param {Object} log - Logger object for output\n * @returns {Promise}\n * @throws {Error} If command execution fails\n */\n await globalThis.root_context.sub({\n injected_logger: log,\n }).arun(async () => {\n await command.execute(commandArgs, log);\n });\n }\n\n /**\n * Executes a raw command string by splitting it into arguments and executing the command\n * @param {string} text - Raw command string to execute\n * @param {object} log - Logger object for output (defaults to console if not provided)\n * @returns {Promise}\n * @todo Replace basic whitespace splitting with proper tokenizer (obvious-json)\n */\n async executeRawCommand (text, log) {\n // TODO: add obvious-json as a tokenizer\n const args = text.split(/\\s+/);\n await this.executeCommand(args, log);\n }\n\n /**\n * Gets a list of all registered command names/IDs\n * @returns {string[]} Array of command identifier strings\n */\n get commandNames () {\n return this.commands_.map(command => command.id);\n }\n\n getCommand (id) {\n return this.commands_.find(command => command.id === id);\n }\n}\n\nmodule.exports = {\n CommandService,\n};\n"
},
{
"path": "src/backend/src/services/CommandService.test.ts",
"content": "import { describe, expect, it, vi } from 'vitest';\nimport { createTestKernel } from '../../tools/test.mjs';\nimport { CommandService } from './CommandService';\n\ndescribe('CommandService', async () => {\n const testKernel = await createTestKernel({\n serviceMap: {\n commands: CommandService,\n },\n initLevelString: 'init',\n });\n\n const commandService = testKernel.services!.get('commands') as CommandService;\n\n it('should be instantiated', () => {\n expect(commandService).toBeInstanceOf(CommandService);\n });\n\n it('should have help command registered by default', () => {\n expect(commandService.commandNames).toContain('help');\n });\n\n it('should register commands', () => {\n commandService.registerCommands('test-service', [\n {\n id: 'test-cmd',\n description: 'A test command',\n handler: async () => {},\n },\n ]);\n expect(commandService.commandNames).toContain('test-service:test-cmd');\n });\n\n it('should execute registered commands', async () => {\n let executed = false;\n commandService.registerCommands('exec-test', [\n {\n id: 'exec-cmd',\n description: 'Execute test',\n handler: async () => { executed = true; },\n },\n ]);\n \n const mockLog = { error: vi.fn(), log: vi.fn() };\n await commandService.executeCommand(['exec-test:exec-cmd'], mockLog);\n expect(executed).toBe(true);\n });\n\n it('should pass arguments to command handler', async () => {\n let receivedArgs: string[] = [];\n commandService.registerCommands('args-test', [\n {\n id: 'args-cmd',\n description: 'Args test',\n handler: async (args) => { receivedArgs = args; },\n },\n ]);\n \n const mockLog = { error: vi.fn(), log: vi.fn() };\n await commandService.executeCommand(['args-test:args-cmd', 'arg1', 'arg2'], mockLog);\n expect(receivedArgs).toEqual(['arg1', 'arg2']);\n });\n\n it('should handle unknown commands', async () => {\n const mockLog = { error: vi.fn(), log: vi.fn() };\n await commandService.executeCommand(['unknown-command'], mockLog);\n expect(mockLog.error).toHaveBeenCalledWith('unknown command: unknown-command');\n });\n\n it('should execute raw commands', async () => {\n let executed = false;\n commandService.registerCommands('raw-test', [\n {\n id: 'raw-cmd',\n description: 'Raw test',\n handler: async () => { executed = true; },\n },\n ]);\n \n const mockLog = { error: vi.fn(), log: vi.fn() };\n await commandService.executeRawCommand('raw-test:raw-cmd', mockLog);\n expect(executed).toBe(true);\n });\n\n it('should get command by id', () => {\n commandService.registerCommands('get-test', [\n {\n id: 'get-cmd',\n description: 'Get test',\n handler: async () => {},\n },\n ]);\n \n const cmd = commandService.getCommand('get-test:get-cmd');\n expect(cmd).toBeDefined();\n expect(cmd?.id).toBe('get-test:get-cmd');\n });\n\n it('should execute help command', async () => {\n const mockLog = { error: vi.fn(), log: vi.fn() };\n await commandService.executeCommand(['help'], mockLog);\n expect(mockLog.log).toHaveBeenCalledWith('available commands:');\n });\n\n it('should support command completers', () => {\n commandService.registerCommands('complete-test', [\n {\n id: 'complete-cmd',\n description: 'Complete test',\n handler: async () => {},\n completer: (args) => ['option1', 'option2'],\n },\n ]);\n \n const cmd = commandService.getCommand('complete-test:complete-cmd');\n const completions = cmd?.completeArgument([]);\n expect(completions).toEqual(['option1', 'option2']);\n });\n});\n\n"
},
{
"path": "src/backend/src/services/ConfigurableCountingService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves when the database connection is established.\n * @memberof ConfigurableCountingService\n */\n async _init () {\n this.db = this.services.get('database').get(DB_WRITE, 'counting');\n }\n\n /**\n * Increments the count for a given service based on the provided parameters.\n * This method builds an SQL query to update the count and other custom values\n * in the database. It handles different SQL dialects (MySQL and SQLite) and\n * ensures that the pricing category is correctly hashed and stored.\n *\n * @param {Object} params - The parameters for incrementing the count.\n * @param {string} params.service_name - The name of the service.\n * @param {string} params.service_type - The type of the service.\n * @param {Object} params.values - The values to be incremented.\n * @throws {Error} If the service type is unknown or if there are no more available columns.\n * @returns {Promise} A promise that resolves when the count is successfully incremented.\n */\n async increment ({ service_name, service_type, values }) {\n values = values ? { ...values } : {};\n\n const now = new Date();\n const year = now.getUTCFullYear();\n const month = now.getUTCMonth() + 1;\n\n const counting_type = this.constructor.counting_types[service_type];\n if ( ! counting_type ) {\n throw new Error(`unknown counting type ${service_type}`);\n }\n\n const available_columns = {};\n for ( const k in this.constructor.sql_columns ) {\n available_columns[k] = [...this.constructor.sql_columns[k]];\n }\n\n const custom_col_names = counting_type.values.map((value, index) => {\n const column = available_columns[value.type].shift();\n if ( ! column ) {\n // TODO: this could be an init check on all the available service types\n throw new Error(`no more available columns for type ${value.type}`);\n }\n return column;\n });\n\n const custom_col_values = counting_type.values.map((value, index) => {\n return values[value.name];\n });\n\n // `pricing_category` is a JSON field. Keys from `values` used for\n // the pricing category will be removed from ths `values` object\n const pricing_category = {};\n for ( const category of counting_type.category ) {\n pricing_category[category.name] = values[category.name];\n delete values[category.name];\n }\n\n // `JSON.stringify` cannot be used here because it does not sort\n // the keys.\n const pricing_category_str = counting_type.category.map((category) => {\n return `${category.name}:${pricing_category[category.name]}`;\n }).join(',');\n\n const pricing_category_hash = hash(pricing_category_str);\n\n const actor = Context.get('actor');\n const actor_key = actor.uid;\n\n const required_data = {\n year,\n month,\n service_name,\n service_type,\n actor_key,\n pricing_category_hash,\n pricing_category: JSON.stringify(pricing_category),\n };\n\n const duplicate_update_part =\n `count = count + 1${\n custom_col_names.length > 0 ? ', ' : ''\n } ${\n custom_col_names.map((name) => `${name} = ${name} + ?`).join(', ')\n }`;\n\n const identifying_keys = [\n 'year', 'month',\n 'service_type', 'service_name',\n 'actor_key',\n 'pricing_category_hash',\n ];\n\n const sql =\n `INSERT INTO monthly_usage_counts (${\n Object.keys(required_data).join(', ')\n }, count, ${\n custom_col_names.join(', ')\n }) ` +\n `VALUES (${\n Object.keys(required_data).map(() => '?').join(', ')\n }, 1, ${custom_col_values.map(() => '?').join(', ')}) ${\n this.db.case({\n mysql: `ON DUPLICATE KEY UPDATE ${ duplicate_update_part}`,\n sqlite: `ON CONFLICT(${\n identifying_keys.map(v => `\\`${v}\\``).join(', ')\n }) DO UPDATE SET ${duplicate_update_part}`,\n })}`\n ;\n\n const value_array = [\n ...Object.values(required_data),\n ...custom_col_values,\n ...custom_col_values,\n ];\n\n await this.db.write(sql, value_array);\n }\n}\n\nmodule.exports = {\n ConfigurableCountingService,\n};\n"
},
{
"path": "src/backend/src/services/ConfigurableCountingService.test.ts",
"content": "import { describe, expect, it } from 'vitest';\nimport { createTestKernel } from '../../tools/test.mjs';\nimport * as config from '../config';\nimport { ConfigurableCountingService } from './ConfigurableCountingService';\n\ndescribe('ConfigurableCountingService', async () => {\n config.load_config({\n 'services': {\n 'database': {\n path: ':memory:',\n },\n },\n });\n\n const testKernel = await createTestKernel({\n serviceMap: {\n 'counting': ConfigurableCountingService,\n },\n initLevelString: 'init',\n testCore: true,\n });\n\n const countingService = testKernel.services!.get('counting') as ConfigurableCountingService;\n\n it('should be instantiated', () => {\n expect(countingService).toBeInstanceOf(ConfigurableCountingService);\n });\n\n it('should have counting types defined', () => {\n expect(ConfigurableCountingService.counting_types).toBeDefined();\n expect(ConfigurableCountingService.counting_types.gpt).toBeDefined();\n expect(ConfigurableCountingService.counting_types.dalle).toBeDefined();\n });\n\n it('should have sql columns defined', () => {\n expect(ConfigurableCountingService.sql_columns).toBeDefined();\n expect(ConfigurableCountingService.sql_columns.uint).toBeDefined();\n expect(ConfigurableCountingService.sql_columns.uint.length).toBe(3);\n });\n\n it('should validate GPT counting type structure', () => {\n const gptType = ConfigurableCountingService.counting_types.gpt;\n expect(gptType.category).toBeDefined();\n expect(gptType.values).toBeDefined();\n expect(gptType.category.length).toBeGreaterThan(0);\n expect(gptType.values.length).toBeGreaterThan(0);\n });\n\n it('should validate DALL-E counting type structure', () => {\n const dalleType = ConfigurableCountingService.counting_types.dalle;\n expect(dalleType.category).toBeDefined();\n expect(dalleType.category.length).toBeGreaterThan(0);\n expect(dalleType.category.some(c => c.name === 'model')).toBe(true);\n expect(dalleType.category.some(c => c.name === 'quality')).toBe(true);\n expect(dalleType.category.some(c => c.name === 'resolution')).toBe(true);\n });\n\n it('should have gpt token value definitions', () => {\n const gptType = ConfigurableCountingService.counting_types.gpt;\n expect(gptType.values.some(v => v.name === 'input_tokens')).toBe(true);\n expect(gptType.values.some(v => v.name === 'output_tokens')).toBe(true);\n expect(gptType.values.every(v => v.type === 'uint')).toBe(true);\n });\n\n it('should have available sql columns for uint type', () => {\n const columns = ConfigurableCountingService.sql_columns.uint;\n expect(columns).toBeDefined();\n expect(Array.isArray(columns)).toBe(true);\n expect(columns.length).toBe(3);\n expect(columns.every(col => typeof col === 'string')).toBe(true);\n });\n\n it('should have model category for gpt', () => {\n const gptType = ConfigurableCountingService.counting_types.gpt;\n const modelCategory = gptType.category.find(c => c.name === 'model');\n expect(modelCategory).toBeDefined();\n expect(modelCategory!.type).toBe('string');\n });\n});\n\n"
},
{
"path": "src/backend/src/services/Container.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } A promise that resolves when all services are\n * initialized or rejects if any service initialization fails.\n */\n async init () {\n for ( const k in this.instances_ ) {\n if ( ! this.instances_[k]._run_as_early_as_possible ) continue;\n await this.instances_[k].run_as_early_as_possible();\n }\n for ( const k in this.instances_ ) {\n await this.instances_[k].construct();\n }\n const init_failures = [];\n const promises = [];\n const PARALLEL = config.experimental_parallel_init;\n for ( const k in this.instances_ ) {\n try {\n if ( PARALLEL ) promises.push(this.instances_[k].init());\n else await this.instances_[k].init();\n } catch (e) {\n init_failures.push({ k, e });\n }\n }\n if ( PARALLEL ) await Promise.all(promises);\n\n if ( init_failures.length ) {\n console.error('init failures', init_failures);\n throw new CompositeError(`failed to initialize these services: ${\n init_failures.map(({ k }) => k).join(', ')}`,\n init_failures.map(({ k, e }) => e));\n }\n }\n\n /**\n * Emits an event to all registered services.\n *\n * This method sends an event identified by `id` along with any additional arguments to all\n * services registered in the container. If a logger is available, it logs the event.\n *\n * @param {string} id - The identifier of the event.\n * @param {...*} args - Additional arguments to pass to the event handler.\n * @returns {Promise} A promise that resolves when all event handlers have completed.\n */\n async emit (id, ...args) {\n if ( this.logger ) {\n this.logger.debug(`services:event ${id}`, { args });\n }\n\n const promises = [];\n for ( const k in this.instances_ ) {\n if ( this.instances_[k].__on ) {\n promises.push(Context.arun(() => this.instances_[k].__on(id, args)));\n }\n }\n await Promise.all(promises);\n }\n}\n\n/**\n* @class ProxyContainer\n* @classdesc The ProxyContainer class is a proxy for the Container class, allowing for delegation of service management tasks.\n* It extends the functionality of the Container class by providing a delegation mechanism.\n* This class is useful for scenarios where you need to manage services through a proxy,\n* enabling additional flexibility and control over service instances.\n*/\nclass ProxyContainer {\n constructor (delegate) {\n this.delegate = delegate;\n this.instances_ = {};\n }\n set (name, instance) {\n this.instances_[name] = instance;\n }\n get (name) {\n if ( this.instances_.hasOwnProperty(name) ) {\n return this.instances_[name];\n }\n return this.delegate.get(name);\n }\n /**\n * Checks if the container has a service with the specified name.\n *\n * @param {string} name - The name of the service to check.\n * @returns {boolean} - Returns true if the service exists, false otherwise.\n */\n has (name) {\n if ( this.instances_.hasOwnProperty(name) ) {\n return true;\n }\n return this.delegate.has(name);\n }\n get values () {\n const values = {};\n Object.assign(values, this.delegate.values);\n for ( const k in this.instances_ ) {\n let k2 = k;\n\n // Replace lowerCamelCase with underscores\n // (just an idea; more effort than it's worth right now)\n // let k2 = k.replace(/([a-z])([A-Z])/g, '$1_$2')\n\n // Replace dashes with underscores\n k2 = k2.replace(/-/g, '_');\n // Convert to lower case\n k2 = k2.toLowerCase();\n\n values[k2] = this.instances_[k];\n }\n return values;\n }\n}\n\nmodule.exports = { Container, ProxyContainer };\n"
},
{
"path": "src/backend/src/services/ContextInitService.js",
"content": "/*\n * Copyright (C) 2024-present Puter Technologies Inc.\n *\n * This file is part of Puter.\n *\n * Puter is free software: you can redistribute it and/or modify\n * it under the terms of the GNU Affero General Public License as published\n * by the Free Software Foundation, either version 3 of the License, or\n * (at your option) any later version.\n *\n * This program is distributed in the hope that it will be useful,\n * but WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU Affero General Public License for more details.\n *\n * You should have received a copy of the GNU Affero General Public License\n * along with this program. If not, see } The populated output object after all\n * providers have been processed.\n */\n async get_details (context, out) {\n out = out || {};\n\n for ( const provider of this.providers_ ) {\n await provider(context, out);\n }\n\n return out;\n }\n}\n\nmodule.exports = { DetailProviderService };\n"
},
{
"path": "src/backend/src/services/DetailProviderService.test.ts",
"content": "import { describe, expect, it } from 'vitest';\nimport { createTestKernel } from '../../tools/test.mjs';\nimport { DetailProviderService } from './DetailProviderService';\n\ndescribe('DetailProviderService', async () => {\n const testKernel = await createTestKernel({\n serviceMap: {\n 'detail-provider': DetailProviderService,\n },\n initLevelString: 'init',\n });\n\n const detailProviderService = testKernel.services!.get('detail-provider') as any;\n\n it('should be instantiated', () => {\n expect(detailProviderService).toBeInstanceOf(DetailProviderService);\n });\n\n it('should have empty providers array initially', () => {\n expect(detailProviderService.providers_).toBeDefined();\n expect(Array.isArray(detailProviderService.providers_)).toBe(true);\n });\n\n it('should register a provider', () => {\n const initialLength = detailProviderService.providers_.length;\n const provider = async (context: any, out: any) => {\n out.test = 'value';\n };\n \n detailProviderService.register_provider(provider);\n \n expect(detailProviderService.providers_.length).toBe(initialLength + 1);\n });\n\n it('should get details with single provider', async () => {\n const service = testKernel.services!.get('detail-provider') as any;\n \n service.register_provider(async (context: any, out: any) => {\n out.name = context.input;\n });\n \n const result = await service.get_details({ input: 'test-name' });\n \n expect(result.name).toBe('test-name');\n });\n\n it('should get details with multiple providers', async () => {\n const service = testKernel.services!.get('detail-provider') as any;\n \n service.register_provider(async (context: any, out: any) => {\n out.field1 = 'value1';\n });\n \n service.register_provider(async (context: any, out: any) => {\n out.field2 = 'value2';\n });\n \n const result = await service.get_details({});\n \n expect(result.field1).toBe('value1');\n expect(result.field2).toBe('value2');\n });\n\n it('should allow providers to modify existing output', async () => {\n const service = testKernel.services!.get('detail-provider') as any;\n \n service.register_provider(async (context: any, out: any) => {\n out.counter = 1;\n });\n \n service.register_provider(async (context: any, out: any) => {\n out.counter = out.counter + 1;\n });\n \n const result = await service.get_details({});\n \n expect(result.counter).toBe(2);\n });\n\n it('should use provided output object', async () => {\n const service = testKernel.services!.get('detail-provider') as any;\n \n service.register_provider(async (context: any, out: any) => {\n out.added = true;\n });\n \n const existingOut = { existing: 'value' };\n const result = await service.get_details({}, existingOut);\n \n expect(result.existing).toBe('value');\n expect(result.added).toBe(true);\n });\n\n it('should handle async providers', async () => {\n const service = testKernel.services!.get('detail-provider') as any;\n \n service.register_provider(async (context: any, out: any) => {\n await new Promise(resolve => setTimeout(resolve, 10));\n out.async = true;\n });\n \n const result = await service.get_details({});\n \n expect(result.async).toBe(true);\n });\n});\n\n"
},
{
"path": "src/backend/src/services/DynamoKVStore/.gitignore",
"content": "*.js\n*.js.map"
},
{
"path": "src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts",
"content": "import { Actor } from '@heyputer/backend/src/services/auth/Actor.js';\nimport { SUService } from '@heyputer/backend/src/services/SUService';\nimport { createTestKernel } from '@heyputer/backend/tools/test.mjs';\nimport { describe, expect, it } from 'vitest';\nimport { config } from '../../loadTestConfig.js';\nimport { DynamoKVStore } from './DynamoKVStore.js';\nimport { DynamoKVStoreWrapper, IDynamoKVStoreWrapper } from './DynamoKVStoreWrapper.js';\n\ndescribe('DynamoKVStore', async () => {\n const TABLE_NAME = 'store-kv-v1';\n\n const makeActor = (userId: number | string, appUid?: string) => ({\n type: {\n user: { id: userId, uuid: String(userId) },\n ...(appUid ? { app: { uid: appUid } } : {}),\n },\n }) as Actor;\n\n const testKernel = await createTestKernel({\n serviceMap: {\n 'puter-kvstore': DynamoKVStoreWrapper,\n },\n initLevelString: 'init',\n testCore: true,\n serviceConfigOverrideMap: {\n 'services': {\n 'puter-kvstore': { tableName: TABLE_NAME },\n },\n },\n });\n\n const testSubject = testKernel.services!.get('puter-kvstore') as IDynamoKVStoreWrapper;\n const kvStore = testSubject.kvStore!;\n const su = testKernel.services!.get('su') as SUService;\n\n it('should be instantiated', () => {\n expect(testSubject).toBeInstanceOf(DynamoKVStoreWrapper);\n });\n\n it('should contain a copy of the public methods of DynamoKVStore too', () => {\n const meteringMethods = Object.getOwnPropertyNames(DynamoKVStore.prototype)\n .filter((name) => name !== 'constructor');\n const wrapperMethods = testSubject as unknown as Record;\n const missing = meteringMethods.filter((name) => typeof wrapperMethods[name] !== 'function');\n\n expect(missing).toEqual([]);\n });\n\n it('should have DynamoKVStore instantiated', async () => {\n expect(testSubject.kvStore).toBeInstanceOf(DynamoKVStore);\n });\n it('sets and retrieves values for the current actor context', async () => {\n const actor = makeActor(1);\n const key = 'greeting';\n const value = { hello: 'world' };\n\n await su.sudo(actor, () => kvStore.set({ key, value }));\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n\n expect(stored).toEqual(value);\n });\n\n it('scopes data to the app when provided', async () => {\n const userId = 2;\n const actorAppOne = makeActor(userId, 'app-one');\n const actorAppTwo = makeActor(userId, 'app-two');\n const key = 'scoped-key';\n\n await su.sudo(actorAppOne, () => kvStore.set({ key, value: 'one' }));\n await su.sudo(actorAppTwo, () => kvStore.set({ key, value: 'two' }));\n\n const fromOne = await su.sudo(actorAppOne, () => kvStore.get({ key }));\n const fromTwo = await su.sudo(actorAppTwo, () => kvStore.get({ key }));\n\n expect(fromOne).toBe('one');\n expect(fromTwo).toBe('two');\n });\n\n it('increments nested numeric paths and persists the aggregated totals', async () => {\n const actor = makeActor(3);\n const key = 'counter-key';\n\n const first = await su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { 'total': 5, 'nested.count': 2 },\n }));\n const second = await su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { 'total': 1, 'nested.count': 3 },\n }));\n\n expect(first).toMatchObject({ total: 5, nested: { count: 2 } });\n expect(second).toMatchObject({ total: 6, nested: { count: 5 } });\n\n const persisted = await su.sudo(actor, () => kvStore.get({ key }));\n expect(persisted).toMatchObject({ total: 6, nested: { count: 5 } });\n });\n\n it('decrements numeric paths via decr and keeps values in sync', async () => {\n const actor = makeActor(4);\n const key = 'decr-key';\n\n await su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { total: 5, 'nested.count': 4 },\n }));\n const afterDecr = await su.sudo(actor, () => kvStore.decr({\n key,\n pathAndAmountMap: { total: 2, 'nested.count': 1 },\n }));\n\n expect(afterDecr).toMatchObject({ total: 3, nested: { count: 3 } });\n\n const persisted = await su.sudo(actor, () => kvStore.get({ key }));\n expect(persisted).toMatchObject({ total: 3, nested: { count: 3 } });\n });\n\n it('deletes keys with del', async () => {\n const actor = makeActor(5);\n const key = 'delete-me';\n await su.sudo(actor, () => {\n return kvStore.set({ key, value: 'bye' });\n });\n\n const res = await su.sudo(actor, () => kvStore.del({ key }));\n const value = await su.sudo(actor, () => kvStore.get({ key }));\n\n expect(res).toBe(true);\n expect(value).toBeNull();\n });\n\n it('lists entries, keys, and values while omitting expired rows', async () => {\n const actor = makeActor(6);\n await su.sudo(actor, () => kvStore.set({ key: 'k1', value: 'v1' }));\n await su.sudo(actor, () => kvStore.set({ key: 'expired', value: 'gone', expireAt: Math.floor(Date.now() / 1000) - 10 }));\n\n const entries = await su.sudo(actor, () => kvStore.list({ as: 'entries' }));\n const keys = await su.sudo(actor, () => kvStore.list({ as: 'keys' }));\n const values = await su.sudo(actor, () => kvStore.list({ as: 'values' }));\n\n expect(entries).toEqual([{ key: 'k1', value: 'v1' }]);\n expect(keys).toEqual(['k1']);\n expect(values).toEqual(['v1']);\n });\n\n it('rejects invalid list selector', async () => {\n const actor = makeActor(7);\n expect(su.sudo(actor, () => kvStore.list({ as: 'bad' as never })))\n .rejects;\n });\n\n it('supports paginated list results with cursors', async () => {\n const actor = makeActor(71);\n await su.sudo(actor, () => kvStore.set({ key: 'a', value: 1 }));\n await su.sudo(actor, () => kvStore.set({ key: 'b', value: 2 }));\n await su.sudo(actor, () => kvStore.set({ key: 'c', value: 3 }));\n\n const firstPage = await su.sudo(actor, () => kvStore.list({ as: 'keys', limit: 2 })) as { items: string[]; cursor?: string };\n expect(firstPage.items).toHaveLength(2);\n expect(firstPage.cursor).toBeTypeOf('string');\n\n const secondPage = await su.sudo(actor, () => kvStore.list({ as: 'keys', limit: 2, cursor: firstPage.cursor })) as { items: string[]; cursor?: string };\n expect(secondPage.items).toHaveLength(1);\n expect(secondPage.cursor).toBeUndefined();\n\n const allKeys = [...firstPage.items, ...secondPage.items].sort();\n expect(allKeys).toEqual(['a', 'b', 'c']);\n });\n\n it('supports prefix pattern semantics', async () => {\n const actor = makeActor(72);\n const allKeys = [\n 'abc',\n 'abc123',\n 'abc123xyz',\n 'ab',\n 'key*literal',\n 'key*literal-2',\n 'k*y',\n 'k*y-extra',\n 'other',\n ];\n\n await Promise.all(allKeys.map((key, idx) => su.sudo(actor, () => kvStore.set({ key, value: idx }))));\n\n const expectedAbc = ['abc', 'abc123', 'abc123xyz'];\n const expectedKeyStar = ['key*literal', 'key*literal-2'];\n const expectedMiddleStar = ['k*y', 'k*y-extra'];\n\n const abcKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'abc' })) as string[];\n expect([...abcKeys].sort()).toEqual([...expectedAbc].sort());\n\n const abcWildcardKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'abc*' })) as string[];\n expect([...abcWildcardKeys].sort()).toEqual([...expectedAbc].sort());\n\n const keyStarKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'key**' })) as string[];\n expect([...keyStarKeys].sort()).toEqual([...expectedKeyStar].sort());\n\n const middleStarKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'k*y*' })) as string[];\n expect([...middleStarKeys].sort()).toEqual([...expectedMiddleStar].sort());\n\n const allList = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: '*' })) as string[];\n expect([...allList].sort()).toEqual([...allKeys].sort());\n });\n\n it('returns ordered values for arrays and null for expired keys', async () => {\n const actor = makeActor(8);\n const now = Math.floor(Date.now() / 1000);\n\n await su.sudo(actor, () => kvStore.set({ key: 'a', value: 1 }));\n await su.sudo(actor, () => kvStore.set({ key: 'b', value: 2, expireAt: now - 5 }));\n await su.sudo(actor, () => kvStore.set({ key: 'c', value: 3 }));\n\n const results = await su.sudo(actor, () => kvStore.get({ key: ['c', 'b', 'a'] }));\n\n expect(results).toEqual([3, null, 1]);\n });\n\n it('flush clears all keys for the actor/app combination', async () => {\n const actor = makeActor(9, 'flush-app');\n await su.sudo(actor, () => kvStore.set({ key: 'one', value: 1 }));\n await su.sudo(actor, () => kvStore.set({ key: 'two', value: 2 }));\n\n const res = await su.sudo(actor, () => kvStore.flush());\n const remaining = await su.sudo(actor, () => kvStore.list({ as: 'entries' }));\n\n expect(res).toBe(true);\n expect(remaining).toEqual([]);\n });\n\n it('expireAt and expire set timestamps that cause reads to return null', async () => {\n const actor = makeActor(10);\n const keyAt = 'expire-at';\n const keyTtl = 'expire-ttl';\n\n await su.sudo(actor, () => kvStore.set({ key: keyAt, value: 'keep' }));\n await su.sudo(actor, () => kvStore.set({ key: keyTtl, value: 'keep' }));\n\n await su.sudo(actor, () => kvStore.expireAt({ key: keyAt, timestamp: Math.floor(Date.now() / 1000) - 1 }));\n await su.sudo(actor, () => kvStore.expire({ key: keyTtl, ttl: -1 }));\n\n const valAt = await su.sudo(actor, () => kvStore.get({ key: keyAt }));\n const valTtl = await su.sudo(actor, () => kvStore.get({ key: keyTtl }));\n\n expect(valAt).toBeNull();\n expect(valTtl).toBeNull();\n });\n\n it('updates nested paths and creates missing maps', async () => {\n const actor = makeActor(12);\n const key = 'update-key';\n\n const updated = await su.sudo(actor, () => kvStore.update({\n key,\n pathAndValueMap: {\n 'profile.name': 'Ada',\n 'profile.stats.score': 7,\n 'active': true,\n },\n }));\n\n expect(updated).toMatchObject({\n profile: { name: 'Ada', stats: { score: 7 } },\n active: true,\n });\n\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n expect(stored).toMatchObject({\n profile: { name: 'Ada', stats: { score: 7 } },\n active: true,\n });\n });\n\n it('update can set ttl for the whole object', async () => {\n const actor = makeActor(13);\n const key = 'update-ttl';\n\n await su.sudo(actor, () => kvStore.update({\n key,\n pathAndValueMap: { 'count': 1 },\n ttl: -1,\n }));\n\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n expect(stored).toBeNull();\n });\n\n it('supports list index paths when updating', async () => {\n const actor = makeActor(17);\n const key = 'update-list-index';\n\n await su.sudo(actor, () => kvStore.set({\n key,\n value: { a: { b: [1, 2] } },\n }));\n\n const updated = await su.sudo(actor, () => kvStore.update({\n key,\n pathAndValueMap: { 'a.b[1]': 5 },\n }));\n\n expect((updated as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);\n\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n expect((stored as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);\n });\n\n it('adds values to nested lists and creates missing maps', async () => {\n const actor = makeActor(15);\n const key = 'add-key';\n\n const first = await su.sudo(actor, () => kvStore.add({\n key,\n pathAndValueMap: {\n 'a.b': 1,\n },\n }));\n\n expect(first).toMatchObject({ a: { b: [1] } });\n\n const second = await su.sudo(actor, () => kvStore.add({\n key,\n pathAndValueMap: {\n 'a.b': 2,\n 'a.c': ['x', 'y'],\n },\n }));\n\n expect(second).toMatchObject({ a: { b: [1, 2], c: ['x', 'y'] } });\n\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n expect(stored).toMatchObject({ a: { b: [1, 2], c: ['x', 'y'] } });\n });\n\n it('supports list index paths when appending', async () => {\n const actor = makeActor(18);\n const key = 'add-list-index';\n\n await su.sudo(actor, () => kvStore.set({\n key,\n value: { a: { b: [[1], [2]] } },\n }));\n\n const updated = await su.sudo(actor, () => kvStore.add({\n key,\n pathAndValueMap: { 'a.b[1]': 3 },\n }));\n\n expect((updated as { a?: { b?: number[][] } }).a?.b).toEqual([[1], [2, 3]]);\n\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n expect((stored as { a?: { b?: number[][] } }).a?.b).toEqual([[1], [2, 3]]);\n });\n\n it('supports nested list indexing for add, update, remove, and incr', async () => {\n const actor = makeActor(21);\n const key = 'nested-list-index';\n\n await su.sudo(actor, () => kvStore.set({\n key,\n value: { a: [1, { b: { c: [1] } }, 2] },\n }));\n\n const added = await su.sudo(actor, () => kvStore.add({\n key,\n pathAndValueMap: { 'a[1].b.c': 2 },\n }));\n expect((added as { a?: Array }).a).toEqual([1, { b: { c: [1, 2] } }, 2]);\n\n const updated = await su.sudo(actor, () => kvStore.update({\n key,\n pathAndValueMap: { 'a[1].b.c': [9] },\n }));\n expect((updated as { a?: Array }).a).toEqual([1, { b: { c: [9] } }, 2]);\n\n const removed = await su.sudo(actor, () => kvStore.remove({\n key,\n paths: ['a[1].b.c'],\n }));\n expect((removed as { a?: Array }).a).toEqual([1, { b: {} }, 2]);\n\n await su.sudo(actor, () => kvStore.set({\n key,\n value: { a: [1, { b: { c: 1 } }, 2] },\n }));\n const incrRes = await su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { 'a[1].b.c': 3 },\n }));\n expect((incrRes as { a?: Array }).a).toEqual([1, { b: { c: 4 } }, 2]);\n });\n\n it('removes nested values including indexed list paths', async () => {\n const actor = makeActor(19);\n const key = 'remove-list-index';\n\n await su.sudo(actor, () => kvStore.set({\n key,\n value: { a: { b: [1, 2, 3], c: { d: 4 }, e: 'keep' } },\n }));\n\n const updated = await su.sudo(actor, () => kvStore.remove({\n key,\n paths: ['a.b[1]', 'a.c'],\n }));\n\n expect((updated as { a?: { b?: number[]; e?: string } }).a).toEqual({ b: [1, 3], e: 'keep' });\n\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n expect((stored as { a?: { b?: number[]; e?: string } }).a).toEqual({ b: [1, 3], e: 'keep' });\n });\n\n it('rejects overlapping parent/child paths in a single request', async () => {\n const actor = makeActor(20);\n const key = 'overlap-paths';\n\n await su.sudo(actor, () => kvStore.set({\n key,\n value: { a: { b: { c: 1 } } },\n }));\n\n await expect(su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { 'a.b': 1, 'a.b.c': 1 },\n }))).rejects.toThrow(/paths overlap/i);\n\n await expect(su.sudo(actor, () => kvStore.add({\n key,\n pathAndValueMap: { 'a.b': 1, 'a.b.c': 2 },\n }))).rejects.toThrow(/paths overlap/i);\n\n await expect(su.sudo(actor, () => kvStore.update({\n key,\n pathAndValueMap: { 'a.b': 1, 'a.b.c': 2 },\n }))).rejects.toThrow(/paths overlap/i);\n\n await expect(su.sudo(actor, () => kvStore.remove({\n key,\n paths: ['a.b', 'a.b.c'],\n }))).resolves.not.toThrow();\n });\n\n it('incr initializes nested maps for missing keys', async () => {\n const actor = makeActor(14);\n const key = 'incr-missing';\n\n const first = await su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { 'a.b.c': 2, 'x': 1 },\n }));\n\n expect(first).toMatchObject({ a: { b: { c: 2 } }, x: 1 });\n\n const second = await su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { 'a.b.c': 3 },\n }));\n\n expect(second).toMatchObject({ a: { b: { c: 5 } }, x: 1 });\n });\n\n it('supports list index paths when incrementing', async () => {\n const actor = makeActor(16);\n const key = 'incr-list-index';\n\n await su.sudo(actor, () => kvStore.set({\n key,\n value: { a: { b: [1, 2] } },\n }));\n\n const updated = await su.sudo(actor, () => kvStore.incr({\n key,\n pathAndAmountMap: { 'a.b[1]': 3 },\n }));\n\n expect((updated as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);\n\n const stored = await su.sudo(actor, () => kvStore.get({ key }));\n expect((stored as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);\n });\n\n it('enforces key and value size limits', async () => {\n const actor = makeActor(11);\n const oversizedKey = 'a'.repeat(((config as unknown as Record).kv_max_key_size as number) + 1);\n const oversizedValue = 'b'.repeat(((config as unknown as Record).kv_max_value_size as number) + 1);\n\n await expect(su.sudo(actor, () => kvStore.set({ key: oversizedKey, value: 'x' })))\n .rejects\n .toThrow(/1024/i);\n\n await expect(su.sudo(actor, () => kvStore.set({ key: 'ok', value: oversizedValue })))\n .rejects\n .toThrow(/has exceeded the maximum allowed size/i);\n });\n});\n"
},
{
"path": "src/backend/src/services/DynamoKVStore/DynamoKVStore.ts",
"content": "import { Actor, SystemActorType } from '@heyputer/backend/src/services/auth/Actor.js';\nimport type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js';\nimport type { MeteringService } from '@heyputer/backend/src/services/MeteringService/MeteringService.js';\nimport { RecursiveRecord } from '@heyputer/backend/src/services/MeteringService/types.js';\nimport { Context } from '@heyputer/backend/src/util/context.js';\nimport murmurhash from 'murmurhash';\nimport type { DDBClient } from '../../clients/dynamodb/DDBClient.js';\nimport { PUTER_KV_STORE_TABLE_DEFINITION } from './tableDefinition.js';\nimport { Span } from '../../util/otelutil.js';\nimport APIError from '../../api/APIError.js';\n\nexport class DynamoKVStore {\n static GLOBAL_APP_KEY = 'os-global';\n static LEGACY_GLOBAL_APP_KEY = 'global';\n\n #ddbClient: DDBClient;\n #sqlClient: BaseDatabaseAccessService;\n #meteringService: MeteringService;\n #tableName = 'store-kv-v1';\n #pathCleanerRegex = /[:\\-+/*]/g;\n #enableMigrationFromSQL = false;\n\n constructor ({ ddbClient, sqlClient, tableName, meteringService }: { ddbClient: DDBClient, sqlClient: BaseDatabaseAccessService, tableName: string, meteringService: MeteringService }) {\n this.#ddbClient = ddbClient;\n this.#sqlClient = sqlClient;\n this.#tableName = tableName;\n this.#meteringService = meteringService;\n this.#enableMigrationFromSQL = !this.#ddbClient.config?.aws; // TODO: disable via config after some time passes\n }\n\n async createTableIfNotExists () {\n if ( ! this.#enableMigrationFromSQL ) return;\n await this.#ddbClient.createTableIfNotExists({ ...PUTER_KV_STORE_TABLE_DEFINITION, TableName: this.#tableName }, 'ttl');\n }\n\n #getNameSpace (actor: Actor) {\n if ( actor.type instanceof SystemActorType ) {\n return 'v1:system';\n } else {\n const app = actor.type?.app ?? undefined;\n const user = actor.type?.user ?? undefined;\n if ( ! user ) throw new Error('User not found');\n\n return `v1:${app ? `${user.uuid}:${app.uid}`\n : `${user.uuid}:${this.#enableMigrationFromSQL ? DynamoKVStore.LEGACY_GLOBAL_APP_KEY : DynamoKVStore.GLOBAL_APP_KEY}`}`;\n }\n }\n\n @Span('kv:get')\n async get ({ key }: { key: string | string[]; }): Promise {\n if ( key === '' ) {\n throw APIError.create('field_empty', null, {\n key: 'key',\n });\n }\n\n const actor = Context.get('actor');\n const app = actor.type?.app ?? undefined;\n const user = actor.type?.user ?? undefined;\n\n const namespace = this.#getNameSpace(actor);\n\n const multi = Array.isArray(key);\n const keys = multi ? key : [key];\n const values: unknown[] = [];\n\n let kvEntries;\n let usage;\n if ( multi ) {\n const entriesAndUsage = (await this.#getBatches(namespace, keys));\n kvEntries = entriesAndUsage.kvEntries;\n usage = entriesAndUsage.usage;\n } else {\n const res = await this.#ddbClient.get(this.#tableName, { namespace, key });\n kvEntries = res.Item ? [res.Item] : [];\n usage = res.ConsumedCapacity?.CapacityUnits ?? 0;\n }\n\n this.#meteringService.incrementUsage(actor, 'kv:read', usage || 0);\n\n for ( const key of keys ) {\n const kv_entry = kvEntries?.find(e => e.key === key);\n const time = Date.now() / 1000;\n if ( kv_entry?.ttl && kv_entry.ttl <= (time) ) {\n values.push(null);\n continue;\n }\n if ( kv_entry?.value ) {\n values.push(kv_entry.value);\n continue;\n }\n\n if ( this.#enableMigrationFromSQL ) {\n const key_hash = murmurhash.v3(key);\n const kv_row = await this.#sqlClient.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1',\n [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]);\n\n if ( kv_row[0]?.value ) {\n // update and delete from this table\n (async () => {\n await this.set({ key: kv_row[0].key, value: kv_row[0].value });\n await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?',\n [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]);\n })();\n values.push(kv_row[0]?.value);\n continue;\n }\n }\n values.push(kv_entry?.value ?? null);\n }\n return multi ? values : values[0];\n }\n /**\n *\n * @param {string} namespace\n * @param {string[]} allKeys\n * @returns\n */\n async #getBatches (namespace: string, allKeys: string[]) {\n\n const batches: string[][] = [];\n for ( let i = 0; i < allKeys.length; i += 100 ) {\n batches.push(allKeys.slice(i, i + 100));\n }\n const batchPromises = batches.map(async (keys) => {\n const requests = [...new Set(keys)].map(k => ({ table: this.#tableName, items: { namespace, key: k } }));\n const res = await this.#ddbClient.batchGet(requests);\n const kvEntries = res.Responses?.[this.#tableName];\n const usage = res.ConsumedCapacity?.reduce((acc, curr) => acc + (curr.CapacityUnits ?? 0), 0);\n return { kvEntries, usage };\n });\n\n const batchGets = await Promise.all(batchPromises);\n\n return batchGets.reduce((acc, curr) => {\n acc.kvEntries!.push(...curr?.kvEntries ?? []);\n acc.usage! += curr.usage || 0;\n return acc;\n }, { kvEntries: [], usage: 0 });\n\n }\n\n @Span('kv:set')\n async set ({ key, value, expireAt }: { key: string; value: unknown; expireAt?: number; }): Promise {\n\n const context = Context.get();\n const actor = context.get('actor');\n\n if ( key === '' ) {\n throw APIError.create('field_empty', undefined, {\n key: 'key',\n });\n }\n\n key = String(key);\n if ( Buffer.byteLength(key, 'utf8') > 1024 ) {\n throw new Error(`key is too large. Max size is ${1024}.`);\n }\n\n if ( this.#enableMigrationFromSQL ) {\n this.get({ key });\n }\n\n const namespace = this.#getNameSpace(actor);\n\n const res = await this.#ddbClient.put(this.#tableName, {\n namespace,\n key,\n value,\n ttl: expireAt,\n });\n\n this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1);\n return true;\n }\n\n @Span('kv:del')\n async del ({ key }: { key: string; }): Promise {\n const actor = Context.get('actor');\n\n const app = actor.type?.app ?? undefined;\n const user = actor.type?.user ?? undefined;\n if ( ! user ) throw new Error('User not found');\n\n const namespace = this.#getNameSpace(actor);\n\n const res = await this.#ddbClient.del(this.#tableName, {\n namespace,\n key,\n });\n\n this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1);\n\n if ( this.#enableMigrationFromSQL ) {\n const key_hash = murmurhash.v3(key);\n await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?',\n [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]);\n }\n\n return true;\n }\n\n #encodeCursor (pageKey?: Record) {\n if ( !pageKey || Object.keys(pageKey).length === 0 ) {\n return undefined;\n }\n return Buffer.from(JSON.stringify(pageKey)).toString('base64');\n }\n\n #decodeCursor (cursor?: string | Record) {\n if ( ! cursor ) {\n return undefined;\n }\n if ( typeof cursor === 'object' ) {\n return cursor;\n }\n if ( typeof cursor !== 'string' ) {\n throw APIError.create('field_invalid', undefined, {\n key: 'cursor',\n });\n }\n const trimmed = cursor.trim();\n if ( trimmed === '' ) {\n return undefined;\n }\n try {\n const decoded = Buffer.from(trimmed, 'base64').toString('utf8');\n return JSON.parse(decoded);\n } catch ( e ) {\n try {\n return JSON.parse(trimmed);\n } catch ( err ) {\n throw APIError.create('field_invalid', undefined, {\n key: 'cursor',\n });\n }\n }\n }\n\n #normalizeLimit (limit?: number) {\n if ( limit === undefined || limit === null ) {\n return undefined;\n }\n const parsed = Number(limit);\n if ( !Number.isFinite(parsed) || parsed <= 0 ) {\n throw APIError.create('field_invalid', undefined, {\n key: 'limit',\n expected: 'positive number',\n });\n }\n return Math.floor(parsed);\n }\n\n #normalizePattern (pattern?: string) {\n if ( pattern === undefined || pattern === null ) {\n return undefined;\n }\n if ( typeof pattern !== 'string' ) {\n throw APIError.create('field_invalid', undefined, {\n key: 'pattern',\n });\n }\n const trimmed = pattern.trim();\n if ( trimmed === '' ) {\n return undefined;\n }\n if ( trimmed.endsWith('*') ) {\n const prefix = trimmed.slice(0, -1);\n return prefix === '' ? undefined : prefix;\n }\n return trimmed;\n }\n\n @Span('kv:list')\n async list ({\n as,\n limit,\n cursor,\n pattern,\n }: {\n as?: 'keys' | 'values' | 'entries';\n limit?: number;\n cursor?: string | Record;\n pattern?: string;\n }): Promise<\n | string[]\n | unknown[]\n | { key: string; value: unknown; }[]\n | { items: string[]; cursor?: string; }\n | { items: unknown[]; cursor?: string; }\n | { items: { key: string; value: unknown; }[]; cursor?: string; }\n > {\n const actor = Context.get('actor');\n\n const app = actor.type?.app ?? undefined;\n const user = actor.type?.user ?? undefined;\n if ( ! user ) throw new Error('User not found');\n\n const namespace = this.#getNameSpace(actor);\n\n const normalizedLimit = this.#normalizeLimit(limit);\n const pageKey = this.#decodeCursor(cursor);\n const normalizedPattern = this.#normalizePattern(pattern);\n const paginated = normalizedLimit !== undefined || pageKey !== undefined;\n\n const entriesRes = await this.#ddbClient.query(this.#tableName,\n { namespace },\n normalizedLimit ?? 0,\n pageKey,\n '',\n false,\n normalizedPattern ? { beginsWith: { key: 'key', value: normalizedPattern } } : undefined);\n\n this.#meteringService.incrementUsage(actor, 'kv:read', entriesRes.ConsumedCapacity?.CapacityUnits ?? 1);\n\n let entries = entriesRes.Items ?? [];\n\n entries = entries?.filter(entry => {\n if ( ! entry ) {\n return false;\n }\n if ( entry.ttl && entry.ttl <= (Date.now() / 1000) ) {\n return false;\n }\n return true;\n });\n\n if ( this.#enableMigrationFromSQL && !paginated ) {\n const oldEntries = await this.#sqlClient.read('SELECT * FROM kv WHERE user_id=? AND app=?',\n [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY]);\n oldEntries.forEach(oldEntry => {\n if ( normalizedPattern && !oldEntry.kkey?.startsWith(normalizedPattern) ) {\n return;\n }\n if ( ! entries.find(e => e.key === oldEntry.kkey) ) {\n if ( oldEntry.ttl && oldEntry.ttl <= (Date.now() / 1000) ) {\n entries.push({ key: oldEntry.kkey, value: oldEntry.value });\n }\n }\n });\n }\n\n entries = entries?.map(entry => ({\n key: entry.key,\n value: entry.value,\n }));\n\n as = as || 'entries';\n\n if ( ! ['keys', 'values', 'entries'].includes(as) ) {\n throw APIError.create('field_invalid', undefined, {\n key: 'as',\n expected: '\"keys\", \"values\", or \"entries\"',\n });\n }\n\n let items: string[] | unknown[] | { key: string; value: unknown; }[] = entries;\n if ( as === 'keys' ) items = entries.map(entry => entry.key);\n else if ( as === 'values' ) items = entries.map(entry => entry.value);\n\n if ( paginated ) {\n const nextCursor = this.#encodeCursor(entriesRes.LastEvaluatedKey as Record | undefined);\n if ( nextCursor ) {\n return { items, cursor: nextCursor };\n }\n return { items };\n }\n\n return items;\n }\n\n @Span('kv:flush')\n async flush () {\n const actor = Context.get('actor');\n\n const app = actor.type.app ?? undefined;\n const user = actor.type?.user ?? undefined;\n if ( ! user ) throw new Error('User not found');\n\n const namespace = this.#getNameSpace(actor);\n\n // Query all keys\n const entriesRes = await this.#ddbClient.query(this.#tableName,\n { namespace });\n const entries = entriesRes.Items ?? [];\n const readUsage = entriesRes?.ConsumedCapacity?.CapacityUnits ?? 0;\n\n // meter usage\n this.#meteringService.incrementUsage(actor, 'kv:read', readUsage);\n\n // TODO DS: implement batch delete so its faster and less demanding on server\n const allRes = (await Promise.all(entries.map(entry => {\n try {\n return this.#ddbClient.del(this.#tableName, {\n namespace,\n key: entry.key,\n });\n } catch ( e ) {\n console.error('Error deleting key', entry.key, e);\n }\n }))).filter(Boolean);\n\n const writeUsage = allRes.reduce((acc, curr) => acc + (curr?.ConsumedCapacity?.CapacityUnits ?? 0), 0);\n\n // meter usage\n this.#meteringService.incrementUsage(actor, 'kv:write', writeUsage);\n\n if ( this.#enableMigrationFromSQL ) {\n await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=?',\n [user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY]);\n }\n\n return !!allRes;\n }\n\n @Span('kv:expireAt')\n async expireAt ({ key, timestamp }: { key: string; timestamp: number; }): Promise {\n if ( key === '' ) {\n throw APIError.create('field_empty', null, {\n key: 'key',\n });\n }\n\n timestamp = Number(timestamp);\n\n return await this.#expireAt(key, timestamp);\n }\n\n @Span('kv:expire')\n async expire ({ key, ttl }: { key: string; ttl: number; }): Promise {\n if ( key === '' ) {\n throw APIError.create('field_empty', null, {\n key: 'key',\n });\n }\n\n ttl = Number(ttl);\n\n // timestamp in seconds\n let timestamp = Math.floor(Date.now() / 1000) + ttl;\n\n return await this.#expireAt(key, timestamp);\n }\n\n async #createPaths ( namespace: string, key: string, pathList: string[]) {\n\n const nestedMapValue = (() => {\n const valueRoot: Record = {};\n let hasPaths = false;\n pathList.forEach((valPath) => {\n if ( ! valPath ) return;\n hasPaths = true;\n const chunks = valPath.split('.').filter(Boolean);\n let cursor: Record = valueRoot;\n for ( let i = 0; i < chunks.length - 1; i++ ) {\n const chunk = chunks[i];\n const existing = cursor[chunk];\n if ( !existing || typeof existing !== 'object' || Array.isArray(existing) ) {\n cursor[chunk] = {};\n }\n cursor = cursor[chunk] as Record;\n }\n });\n return hasPaths ? valueRoot : null;\n })();\n\n if ( ! nestedMapValue ) {\n return 0;\n }\n\n const isPlainObject = (value: unknown): value is Record => {\n return !!value && typeof value === 'object' && !Array.isArray(value);\n };\n\n const objectsEqual = (left: unknown, right: unknown): boolean => {\n if ( left === right ) return true;\n if ( !isPlainObject(left) || !isPlainObject(right) ) return false;\n const leftKeys = Object.keys(left);\n const rightKeys = Object.keys(right);\n if ( leftKeys.length !== rightKeys.length ) return false;\n for ( const key of leftKeys ) {\n if ( ! rightKeys.includes(key) ) return false;\n if ( ! objectsEqual(left[key], right[key]) ) return false;\n }\n return true;\n };\n\n // Collect all intermediate map paths for all entries\n const allIntermediatePaths = new Set();\n pathList.forEach((valPath) => {\n const chunks = ['value', ...valPath.split('.')].filter(Boolean);\n // For each intermediate map (excluding the leaf)\n for ( let i = 1; i < chunks.length; i++ ) {\n const subPath = chunks.slice(0, i).join('.');\n allIntermediatePaths.add(subPath);\n }\n });\n\n let writeUnits = 0;\n // Ensure each intermediate map layer exists by issuing a separate DynamoDB update for each\n const orderedPaths = [...allIntermediatePaths]\n .sort((left, right) => left.split('.').length - right.split('.').length);\n for ( const layerPath of orderedPaths ) {\n // Build attribute names for the layer\n const chunks = layerPath.split('.');\n const attrName = chunks.map((chunk) => `#${chunk}`.replaceAll(this.#pathCleanerRegex, '')).join('.');\n const expressionNames: Record = {};\n chunks.forEach((chunk) => {\n const cleanedChunk = chunk.split(/\\[\\d*\\]/g)[0];\n expressionNames[`#${cleanedChunk}`.replaceAll(this.#pathCleanerRegex, '')] = cleanedChunk;\n });\n const isRootLayer = layerPath === 'value';\n const expressionValues = isRootLayer\n ? { ':nestedMap': nestedMapValue }\n : { ':emptyMap': {} };\n const valueToken = isRootLayer ? ':nestedMap' : ':emptyMap';\n // Issue update to set layer to {} if not exists\n const layerUpsertRes = await this.#ddbClient.update(this.#tableName,\n { key, namespace },\n `SET ${attrName} = if_not_exists(${attrName}, ${valueToken})`,\n expressionValues,\n expressionNames);\n writeUnits += layerUpsertRes.ConsumedCapacity?.CapacityUnits ?? 0;\n if ( isRootLayer && objectsEqual(layerUpsertRes.Attributes?.value, nestedMapValue) ) {\n return writeUnits;\n }\n }\n return writeUnits;\n }\n\n // Ideally the paths support syntax like \"a.b[2].c\"\n @Span('kv:incr')\n async incr>({ key, pathAndAmountMap }: { key: string; pathAndAmountMap: T; }): Promise
![\"Puter.com,](\"https://assets.puter.site/puter-logo.png\"){kind=logo}
![\"screenshot\"](\"https://assets.puter.site/puter.com-screenshot-3.webp\")
![\"Puter](\"https://puter.com/images/logo.png\"){kind=logo}
\n\n