= Record<
string,
string | undefined
>,
>(
target: RequestHandler,
_context: ClassMethodDecoratorContext<
This,
(
this: This,
...args: Parameters>
) => ReturnType>
>,
) => {
_context.addInitializer(function () {
// eslint-disable-next-line no-invalid-this
const proto = Object.getPrototypeOf(this); // will be bound to class
if ( ! proto.__routes ) {
proto.__routes = [];
}
proto.__routes.push({
method,
path,
options: options as EndpointOptions | undefined,
adminUsernames: adminUsernames
? [...adminUsernames, 'admin', 'system']
: undefined,
allowedAppIds,
handler: target,
});
});
};
};
};
// HTTP method decorators
export const Get = createMethodDecorator('get');
export const Post = createMethodDecorator('post');
export const Put = createMethodDecorator('put');
export const Delete = createMethodDecorator('delete');
// TODO DS: add others as needed (patch, etc)
export class HttpError extends Error {
statusCode: number;
constructor (statusCode: StatusCodes, message: string, cause?: unknown) {
super(`${statusCode} - ${message}`, { cause });
this.statusCode = statusCode;
}
}
// Registers all routes from a decorated controller instance to an Express router
export class ExtensionController {
logger?: Console;
// TODO DS: make this work with other express-like routers
registerRoutes () {
const logger = this.logger || console;
const prefix = Object.getPrototypeOf(this).__controllerPrefix || '';
const adminsForController = Object.getPrototypeOf(this).__adminUsernames as
| string[]
| undefined;
const allowedAppIdsForController = Object.getPrototypeOf(this).__allowedAppIds as
| string[]
| undefined;
const routes: RouteMeta[] = Object.getPrototypeOf(this).__routes || [];
for ( const route of routes ) {
const fullPath = `${prefix}/${route.path}`.replace(/\/+/g, '/');
const adminsForRoute = route.adminUsernames
? adminsForController
? adminsForController.concat(route.adminUsernames)
: route.adminUsernames
: adminsForController
? adminsForController
: undefined;
const allowedAppIds = route.allowedAppIds
? allowedAppIdsForController
? allowedAppIdsForController.concat(route.allowedAppIds)
: route.allowedAppIds
: allowedAppIdsForController
? allowedAppIdsForController
: undefined;
if ( ! extension[route.method] ) {
throw new Error(`Unsupported HTTP method: ${route.method}`);
} else {
logger.log(`Registering route: [${route.method.toUpperCase()}] ${fullPath}`);
(extension[route.method] as RouterMethods[HttpMethod])(
fullPath,
route.options || {},
async (req, res, next) => {
try {
if ( adminsForRoute || allowedAppIds ) {
if ( ! req.actor ) {
throw new HttpError(StatusCodes.UNAUTHORIZED, 'Unauthenticated');
}
}
if ( adminsForRoute ) {
if ( ! adminsForRoute.includes(req.actor!.type.user.username) ) {
throw new HttpError(
StatusCodes.FORBIDDEN,
'Only admins may request this resource.',
);
}
}
if ( allowedAppIds ) {
if ( ( req.actor!.type?.app?.uid && !allowedAppIds.includes(req.actor!.type.app.uid) ) ) {
throw new HttpError(
StatusCodes.FORBIDDEN,
'This app may not request this resource.',
);
}
}
await route.handler.bind(this)(req, res, next);
} catch ( error ) {
if ( error instanceof HttpError ) {
res.status(error.statusCode).send({ error: error.message });
logger.warn('httpError:', error);
return;
}
if ( error instanceof Error ) {
res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: error.message });
logger.error('Non-http error:', error);
return;
}
res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: 'An unknown error occurred' });
logger.error('An unknown error occurred:', error);
}
},
);
}
}
}
}
================================================
FILE: extensions/extensionController/src/index.ts
================================================
import { Controller, Delete, ExtensionController, Get, HttpError, Post, Put } from './ExtensionController.js';
extension.exports = {
ExtensionController,
Controller,
Get,
Put,
Post,
Delete,
HttpError,
};
export {
Controller, Delete, ExtensionController, Get, HttpError, Post, Put,
};
================================================
FILE: extensions/extensionController/tsconfig.json
================================================
{
"compilerOptions": {
"target": "ES2024",
"module": "nodenext",
"moduleResolution": "nodenext",
"strict": true,
"forceConsistentCasingInFileNames": true,
"skipLibCheck": true,
"sourceMap": true,
"noEmitOnError": true,
"noImplicitAny": false,
"allowJs": true,
"checkJs": false,
},
"include": [
"./**/*.ts",
"./**/*.d.ts"
],
"exclude": [
"**/*.test.ts",
"**/*.spec.ts",
"**/test/**",
"**/tests/**",
"node_modules",
"dist",
"*.js"
]
}
================================================
FILE: extensions/hellodriver/config.json
================================================
{
"test": "yes I am a test"
}
================================================
FILE: extensions/hellodriver/hellodriver.js
================================================
const { kv } = extension.import('data');
const span = extension.span;
/**
* Here we create an interface called 'hello-world'. This interface
* specifies that any implementation of 'hello-world' should implement
* a method called `greet`. The greet method has a couple of optional
* parameters including `subject` and `locale`. The `locale` parameter
* is not implemented by the driver implementation in the proceeding
* definition, showing how driver implementations don't always need
* to support optional features.
*
* subject: the person to greet
* locale: a standard locale string (ex: en_US.UTF-8)
*/
extension.on('create.interfaces', event => {
event.createInterface('hello-world', {
description: 'Provides methods for generating greetings',
methods: {
greet: {
description: 'Returns a greeting',
parameters: {
subject: {
type: 'string',
optional: true,
},
locale: {
type: 'string',
optional: true,
},
},
},
},
});
});
/**
* Here we register an implementation of the `hello-world` driver
* interface. This implementation is called "no-frills" which is
* the most basic reasonable implementation of the interface. The
* default return value is "Hello, World!", but if subject is
* provided it will be "Hello, !".
*
* This implementation can be called from puter.js like this:
*
* await puter.call('hello-world', 'no-frills', 'greet', { subject: 'Dave' });
*
* If you get an authorization error it's because the user you're
* logged in as does not have permission to invoke the `no-frills`
* implementation of `hello-world`. Users must be granted the following
* permission to access this driver:
*
* service:no-frills:ii:hello-world
*
* The value of `` can be one of many "special" values
* to demonstrate capabilities of drivers or extensions, including:
* - `%fail%`: simulate an error response from a driver
* - `%config%`: return the effective configuration object
*/
extension.on('create.drivers', event => {
event.createDriver('hello-world', 'no-frills', {
greet ({ subject }) {
return `Hello, ${subject ?? 'World'}!`;
},
});
});
extension.on('create.drivers', event => {
event.createDriver('hello-world', 'slow-hello', {
greet: span('slow-hello:greet', async ({ subject }) => {
await new Promise(rslv => setTimeout(rslv, 1000));
await span.run(async () => {
await new Promise(rslv => setTimeout(rslv, 1000));
});
await new Promise(rslv => setTimeout(rslv, 1000));
return `Hello, ${subject ?? 'World'}!`;
}),
});
});
extension.on('create.drivers', event => {
event.createDriver('hello-world', 'extension-examples', {
greet ({ subject }) {
if ( subject === 'fail' ) {
throw new Error('failing on purpose');
}
if ( subject === 'config' ) {
return JSON.stringify(config ?? null);
}
const STR_KVSET = 'kv-set:';
if ( subject.startsWith(STR_KVSET) ) {
return kv.set({
key: 'extension-examples-test-key',
value: subject.slice(STR_KVSET.length),
});
}
if ( subject === 'kv-get' ) {
return kv.get({
key: 'extension-examples-test-key',
});
}
/* eslint-disable */
const STR_KVSET2 = 'kv-set-2:';
if ( subject.startsWith(STR_KVSET2) ) {
return kv.set(
'extension-examples-test-key',
subject.slice(STR_KVSET2.length),
);
}
if ( subject === 'kv-get-2' ) {
return kv.get(
'extension-examples-test-key',
);
}
/* eslint-enable */
return `Hello, ${subject ?? 'World'}!`;
},
});
});
/**
* Here we specify that both registered and temporary users are allowed
* to access the `no-frills` implementation of the `hello-world` driver.
*/
extension.on('create.permissions', event => {
event.grant_to_everyone('service:no-frills:ii:hello-world');
event.grant_to_everyone('service:slow-hello:ii:hello-world');
event.grant_to_everyone('service:extension-examples:ii:hello-world');
});
================================================
FILE: extensions/hellodriver/package.json
================================================
{
"name": "hellodriver",
"main": "hellodriver.js",
"type": "module"
}
================================================
FILE: extensions/imports_something.js
================================================
console.log('importing something...');
const { testval } = extension.import('exports_something');
console.log(testval);
extension.on('hello', event => {
console.log(`received "hello" from: ${event.from}`);
});
================================================
FILE: extensions/metering/config.json
================================================
{
"unlimitedUsage": false,
"unlimitedAllowList": [
"admin"
],
"allowedGlobalUsageUsers": [
"nj",
"salazareos"
],
"priority": 10
}
================================================
FILE: extensions/metering/controllers/UsageController.ts
================================================
/* global extension */
import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js';
import type { MeteringService } from '@heyputer/backend/src/services/MeteringService/MeteringService.js';
import type {
ExtensionRequest,
ExtensionResponse,
} from '../../api.d.ts';
const { Controller, Get, ExtensionController } = extension.import('extensionController');
@Controller('/metering')
export class UsageController extends ExtensionController {
#meteringService: MeteringService;
#sqlClient: BaseDatabaseAccessService;
constructor (
meteringService: MeteringService,
sqlClient: BaseDatabaseAccessService,
) {
super();
this.#meteringService = meteringService;
this.#sqlClient = sqlClient;
}
@Get('usage', { subdomain: 'api' })
async getUsage (req: ExtensionRequest, res: ExtensionResponse) {
const actor = req.actor;
if ( ! actor ) {
throw Error('actor not found in context');
}
const actorUsagePromise = this.#meteringService.getActorCurrentMonthUsageDetails(actor);
const actorAllowanceInfoPromise = this.#meteringService.getAllowedUsage(actor);
const [actorUsage, allowanceInfo] = await Promise.all([
actorUsagePromise,
actorAllowanceInfoPromise,
]);
res.status(200).json({ ...actorUsage, allowanceInfo });
return;
}
@Get('usage/:appIdOrName', { subdomain: 'api' })
async getUsageByApp (req: ExtensionRequest, res: ExtensionResponse) {
const actor = req.actor;
if ( ! actor ) {
throw Error('actor not found in context');
}
const appIdOrName = req.params.appIdOrName;
if ( ! appIdOrName ) {
res.status(400).json({ error: 'appId parameter is required' });
return;
}
if ( typeof appIdOrName !== 'string' ) {
res.status(400).json({ error: 'appId parameter must be a string' });
return;
}
let appId = appIdOrName;
if ( !appIdOrName.startsWith('app-') || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(appIdOrName.split('app-')[1]) ) {
// Check if the part after 'app-' is a valid UUID (v4)
const appRows = await this.#sqlClient.read(
'SELECT `uid` FROM `apps` WHERE `name` = ? LIMIT 1',
[appIdOrName],
);
if ( appRows.length > 0 ) {
appId = appRows[0].uid;
} else {
res.status(404).json({ error: 'App not found' });
return;
}
} else {
appId = appIdOrName;
}
const appUsage =
await this.#meteringService.getActorCurrentMonthAppUsageDetails(
actor,
appId,
);
res.status(200).json(appUsage);
return;
}
@Get('globalUsage', { subdomain: 'api' }, extension.config.allowedGlobalUsageUsers || [])
async getGlobalUsage (req: ExtensionRequest, res: ExtensionResponse) {
const actor = req.actor;
if ( ! actor ) {
throw Error('actor not found in context');
}
const globalUsage = await this.#meteringService.getGlobalUsage();
res.status(200).json(globalUsage);
return;
}
}
================================================
FILE: extensions/metering/eventListeners/subscriptionEvents.ts
================================================
extension.on('metering:overrideDefaultSubscription', async (event) => {
// bit of a stub implementation for OSS, technically can be always free if you set this config true
if ( config.unlimitedUsage ) {
console.warn('WARNING!!! unlimitedUsage is enabled, this is not recommended for production use');
event.defaultSubscriptionId = 'unlimited';
}
});
extension.on('metering:registerAvailablePolicies', async (event) => {
// bit of a stub implementation for OSS, technically can be always free if you set this config true
if ( config.unlimitedUsage || config.unlimitedAllowList?.length ) {
event.availablePolicies.push({
id: 'unlimited',
monthUsageAllowance: 5_000_000 * 1_000_000 * 100, // unless you're like, jeff's, mark's, and elon's illegitamate son, you probably won't hit $5m a month
monthlyStorageAllowance: 100_000 * 1024 * 1024, // 100MiB but ignored in local dev
});
}
});
extension.on('metering:getUserSubscription', async (event) => {
const userName = event?.actor?.type?.user?.username;
if ( config.unlimitedAllowList?.includes(userName) ) {
event.userSubscriptionId;
}
else {
event.userSubscriptionId = event?.actor?.type?.user?.subscription?.active ? event.actor.type.user.subscription?.tier : undefined;
}
// default location for user sub, but can techinically be anywhere else or fetched on request
});
================================================
FILE: extensions/metering/main.ts
================================================
import { UsageController } from './controllers/UsageController.js';
import './eventListeners/subscriptionEvents.js';
const meteringService = extension.import('service:meteringService');
const sqlClient = extension.import('service:database');
const controller = new UsageController(meteringService, sqlClient);
controller.registerRoutes();
================================================
FILE: extensions/metering/package.json
================================================
{
"name": "@heyputer/extension-metering-service",
"main": "main.js",
"type": "module",
"scripts": {
"postinstall": "tsc --noCheck"
},
"devDependencies": {
"typescript": "^5.9.3"
}
}
================================================
FILE: extensions/metering/tsconfig.json
================================================
{
"compilerOptions": {
"target": "ES2024",
"module": "nodenext",
"moduleResolution": "nodenext",
"strict": true,
"forceConsistentCasingInFileNames": true,
"skipLibCheck": true,
"sourceMap": true,
"noEmitOnError": true,
"noImplicitAny": false,
"allowJs": true,
"checkJs": false,
},
"include": [
"./**/*.ts",
"./**/*.d.ts"
],
"exclude": [
"**/*.test.ts",
"**/*.spec.ts",
"**/test/**",
"**/tests/**",
"node_modules",
"dist",
"*.js"
]
}
================================================
FILE: extensions/metering/types.ts
================================================
import '../api.js';
================================================
FILE: extensions/puterfs/PuterFSProvider.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
async #getSourceTreeMaxRelativeDepth (node) {
await node.fetchEntry();
if ( ! node.entry.is_dir ) return 0;
const child_uuids = await this.fsEntryController.fast_get_direct_descendants(await node.get('uid'));
let max = 0;
for ( const child_uuid of child_uuids ) {
const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid));
const child_relative = 1 + await this.#getSourceTreeMaxRelativeDepth(child_node);
max = Math.max(max, child_relative);
}
return max;
}
/**
* Throws if destination depth plus source tree depth would exceed MAX_DIRECTORY_DEPTH.
* @param {number} destinationPathDepth
* @param {FSNode} sourceNode
*/
async #assertDepthLimitForTreeOp (destinationPathDepth, sourceNode) {
const source_relative = await this.#getSourceTreeMaxRelativeDepth(sourceNode);
const max_depth = destinationPathDepth + source_relative;
if ( max_depth > MAX_DIRECTORY_DEPTH ) {
throw APIError.create('directory_depth_limit_exceeded', null, {
limit: MAX_DIRECTORY_DEPTH,
would_be: max_depth,
});
}
}
// #endregion
// TODO: should this be a static member instead?
get_capabilities () {
return new Set([
capabilities.THUMBNAIL,
capabilities.UPDATE_THUMBNAIL,
capabilities.UUID,
capabilities.OPERATION_TRACE,
capabilities.READDIR_UUID_MODE,
capabilities.READDIRSTAT_UUID,
capabilities.PUTER_SHORTCUT,
capabilities.COPY_TREE,
capabilities.GET_RECURSIVE_SIZE,
capabilities.READ,
capabilities.WRITE,
capabilities.CASE_SENSITIVE,
capabilities.SYMLINK,
capabilities.TRASH,
]);
}
// #region PuterOnly
async update_thumbnail ({ context, node, thumbnail }) {
const {
actor: inputActor,
} = context.values;
const actor = inputActor ?? Context.get('actor');
context = context ?? Context.get();
const services = context.get('services');
// TODO: this ACL check should not be here, but there's no LL method yet
// and it's possible we will never implement the thumbnail
// capability for any other filesystem type
const svc_acl = services.get('acl');
if ( ! await svc_acl.check(actor, node, 'write') ) {
throw await svc_acl.get_safe_acl_error(actor, node, 'write');
}
const uid = await node.get('uid');
const entryOp = await this.fsEntryController.update(uid, {
thumbnail,
});
(async () => {
await entryOp.awaitDone();
svc_event.emit('fs.write.file', {
node,
context,
});
})();
return node;
}
async puter_shortcut ({ parent, name, user, target }) {
const user_id = user?.id ?? Context.get('actor')?.type?.user?.id;
await target.fetchEntry({ thumbnail: true });
const ts = Math.round(Date.now() / 1000);
const uid = uuidv4();
svc_resource.register({
uid,
status: RESOURCE_STATUS_PENDING_CREATE,
});
const raw_fsentry = {
is_shortcut: 1,
shortcut_to: target.mysql_id,
is_dir: target.entry.is_dir,
thumbnail: target.entry.thumbnail,
uuid: uid,
parent_uid: await parent.get('uid'),
path: path_.join(await parent.get('path'), name),
user_id: user_id,
name,
created: ts,
updated: ts,
modified: ts,
immutable: false,
};
const entryOp = await this.fsEntryController.insert(raw_fsentry);
(async () => {
await entryOp.awaitDone();
svc_resource.free(uid);
})();
const node = await svc_fs.node(new NodeUIDSelector(uid));
svc_event.emit('fs.create.shortcut', {
node,
context: Context.get(),
});
return node;
}
// #endregion
// #region Optimization
/**
* The readdirstat_uuid operation is only available for filesystem
* immplementations with READDIR_UUID_MODE enabled. This implements
* an optimized readdir operation when the UUID is already known.
* @param {*} param0
*/
async readdirstat_uuid ({
uuid,
options = {},
}) {
const entries = await this.fsEntryController.get_descendants_full(uuid, options);
const nodes = Promise.all(Array.prototype.map.call(entries, raw_entry => {
const node = svc_fs.node(new NodeRawEntrySelector(raw_entry, {
found_thumbnail: options.thumbnail,
}));
node.found = true; // TODO: how is it possible for this to be false?
return node;
}));
return nodes;
};
// #endregion
// #region Standard FS
/**
* Check if a given node exists.
*
* @param {Object} param
* @param {NodeSelector} param.selector - The selector used for checking.
* @returns {Promise} - True if the node exists, false otherwise.
*/
async quick_check ({
selector,
}) {
// shortcut: has full path
if ( selector?.path ) {
const entry = await this.fsEntryController.findByPath(selector.path);
return Boolean(entry);
}
// shortcut: has uid
if ( selector?.uid ) {
const entry = await this.fsEntryController.findByUID(selector.uid);
return Boolean(entry);
}
// shortcut: parent uid + child name
if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeUIDSelector ) {
return await this.fsEntryController.nameExistsUnderParent(selector.parent.uid,
selector.name);
}
// shortcut: parent id + child name
if ( selector instanceof NodeChildSelector && selector.parent instanceof NodeInternalIDSelector ) {
return await this.fsEntryController.nameExistsUnderParentID(selector.parent.id,
selector.name);
}
return false;
}
async unlink ({ context, node, options = {} }) {
if ( await node.get('type') === TYPE_DIRECTORY ) {
throw new APIError(409, 'Cannot unlink a directory.');
}
await this.#rmnode({ context, node, options });
}
async rmdir ({ context, node, options = {} }) {
if ( await node.get('type') !== TYPE_DIRECTORY ) {
throw new APIError(409, 'Cannot rmdir a file.');
}
if ( await node.get('immutable') ) {
throw APIError.create('immutable');
}
const children = await this.fsEntryController.fast_get_direct_descendants(await node.get('uid'));
if ( children.length > 0 && !options.ignore_not_empty ) {
throw APIError.create('not_empty');
}
await this.#rmnode({ context, node, options });
}
/**
* Create a new directory.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNode} param.parent
* @param {string} param.name
* @param {boolean} param.immutable
* @returns {Promise}
*/
async mkdir ({ context, parent, name, immutable }) {
const { actor, thumbnail } = context.values;
const ts = Math.round(Date.now() / 1000);
const uid = uuidv4();
const existing = await svc_fs.node(new NodeChildSelector(parent.selector, name));
if ( await existing.exists() ) {
throw APIError.create('item_with_same_name_exists', null, {
entry_name: name,
});
}
if ( ! await parent.exists() ) {
throw APIError.create('subject_does_not_exist');
}
const new_path = path_.join(await parent.get('path'), name);
if ( this.#pathDepth(new_path) > MAX_DIRECTORY_DEPTH ) {
throw APIError.create('directory_depth_limit_exceeded', null, {
limit: MAX_DIRECTORY_DEPTH,
would_be: this.#pathDepth(new_path),
});
}
svc_resource.register({
uid,
status: RESOURCE_STATUS_PENDING_CREATE,
});
const raw_fsentry = {
is_dir: 1,
uuid: uid,
parent_uid: await parent.get('uid'),
path: path_.join(await parent.get('path'), name),
user_id: actor.type.user.id,
name,
created: ts,
accessed: ts,
modified: ts,
immutable: immutable ?? false,
...(thumbnail ? {
thumbnail: thumbnail,
} : {}),
};
console.log('raw fsentry', raw_fsentry);
const entryOp = await this.fsEntryController.insert(raw_fsentry);
await entryOp.awaitDone();
svc_resource.free(uid);
const node = await svc_fs.node(new NodeUIDSelector(uid));
svc_event.emit('fs.create.directory', {
node,
context: Context.get(),
});
return node;
}
async read ({ context, node, version_id, range }) {
const svc_mountpoint = context.get('services').get('mountpoint');
const storage = svc_mountpoint.get_storage(this.constructor.name);
const location = await node.get('s3:location') ?? {};
const stream = (await storage.create_read_stream(await node.get('uid'), {
// TODO: fs:decouple-s3
bucket: location.bucket,
bucket_region: location.bucket_region,
version_id,
key: location.key,
memory_file: node.entry,
...(range ? { range } : {}),
}));
return stream;
}
async stat ({
selector,
options,
controls,
node,
}) {
// For Puter FS nodes, we assume we will obtain all properties from
// fsEntryController, except for 'thumbnail' unless it's
// explicitly requested.
if ( options.tracer == null ) {
options.tracer = getTracer();
}
if ( options.op ) {
options.trace_options = {
parent: options.op.span,
};
}
let entry;
// stat doesn't work with RawEntrySelector
if ( selector instanceof NodeRawEntrySelector ) {
selector = new NodeUIDSelector(node.uid);
}
await new Promise (rslv => {
const detachables = new MultiDetachable();
const callback = (_resolver) => {
detachables.as(TDetachable).detach();
rslv();
};
// either the resource is free
{
// no detachale because waitForResource returns a
// Promise that will be resolved when the resource
// is free no matter what, and then it will be
// garbage collected.
svc_resource.waitForResource(selector).then(callback.bind(null, 'resourceService'));
}
// or pending information about the resource
// becomes available
{
// detachable is needed here because waitForEntry keeps
// a map of listeners in memory, and this event may
// never occur. If this never occurs, waitForResource
// is guaranteed to resolve eventually, and then this
// detachable will be detached by `callback` so the
// listener can be garbage collected.
const det = this.fsEntryController.waitForEntry(node, callback.bind(null, 'fsEntryService'));
if ( det ) detachables.add(det);
}
});
const maybe_uid = node.uid;
if ( svc_resource.getResourceInfo(maybe_uid) ) {
entry = await this.fsEntryController.get(maybe_uid, options);
controls.log.debug('got an entry from the future');
} else {
entry = await this.fsEntryController.find(selector, options);
}
if ( ! entry ) {
if ( this.log_fsentriesNotFound ) {
controls.log.warn(`entry not found: ${selector.describe(true)}`);
}
}
if ( entry === null || typeof entry !== 'object' ) {
return null;
}
if ( entry.id ) {
controls.provide_selector(new NodeInternalIDSelector('mysql', entry.id, {
source: 'FSNodeContext optimization',
}));
}
return entry;
}
async copy_tree ({ context, source, parent, target_name }) {
// Context
const actor = (context ?? Context).get('actor');
const user = actor.type.user;
const tracer = getTracer();
const uuid = uuidv4();
const timestamp = Math.round(Date.now() / 1000);
await parent.fetchEntry();
await source.fetchEntry({ thumbnail: true });
const destination_path = path_.join(await parent.get('path'), target_name);
await this.#assertDepthLimitForTreeOp(this.#pathDepth(destination_path), source);
// New filesystem entry
const raw_fsentry = {
uuid,
is_dir: source.entry.is_dir,
...(source.entry.is_shortcut ? {
is_shortcut: source.entry.is_shortcut,
shortcut_to: source.entry.shortcut_to,
} : {}),
parent_uid: parent.uid,
name: target_name,
created: timestamp,
modified: timestamp,
path: path_.join(await parent.get('path'), target_name),
// if property exists but the value is undefined,
// it will still be included in the INSERT, causing
// an error
...(source.entry.thumbnail ?
{ thumbnail: source.entry.thumbnail } : {}),
user_id: user.id,
};
svc_event.emit('fs.pending.file', {
fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),
context: context,
});
if ( await source.get('has-s3') ) {
Object.assign(raw_fsentry, {
size: source.entry.size,
associated_app_id: source.entry.associated_app_id,
bucket: source.entry.bucket,
bucket_region: source.entry.bucket_region,
});
await tracer.startActiveSpan('fs:cp:storage-copy', async span => {
let progress_tracker = new UploadProgressTracker();
svc_event.emit('fs.storage.progress.copy', {
upload_tracker: progress_tracker,
context,
meta: {
item_uid: uuid,
item_path: raw_fsentry.path,
},
});
// const storage = new PuterS3StorageStrategy({ services: svc });
const storage = context.get('storage');
const state_copy = storage.create_copy();
await state_copy.run({
src_node: source,
dst_storage: {
key: uuid,
bucket: raw_fsentry.bucket,
bucket_region: raw_fsentry.bucket_region,
},
storage_api: { progress_tracker },
});
span.end();
});
}
{
await svc_size.add_node_size(undefined, source, user);
}
svc_resource.register({
uid: uuid,
status: RESOURCE_STATUS_PENDING_CREATE,
});
const entryOp = await this.fsEntryController.insert(raw_fsentry);
let node;
const tasks = new ParallelTasks({ tracer, max: 4 });
await context.arun('fs:cp:parallel-portion', async () => {
// Add child copy tasks if this is a directory
if ( source.entry.is_dir ) {
const children = await this.fsEntryController.fast_get_direct_descendants(source.uid);
for ( const child_uuid of children ) {
tasks.add('fs:cp:copy-child', async () => {
const child_node = await svc_fs.node(new NodeUIDSelector(child_uuid));
const child_name = await child_node.get('name');
await this.copy_tree({
context,
source: await svc_fs.node(new NodeUIDSelector(child_uuid)),
parent: await svc_fs.node(new NodeUIDSelector(uuid)),
target_name: child_name,
});
});
}
}
// Add task to await entry
tasks.add('fs:cp:entry-op', async () => {
await entryOp.awaitDone();
svc_resource.free(uuid);
const copy_fsNode = await svc_fs.node(new NodeUIDSelector(uuid));
copy_fsNode.entry = raw_fsentry;
copy_fsNode.found = true;
copy_fsNode.path = raw_fsentry.path;
node = copy_fsNode;
svc_event.emit('fs.create.file', {
node,
context,
});
}, { force: true });
await tasks.awaitAll();
});
node = node || await svc_fs.node(new NodeUIDSelector(uuid));
// TODO: What event do we emit? How do we know if we're overwriting?
return node;
}
async move ({ context, node, new_parent, new_name, metadata }) {
const old_path = await node.get('path');
const new_path = path_.join(await new_parent.get('path'), new_name);
await this.#assertDepthLimitForTreeOp(this.#pathDepth(new_path), node);
const op_update = await this.fsEntryController.update(node.uid, {
...(
await node.get('parent_uid') !== await new_parent.get('uid')
? { parent_uid: await new_parent.get('uid') }
: {}
),
path: new_path,
name: new_name,
...(metadata ? { metadata } : {}),
});
node.entry.name = new_name;
node.entry.path = new_path;
// NOTE: this is a safeguard passed to update_child_paths to isolate
// changes to the owner's directory tree, ut this may need to be
// removed in the future.
const user_id = await node.get('user_id');
await op_update.awaitDone();
await svc_fs.update_child_paths(old_path, node.entry.path, user_id);
const promises = [];
promises.push(svc_event.emit('fs.move.file', {
context,
moved: node,
old_path,
}));
promises.push(svc_event.emit('fs.rename', {
uid: await node.get('uid'),
new_name,
}));
return node;
}
async readdir ({ node }) {
const uuid = await node.get('uid');
const child_uuids = await this.fsEntryController.fast_get_direct_descendants(uuid);
return child_uuids;
}
async directory_has_name ({ parent, name }) {
const uid = await parent.get('uid');
let check_dupe = await db.read(
'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1',
[uid, name],
);
return !!check_dupe[0];
}
/**
* Write a new file to the filesystem. Throws an error if the destination
* already exists.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNode} param.parent: The parent directory of the file.
* @param {string} param.name: The name of the file.
* @param {File} param.file: The file to write.
* @returns {Promise}
*/
async write_new ({ context, parent, name, file }) {
console.log('calling write new');
const {
tmp, fsentry_tmp, message, actor: inputActor, app_id,
} = context.values;
const actor = inputActor ?? Context.get('actor');
const uid = uuidv4();
// determine bucket region
let bucket_region = global_config.s3_region ?? global_config.region;
let bucket = global_config.s3_bucket;
if ( ! await svc_acl.check(actor, parent, 'write') ) {
throw await svc_acl.get_safe_acl_error(actor, parent, 'write');
}
const storage_resp = await this.#storage_upload({
uuid: uid,
bucket,
bucket_region,
file,
tmp: {
...tmp,
path: path_.join(await parent.get('path'), name),
},
});
fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;
delete fsentry_tmp.thumbnail_promise;
const timestamp = Math.round(Date.now() / 1000);
const raw_fsentry = {
uuid: uid,
is_dir: 0,
user_id: actor.type.user.id,
created: timestamp,
accessed: timestamp,
modified: timestamp,
parent_uid: await parent.get('uid'),
name,
size: file.size,
path: path_.join(await parent.get('path'), name),
...fsentry_tmp,
bucket_region,
bucket,
associated_app_id: app_id ?? null,
};
svc_event.emit('fs.pending.file', {
fsentry: FSNodeContext.sanitize_pending_entry_info(raw_fsentry),
context,
});
svc_resource.register({
uid,
status: RESOURCE_STATUS_PENDING_CREATE,
});
const filesize = file.size;
svc_size.change_usage(actor.type.user.id, filesize);
// Meter ingress
const ownerId = await parent.get('user_id');
const ownerActor = new Actor({
type: new UserActorType({
user: await get_user({ id: ownerId }),
}),
});
svc_metering.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize);
const entryOp = await this.fsEntryController.insert(raw_fsentry);
(async () => {
await entryOp.awaitDone();
svc_resource.free(uid);
const new_item_node = await svc_fs.node(new NodeUIDSelector(uid));
const new_item = await new_item_node.get('entry');
const store_version_id = storage_resp.VersionId;
if ( store_version_id ) {
// insert version into db
db.write('INSERT INTO `fsentry_versions` (`user_id`, `fsentry_id`, `fsentry_uuid`, `version_id`, `message`, `ts_epoch`) VALUES (?, ?, ?, ?, ?, ?)',
[
actor.type.user.id,
new_item.id,
new_item.uuid,
store_version_id,
message ?? null,
timestamp,
]);
}
})();
const node = await svc_fs.node(new NodeUIDSelector(uid));
svc_event.emit('fs.create.file', {
node,
context,
});
return node;
}
/**
* Overwrite an existing file. Throws an error if the destination does not
* exist.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNodeContext} param.node: The node to write to.
* @param {File} param.file: The file to write.
* @returns {Promise}
*/
async write_overwrite ({ context, node, file }) {
const {
tmp, fsentry_tmp, message, actor: inputActor,
} = context.values;
const actor = inputActor ?? Context.get('actor');
if ( ! await svc_acl.check(actor, node, 'write') ) {
throw await svc_acl.get_safe_acl_error(actor, node, 'write');
}
const uid = await node.get('uid');
const bucket_region = node.entry.bucket_region;
const bucket = node.entry.bucket;
const state_upload = await this.#storage_upload({
uuid: node.entry.uuid,
bucket,
bucket_region,
file,
tmp: {
...tmp,
path: await node.get('path'),
},
});
if ( fsentry_tmp?.thumbnail_promise ) {
fsentry_tmp.thumbnail = await fsentry_tmp.thumbnail_promise;
delete fsentry_tmp.thumbnail_promise;
}
const ts = Math.round(Date.now() / 1000);
const raw_fsentry_delta = {
modified: ts,
accessed: ts,
size: file.size,
...fsentry_tmp,
};
svc_resource.register({
uid,
status: RESOURCE_STATUS_PENDING_CREATE,
});
const filesize = file.size;
svc_size.change_usage(actor.type.user.id, filesize);
// Meter ingress
const ownerId = await node.get('user_id');
const ownerActor = new Actor({
type: new UserActorType({
user: await get_user({ id: ownerId }),
}),
});
svc_metering.incrementUsage(ownerActor, 'filesystem:ingress:bytes', filesize);
const entryOp = await this.fsEntryController.update(uid, raw_fsentry_delta);
// depends on fsentry, does not depend on S3
const entryOpPromise = (async () => {
await entryOp.awaitDone();
svc_resource.free(uid);
})();
(async () => {
await entryOpPromise;
svc_event.emit('fs.write.file', {
node,
context,
});
})();
// TODO (xiaochen): determine if this can be removed, post_insert handler need
// to skip events from other servers (why? 1. current write logic is inside
// the local server 2. broadcast system conduct "fire-and-forget" behavior)
state_upload.post_insert({
db, user: actor.type.user, node, uid, message, ts,
});
return node;
}
async get_recursive_size ({ node }) {
const uuid = await node.get('uid');
const cte_query = `
WITH RECURSIVE descendant_cte AS (
SELECT uuid, parent_uid, size
FROM fsentries
WHERE parent_uid = ?
UNION ALL
SELECT f.uuid, f.parent_uid, f.size
FROM fsentries f
INNER JOIN descendant_cte d
ON f.parent_uid = d.uuid
)
SELECT SUM(size) AS total_size FROM descendant_cte
`;
const rows = await db.read(cte_query, [uuid]);
return rows[0].total_size;
}
// #endregion
// #region internal
/**
* @param {Object} param
* @param {File} param.file: The file to write.
* @returns
*/
async #storage_upload ({
uuid,
bucket,
bucket_region,
file,
tmp,
}) {
const storage = svc_mountpoint.get_storage(this.constructor.name);
bucket ??= global_config.s3_bucket;
bucket_region ??= global_config.s3_region ?? global_config.region;
let upload_tracker = new UploadProgressTracker();
svc_event.emit('fs.storage.upload-progress', {
upload_tracker,
context: Context.get(),
meta: {
item_uid: uuid,
item_path: tmp.path,
},
});
if ( ! file.buffer ) {
let stream = file.stream;
let alarm_timeout = null;
stream = stuck_detector_stream(stream, {
timeout: STUCK_STATUS_TIMEOUT,
on_stuck: () => {
console.warn('Upload stream stuck might be stuck', {
bucket_region,
bucket,
uuid,
});
alarm_timeout = setTimeout(() => {
extension.errors.report('fs.write.s3-upload', {
message: 'Upload stream stuck for too long',
alarm: true,
extra: {
bucket_region,
bucket,
uuid,
},
});
}, STUCK_ALARM_TIMEOUT);
},
on_unstuck: () => {
clearTimeout(alarm_timeout);
},
});
file = { ...file, stream };
}
let hashPromise;
if ( file.buffer ) {
const hash = crypto.createHash('sha256');
hash.update(file.buffer);
hashPromise = Promise.resolve(hash.digest('hex'));
} else {
const hs = hashing_stream(file.stream);
file.stream = hs.stream;
hashPromise = hs.hashPromise;
}
hashPromise.then(hash => {
svc_event.emit('outer.fs.write-hash', {
hash, uuid,
});
});
const state_upload = storage.create_upload();
try {
await this.storageController.upload({
uid: uuid,
file,
storage_meta: { bucket, bucket_region },
storage_api: { progress_tracker: upload_tracker },
});
} catch (e) {
extension.errors.report('fs.write.storage-upload', {
source: e || new Error('unknown'),
trace: true,
alarm: true,
extra: {
bucket_region,
bucket,
uuid,
},
});
throw APIError.create('upload_failed');
}
return state_upload;
}
async #rmnode ({ node, options }) {
// Services
if ( !options.override_immutable && await node.get('immutable') ) {
throw new APIError(403, 'File is immutable.');
}
const userId = await node.get('user_id');
const fileSize = await node.get('size');
svc_size.change_usage(userId,
-1 * fileSize);
const ownerActor = new Actor({
type: new UserActorType({
user: await get_user({ id: userId }),
}),
});
svc_metering.incrementUsage(ownerActor, 'filesystem:delete:bytes', fileSize);
const tracer = getTracer();
const tasks = new ParallelTasks({ tracer, max: 4 });
tasks.add('remove-fsentry', async () => {
await this.fsEntryController.delete(await node.get('uid'));
});
if ( await node.get('has-s3') ) {
tasks.add('remove-from-s3', async () => {
// const storage = new PuterS3StorageStrategy({ services: svc });
const storage = Context.get('storage');
const state_delete = storage.create_delete();
await state_delete.run({
node: node,
});
});
}
await tasks.awaitAll();
}
// #endregion
}
================================================
FILE: extensions/puterfs/fsentries/BaseOperation.js
================================================
import { TeePromise } from 'teepromise';
export default class BaseOperation {
static STATUS_PENDING = {};
static STATUS_RUNNING = {};
static STATUS_DONE = {};
/** @type {PromiseLike & { resolve: () => void }} */
#donePromise;
constructor () {
this.status_ = this.constructor.STATUS_PENDING;
this.#donePromise = new TeePromise();
}
get status () {
return this.status_;
}
set status (status) {
this.status_ = status;
if ( status === this.constructor.STATUS_DONE ) {
this.#donePromise.resolve();
}
}
async awaitDone () {
await this.#donePromise;
}
async onComplete (fn) {
await this.#donePromise;
fn();
}
}
================================================
FILE: extensions/puterfs/fsentries/Delete.js
================================================
import BaseOperation from './BaseOperation.js';
export default class extends BaseOperation {
constructor (uuid) {
super();
this.uuid = uuid;
}
getStatement () {
const statement = 'DELETE FROM fsentries WHERE uuid = ? LIMIT 1';
const values = [this.uuid];
return { statement, values };
}
apply (answer) {
answer.entry = null;
}
}
================================================
FILE: extensions/puterfs/fsentries/FSEntryController.js
================================================
import { TeePromise } from 'teepromise';
import BaseOperation from './BaseOperation.js';
import Delete from './Delete.js';
import Insert from './Insert.js';
import Update from './Update.js';
const { db } = extension.import('data');
const svc_params = extension.import('service:params');
const { PuterPath } = extension.import('fs');
const {
RootNodeSelector,
NodeChildSelector,
NodeUIDSelector,
NodePathSelector,
NodeInternalIDSelector,
} = extension.import('core').fs.selectors;
export default class FSEntryController {
static CONCERN = 'filesystem';
static STATUS_READY = {};
static STATUS_RUNNING_JOB = {};
constructor () {
this.status = FSEntryController.STATUS_READY;
this.currentState = {
queue: [],
updating_uuids: {},
};
this.deferredState = {
queue: [],
updating_uuids: {},
};
this.entryListeners_ = {};
this.mkPromiseForQueueSize_();
// this list of properties is for read operations
// (originally in FSEntryFetcher)
this.defaultProperties = [
'id',
'associated_app_id',
'uuid',
'public_token',
'bucket',
'bucket_region',
'file_request_token',
'user_id',
'parent_uid',
'is_dir',
'is_public',
'is_shortcut',
'is_symlink',
'symlink_path',
'shortcut_to',
'sort_by',
'sort_order',
'immutable',
'name',
'metadata',
'modified',
'created',
'accessed',
'size',
'layout',
'path',
];
this.subdomainProperties = [
'uuid',
'subdomain',
];
}
init () {
svc_params.createParameters('fsentry-service', [
{
id: 'max_queue',
description: 'Maximum queue size',
default: 50,
},
], this);
}
mkPromiseForQueueSize_ () {
this.queueSizePromise = new Promise((resolve, reject) => {
this.queueSizeResolve = resolve;
});
}
// #region write operations
async insert (entry) {
const op = new Insert(entry);
await this.enqueue_(op);
return op;
}
async update (uuid, entry) {
const op = new Update(uuid, entry);
await this.enqueue_(op);
return op;
}
async delete (uuid) {
const op = new Delete(uuid);
await this.enqueue_(op);
return op;
}
// #endregion
// #region read operations
async fast_get_descendants (uuid) {
return (await db.read(`
WITH RECURSIVE descendant_cte AS (
SELECT uuid, parent_uid
FROM fsentries
WHERE parent_uid = ?
UNION ALL
SELECT f.uuid, f.parent_uid
FROM fsentries f
INNER JOIN descendant_cte d ON f.parent_uid = d.uuid
)
SELECT uuid FROM descendant_cte
`, [uuid])).map(x => x.uuid);
}
async fast_get_direct_descendants (uuid) {
return (uuid === PuterPath.NULL_UUID
? await db.read('SELECT uuid FROM fsentries WHERE parent_uid IS NULL')
: await db.read(
'SELECT uuid FROM fsentries WHERE parent_uid = ?',
[uuid],
)).map(x => x.uuid);
}
waitForEntry (node, callback) {
// *** uncomment to debug slow waits ***
// console.log('ATTEMPT TO WAIT FOR', selector.describe())
let selector = node.get_selector_of_type(NodeUIDSelector);
if ( selector === null ) {
// console.log(new Error('========'));
return;
}
const entry_already_enqueued =
Object.prototype.hasOwnProperty.call(this.currentState.updating_uuids, selector.value) ||
Object.prototype.hasOwnProperty.call(this.deferredState.updating_uuids, selector.value) ;
if ( entry_already_enqueued ) {
callback();
return;
}
const k = `uid:${selector.value}`;
if ( ! Object.prototype.hasOwnProperty.call(this.entryListeners_, k) ) {
this.entryListeners_[k] = [];
}
const det = {
detach: () => {
const i = this.entryListeners_[k].indexOf(callback);
if ( i === -1 ) return;
this.entryListeners_[k].splice(i, 1);
if ( this.entryListeners_[k].length === 0 ) {
delete this.entryListeners_[k];
}
},
};
this.entryListeners_[k].push(callback);
return det;
}
async get (uuid, fetch_entry_options) {
const answer = {};
for ( const op of this.currentState.queue ) {
if ( op.uuid != uuid ) continue;
op.apply(answer);
}
for ( const op of this.deferredState.queue ) {
if ( op.uuid != uuid ) continue;
op.apply(answer);
op.apply(answer);
}
if ( answer.is_diff ) {
const base_entry = await this.find(
new NodeUIDSelector(uuid),
fetch_entry_options,
);
answer.entry = { ...base_entry, ...answer.entry };
}
return answer.entry;
}
/**
* Returns UUIDs of child fsentries under the specified
* parent fsentry
* @param {string} uuid - UUID of parent fsentry
* @returns fsentry[]
*/
async get_descendants (uuid) {
return uuid === PuterPath.NULL_UUID
? await db.read(
'SELECT uuid FROM fsentries WHERE parent_uid IS NULL',
[uuid],
)
: await db.read(
'SELECT uuid FROM fsentries WHERE parent_uid = ?',
[uuid],
)
;
}
/**
* Returns full fsentry nodes for entries under the specified
* parent fsentry
* @param {string} uuid - UUID of parent fsentry
* @returns fsentry[]
*/
async get_descendants_full (uuid, fetch_entry_options) {
const { thumbnail } = fetch_entry_options;
const columns = `${
[
...this.defaultProperties.map(v => `f.${v}`),
...this.subdomainProperties
.map(v => `s.${v} AS subdomain_${v}`),
].join(', ')
}${thumbnail ? ', thumbnail' : ''}`;
const results_with_dupes = uuid === PuterPath.NULL_UUID
? await db.read(
`SELECT ${columns} FROM fsentries WHERE parent_uid IS NULL`,
[uuid],
)
: await db.read(
`SELECT ${columns} FROM fsentries AS f ` +
'LEFT JOIN subdomains AS s ON f.id=s.root_dir_id ' +
'WHERE parent_uid = ? ORDER BY f.id',
[uuid],
)
;
const byId = new Map();
for ( const row of results_with_dupes ) {
const id = row.id;
let entry = byId.get(id);
if ( ! entry ) {
entry = { ...row };
if ( thumbnail ) entry.thumbnail = row.thumbnail;
entry.subdomains = [];
byId.set(id, entry);
}
if ( row.subdomain_uuid != null ) {
entry.subdomains.push({
uuid: row.subdomain_uuid,
subdomain: row.subdomain_subdomain,
});
}
}
return Array.from(byId.values());
}
async get_recursive_size (uuid) {
const cte_query = `
WITH RECURSIVE descendant_cte AS (
SELECT uuid, parent_uid, size
FROM fsentries
WHERE parent_uid = ?
UNION ALL
SELECT f.uuid, f.parent_uid, f.size
FROM fsentries f
INNER JOIN descendant_cte d
ON f.parent_uid = d.uuid
)
SELECT SUM(size) AS total_size FROM descendant_cte
`;
const rows = await db.read(cte_query, [uuid]);
return rows[0].total_size;
}
/**
* Finds a filesystem entry using the provided selector.
* @param {Object} selector - The selector object specifying how to find the entry
* @param {Object} fetch_entry_options - Options for fetching the entry
* @returns {Promise} The filesystem entry or null if not found
*/
async find (selector, fetch_entry_options) {
if ( selector instanceof RootNodeSelector ) {
return selector.entry;
}
if ( selector instanceof NodePathSelector ) {
return await this.findByPath(selector.value, fetch_entry_options);
}
if ( selector instanceof NodeUIDSelector ) {
return await this.findByUID(selector.value, fetch_entry_options);
}
if ( selector instanceof NodeInternalIDSelector ) {
return await this.findByID(selector.id, fetch_entry_options);
}
if ( selector instanceof NodeChildSelector ) {
let id;
if ( selector.parent instanceof RootNodeSelector ) {
id = await this.findNameInRoot(selector.name);
} else {
const parentEntry = await this.find(selector.parent);
if ( ! parentEntry ) return null;
id = await this.findNameInParent(parentEntry.uuid, selector.name);
}
if ( id === undefined ) return null;
if ( typeof id !== 'number' ) {
throw new Error(
'unexpected type for id value',
typeof id,
id,
);
}
return this.find(new NodeInternalIDSelector('mysql', id));
}
}
/**
* Finds a filesystem entry by its UUID.
* @param {string} uuid - The UUID of the entry to find
* @param {Object} fetch_entry_options - Options including thumbnail flag
* @returns {Promise} The filesystem entry or undefined if not found
*/
async findByUID (uuid, fetch_entry_options = {}) {
const { thumbnail } = fetch_entry_options;
let fsentry = await db.tryHardRead(
`SELECT ${
this.defaultProperties.join(', ')
}${thumbnail ? ', thumbnail' : ''
} FROM fsentries WHERE uuid = ? LIMIT 1`,
[uuid],
);
return fsentry[0];
}
/**
* Finds a filesystem entry by its internal database ID.
* @param {number} id - The internal ID of the entry to find
* @param {Object} fetch_entry_options - Options including thumbnail flag
* @returns {Promise} The filesystem entry or undefined if not found
*/
async findByID (id, fetch_entry_options = {}) {
const { thumbnail } = fetch_entry_options;
let fsentry = await db.tryHardRead(
`SELECT ${
this.defaultProperties.join(', ')
}${thumbnail ? ', thumbnail' : ''
} FROM fsentries WHERE id = ? LIMIT 1`,
[id],
);
return fsentry[0];
}
/**
* Finds a filesystem entry by its full path.
* @param {string} path - The full path of the entry to find
* @param {Object} fetch_entry_options - Options including thumbnail flag and tracer
* @returns {Promise} The filesystem entry or false if not found
*/
async findByPath (path, fetch_entry_options = {}) {
const { thumbnail } = fetch_entry_options;
if ( path === '/' ) {
return this.find(new RootNodeSelector());
}
const parts = path.split('/').filter(path => path !== '');
if ( parts.length === 0 ) {
// TODO: invalid path; this should be an error
return false;
}
// TODO: use a closure table for more efficient path resolving
let parent_uid = null;
let result;
const resultColsSql = this.defaultProperties.join(', ') +
(thumbnail ? ', thumbnail' : '');
result = await db.read(
`SELECT ${ resultColsSql
} FROM fsentries WHERE path=? LIMIT 1`,
[path],
);
// using knex instead
if ( result[0] ) return result[0];
const loop = async () => {
for ( let i = 0 ; i < parts.length ; i++ ) {
const part = parts[i];
const isLast = i == parts.length - 1;
const colsSql = isLast ? resultColsSql : 'uuid';
if ( parent_uid === null ) {
result = await db.read(
`SELECT ${ colsSql
} FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1`,
[part],
);
} else {
result = await db.read(
`SELECT ${ colsSql
} FROM fsentries WHERE parent_uid=? AND name=? LIMIT 1`,
[parent_uid, part],
);
}
if ( ! result[0] ) return false;
parent_uid = result[0].uuid;
}
};
if ( fetch_entry_options.tracer ) {
const tracer = fetch_entry_options.tracer;
const options = fetch_entry_options.trace_options;
await tracer.startActiveSpan(
'fs:sql:findByPath',
...(options ? [options] : []),
async span => {
await loop();
span.end();
},
);
} else {
await loop();
}
return result[0];
}
/**
* Finds the ID of a child entry with the given name in the root directory.
* @param {string} name - The name of the child entry to find
* @returns {Promise} The ID of the child entry or undefined if not found
*/
async findNameInRoot (name) {
let child_id = await db.read(
'SELECT `id` FROM `fsentries` WHERE `parent_uid` IS NULL AND name = ? LIMIT 1',
[name],
);
return child_id[0]?.id;
}
/**
* Finds the ID of a child entry with the given name under a specific parent.
* @param {string} parent_uid - The UUID of the parent directory
* @param {string} name - The name of the child entry to find
* @returns {Promise} The ID of the child entry or undefined if not found
*/
async findNameInParent (parent_uid, name) {
let child_id = await db.read(
'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1',
[parent_uid, name],
);
return child_id[0]?.id;
}
/**
* Checks if an entry with the given name exists under a specific parent.
* @param {string} parent_uid - The UUID of the parent directory
* @param {string} name - The name to check for
* @returns {Promise} True if the name exists under the parent, false otherwise
*/
async nameExistsUnderParent (parent_uid, name) {
let check_dupe = await db.read(
'SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1',
[parent_uid, name],
);
return !!check_dupe[0];
}
/**
* Checks if an entry with the given name exists under a parent specified by ID.
* @param {number} parent_id - The internal ID of the parent directory
* @param {string} name - The name to check for
* @returns {Promise} True if the name exists under the parent, false otherwise
*/
async nameExistsUnderParentID (parent_id, name) {
const parent = await this.findByID(parent_id);
if ( ! parent ) {
return false;
}
return this.nameExistsUnderParent(parent.uuid, name);
}
// #endregion
// #region queue logic
async enqueue_ (op) {
const tp = new TeePromise();
while (
this.currentState.queue.length > this.max_queue ||
this.deferredState.queue.length > this.max_queue
) {
await this.queueSizePromise;
}
if ( ! (op instanceof BaseOperation) ) {
throw new Error('Invalid operation');
}
const state = this.status === FSEntryController.STATUS_READY ?
this.currentState : this.deferredState;
if ( ! Object.prototype.hasOwnProperty.call(state.updating_uuids, op.uuid) ) {
state.updating_uuids[op.uuid] = [];
}
state.updating_uuids[op.uuid].push(state.queue.length);
state.queue.push(op);
// DRY: same pattern as FSOperationContext:provideValue
// DRY: same pattern as FSOperationContext:rejectValue
if ( Object.prototype.hasOwnProperty.call(this.entryListeners_, op.uuid) ) {
const listeners = this.entryListeners_[op.uuid];
delete this.entryListeners_[op.uuid];
for ( const lis of listeners ) lis();
}
this.checkShouldExec_();
await op.awaitDone();
}
checkShouldExec_ () {
if ( this.status !== FSEntryController.STATUS_READY ) return;
if ( this.currentState.queue.length === 0 ) return;
this.exec_();
}
async exec_ () {
if ( this.status !== FSEntryController.STATUS_READY ) {
throw new Error('Duplicate exec_ call');
}
const queue = this.currentState.queue;
this.status = FSEntryController.STATUS_RUNNING_JOB;
// const conn = await db_primary.promise().getConnection();
// await conn.beginTransaction();
for ( const op of queue ) {
op.status = op.constructor.STATUS_RUNNING;
// await conn.execute(stmt, values);
}
// await conn.commit();
// conn.release();
// const stmtAndVals = queue.map(op => op.getStatementAndValues());
// const stmts = stmtAndVals.map(x => x.stmt).join('; ');
// const vals = stmtAndVals.reduce((acc, x) => acc.concat(x.values), []);
// *** uncomment to debug batch queries ***
// this.log.debug({ stmts, vals });
// console.log('<<========================');
// console.log({ stmts, vals });
// console.log('>>========================');
// this.log.debug('array?', Array.isArray(vals))
await db.batch_write(queue.map(op => op.getStatement()));
for ( const op of queue ) {
op.status = op.constructor.STATUS_DONE;
}
this.flipState_();
this.status = FSEntryController.STATUS_READY;
for ( const op of queue ) {
op.status = op.constructor.STATUS_DONE;
}
this.checkShouldExec_();
}
flipState_ () {
this.currentState = this.deferredState;
this.deferredState = {
queue: [],
updating_uuids: {},
};
const queueSizeResolve = this.queueSizeResolve;
this.mkPromiseForQueueSize_();
queueSizeResolve();
}
// #endregion
}
================================================
FILE: extensions/puterfs/fsentries/Insert.js
================================================
import { safeHasOwnProperty } from '../lib/objectfn.js';
import BaseOperation from './BaseOperation.js';
export default class extends BaseOperation {
static requiredForCreate = [
'uuid',
'parent_uid',
];
static allowedForCreate = [
...this.requiredForCreate,
'name',
'user_id',
'is_dir',
'created',
'modified',
'immutable',
'shortcut_to',
'is_shortcut',
'metadata',
'bucket',
'bucket_region',
'thumbnail',
'accessed',
'size',
'symlink_path',
'is_symlink',
'associated_app_id',
'path',
];
constructor (entry) {
super();
const requiredForCreate = this.constructor.requiredForCreate;
const allowedForCreate = this.constructor.allowedForCreate;
{
const sanitized_entry = {};
for ( const k of allowedForCreate ) {
if ( safeHasOwnProperty(entry, k) ) {
sanitized_entry[k] = entry[k];
}
}
entry = sanitized_entry;
}
for ( const k of requiredForCreate ) {
if ( ! safeHasOwnProperty(entry, k) ) {
throw new Error(`Missing required property: ${k}`);
}
}
this.entry = entry;
}
getStatement () {
const fields = Object.keys(this.entry);
const statement = 'INSERT INTO fsentries ' +
`(${fields.join(', ')}) ` +
`VALUES (${fields.map(() => '?').join(', ')})`;
const values = fields.map(k => this.entry[k]);
return { statement, values };
}
apply (answer) {
answer.entry = { ...this.entry };
}
get uuid () {
return this.entry.uuid;
}
};
================================================
FILE: extensions/puterfs/fsentries/Update.js
================================================
import { safeHasOwnProperty } from '../lib/objectfn.js';
import BaseOperation from './BaseOperation.js';
export default class extends BaseOperation {
static allowedForUpdate = [
'name',
'parent_uid',
'user_id',
'modified',
'shortcut_to',
'metadata',
'thumbnail',
'size',
'path',
];
constructor (uuid, entry) {
super();
const allowedForUpdate = this.constructor.allowedForUpdate;
{
const sanitized_entry = {};
for ( const k of allowedForUpdate ) {
if ( safeHasOwnProperty(entry, k) ) {
sanitized_entry[k] = entry[k];
}
}
entry = sanitized_entry;
}
this.uuid = uuid;
this.entry = entry;
}
getStatement () {
const fields = Object.keys(this.entry);
const statement = 'UPDATE fsentries SET ' +
`${fields.map(k => `${k} = ?`).join(', ')} ` +
'WHERE uuid = ? LIMIT 1';
const values = fields.map(k => this.entry[k]);
values.push(this.uuid);
return { statement, values };
}
apply (answer) {
if ( ! answer.entry ) {
answer.is_diff = true;
answer.entry = {};
}
Object.assign(answer.entry, this.entry);
}
};
================================================
FILE: extensions/puterfs/lib/objectfn.js
================================================
/**
* Instead of `myObject.hasOwnProperty(k)`, always write:
* `safeHasOwnProperty(myObject, k)`.
*
* This is a less verbose way to call `Object.prototype.hasOwnProperty.call`.
* This prevents unexpected behavior when `hasOwnProperty` is overridden,
* which is especially possible for objects parsed from user-sent JSON.
*
* explanation: https://eslint.org/docs/latest/rules/no-prototype-builtins
* @param {*} o
* @param {...any} a
* @returns
*/
export const safeHasOwnProperty = (o, ...a) => {
return Object.prototype.hasOwnProperty.call(o, ...a);
};
================================================
FILE: extensions/puterfs/main.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see Puter Worker Sandbox Playground
Puter Worker Sandbox Playground
Use this page to interact with the puter APIs in the same sandbox as your worker.
Run
Clear Logs
`;
extension.get('/', { noauth: true, subdomain: 'worker-sandbox' }, (req, res) => {
res.type('html').send(page);
});
================================================
FILE: install.md
================================================
# INSTALL.md
## Node.js & npm Installation Guide
## 1. Arch Linux / Manjaro
```bash
# Update package database
sudo pacman -Syu
# Install Node.js and npm
sudo pacman -S nodejs npm
# Verify installation
node -v
npm -v
````
---
## 2. Debian / Ubuntu
```bash
# Update package database and install curl
sudo apt update
sudo apt install -y curl
# Install nvm (Node Version Manager)
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash
# Load nvm
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
# Reload shell
source ~/.bashrc # Or source ~/.zshrc if using Zsh
# Install latest Node.js and npm
nvm install node
# Verify installation
node -v
npm -v
```
---
## 3. CentOS / RHEL
```bash
# Install curl if missing
sudo yum install -y curl
# Install nvm
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash
# Load nvm
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
# Reload shell
source ~/.bashrc # Or source ~/.zshrc if using Zsh
# Install latest Node.js and npm
nvm install node
# Verify installation
node -v
npm -v
```
---
## 4. Fedora
```bash
# Update system
sudo dnf update -y
# Install Node.js and npm from modules
sudo dnf module list nodejs # Check available versions
sudo dnf module enable nodejs:18 # Example: enable Node 18 LTS
sudo dnf install -y nodejs npm
# Verify installation
node -v
npm -v
```
---
## 5. openSUSE
```bash
# Refresh repositories
sudo zypper refresh
# Install Node.js and npm
sudo zypper install -y nodejs npm
# Verify installation
node -v
npm -v
```
---
## 6. Using nvm (Optional, Recommended)
`nvm` allows installing multiple Node.js versions and switching between them easily:
```bash
# Install nvm
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash
# Load nvm
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
# Reload shell
source ~/.bashrc # Or source ~/.zshrc if using Zsh
# Install latest Node.js and npm
nvm install node
# Or install latest LTS version
nvm install --lts
# Switch Node versions
nvm use node
# Verify installation
node -v
npm -v
```
================================================
FILE: mod_packages/testex/package.json
================================================
{}
================================================
FILE: mods/README.md
================================================
# Puter Mods
A list of Puter mods which may be expanded in the future.
**Contributions of new mods are welcome.**
## kdmod
- **location:** [./kdmod](./kdmod)
- **description:**
> "kernel dev mod"; specifically for the devex needs of
> GitHub user KernelDeimos and provided in case anyone else
> finds it of any use.
================================================
FILE: mods/mods_available/dev-socket/main.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see Test actions
Test actions
`;
extension.get('/test-actions', (req, res) => {
res.setHeader('Content-Type', 'text/html; charset=utf-8');
res.send(PAGE_HTML(JSON.stringify(ACTIONS)));
});
extension.post('/test-actions/invoke/:actionId', async (req, res) => {
const actionId = req.params.actionId;
const handler = INVOKE_HANDLERS[actionId];
if ( ! handler ) {
return res.status(404).json({ ok: false, error: 'Unknown action' });
}
return handler(req, res, req.body || {});
});
extension.on('ai.prompt.validate', async event => {
console.log('ai.prompt.validate');
const messages = event.parameters?.messages ?? [];
console.log(`ai prompt validate: ${messages.length} messages`);
console.log('is user suspended?', event.actor.type.user.suspended);
});
================================================
FILE: mods/mods_available/test-actions/package.json
================================================
{
"name": "@heyputer/test-actions",
"version": "1.0.0",
"description": "Actions for test purposes",
"main": "main.js",
"type": "module",
"private": true
}
================================================
FILE: mods/mods_available/testex.js
================================================
// Test extension for event listeners
extension.on('ai.prompt.complete', event => {
console.log('GOT AI.PROMPT.COMPLETE EVENT', event);
});
extension.on('ai.prompt.validate', event => {
console.log('GOT AI.PROMPT.VALIDATE EVENT', event);
});
extension.on('app.new-icon', event => {
console.log('GOT APP.NEW-ICON EVENT', event);
});
extension.on('app.rename', event => {
console.log('GOT APP.RENAME EVENT', event);
});
extension.on('apps.invalidate', event => {
console.log('GOT APPS.INVALIDATE EVENT', event);
});
extension.on('email.validate', event => {
console.log('GOT EMAIL.VALIDATE EVENT', event);
});
extension.on('fs.create.directory', event => {
console.log('GOT FS.CREATE.DIRECTORY EVENT', event);
});
extension.on('fs.create.file', event => {
console.log('GOT FS.CREATE.FILE EVENT', event);
});
extension.on('fs.create.shortcut', event => {
console.log('GOT FS.CREATE.SHORTCUT EVENT', event);
});
extension.on('fs.create.symlink', event => {
console.log('GOT FS.CREATE.SYMLINK EVENT', event);
});
extension.on('fs.move.file', event => {
console.log('GOT FS.MOVE.FILE EVENT', event);
});
extension.on('fs.pending.file', event => {
console.log('GOT FS.PENDING.FILE EVENT', event);
});
extension.on('fs.storage.progress.copy', event => {
console.log('GOT FS.STORAGE.PROGRESS.COPY EVENT', event);
});
extension.on('fs.storage.upload-progress', event => {
console.log('GOT FS.STORAGE.UPLOAD-PROGRESS EVENT', event);
});
extension.on('fs.write.file', event => {
console.log('GOT FS.WRITE.FILE EVENT', event);
});
extension.on('ip.validate', event => {
console.log('GOT IP.VALIDATE EVENT', event);
});
extension.on('outer.fs.write-hash', event => {
console.log('GOT OUTER.FS.WRITE-HASH EVENT', event);
});
extension.on('outer.gui.item.added', event => {
console.log('GOT OUTER.GUI.ITEM.ADDED EVENT', event);
});
extension.on('outer.gui.item.moved', event => {
console.log('GOT OUTER.GUI.ITEM.MOVED EVENT', event);
});
extension.on('outer.gui.item.pending', event => {
console.log('GOT OUTER.GUI.ITEM.PENDING EVENT', event);
});
extension.on('outer.gui.item.updated', event => {
console.log('GOT OUTER.GUI.ITEM.UPDATED EVENT', event);
});
extension.on('outer.gui.notif.ack', event => {
console.log('GOT OUTER.GUI.NOTIF.ACK EVENT', event);
});
extension.on('outer.gui.notif.message', event => {
console.log('GOT OUTER.GUI.NOTIF.MESSAGE EVENT', event);
});
extension.on('outer.gui.notif.persisted', event => {
console.log('GOT OUTER.GUI.NOTIF.PERSISTED EVENT', event);
});
extension.on('outer.gui.notif.unreads', event => {
console.log('GOT OUTER.GUI.NOTIF.UNREADS EVENT', event);
});
extension.on('outer.gui.submission.done', event => {
console.log('GOT OUTER.GUI.SUBMISSION.DONE EVENT', event);
});
extension.on('puter-exec.submission.done', event => {
console.log('GOT PUTER-EXEC.SUBMISSION.DONE EVENT', event);
});
extension.on('request.measured', event => {
console.log('GOT REQUEST.MEASURED EVENT', event);
});
extension.on('sns', event => {
console.log('GOT SNS EVENT', event);
});
extension.on('template-service.hello', event => {
console.log('GOT TEMPLATE-SERVICE.HELLO EVENT', event);
});
extension.on('usages.query', event => {
console.log('GOT USAGES.QUERY EVENT', event);
});
extension.on('user.email-changed', event => {
console.log('GOT USER.EMAIL-CHANGED EVENT', event);
});
extension.on('user.email-confirmed', event => {
console.log('GOT USER.EMAIL-CONFIRMED EVENT', event);
});
extension.on('user.save_account', event => {
console.log('GOT USER.SAVE_ACCOUNT EVENT', event);
});
extension.on('web.socket.connected', event => {
console.log('GOT WEB.SOCKET.CONNECTED EVENT', event);
});
extension.on('web.socket.user-connected', event => {
console.log('GOT WEB.SOCKET.USER-CONNECTED EVENT', event);
});
extension.on('wisp.get-policy', event => {
console.log('GOT WISP.GET-POLICY EVENT', event);
});
================================================
FILE: mods/mods_enabled/.gitignore
================================================
*
!.gitignore
================================================
FILE: package.json
================================================
{
"name": "puter.com",
"version": "2.5.1",
"author": "Puter Technologies Inc.",
"license": "AGPL-3.0-only",
"description": "Desktop environment in the browser!",
"homepage": "https://puter.com",
"type": "module",
"main": "exports.js",
"directories": {
"lib": "lib"
},
"devDependencies": {
"@eslint/js": "^9.35.0",
"@playwright/test": "^1.56.1",
"@stylistic/eslint-plugin": "^5.3.1",
"@types/mime-types": "^3.0.1",
"@types/uuid": "^10.0.0",
"@typescript-eslint/eslint-plugin": "^8.46.1",
"@typescript-eslint/parser": "^8.46.1",
"@vitest/coverage-v8": "^4.1.0",
"@vitest/ui": "^4.1.0",
"chalk": "^4.1.0",
"clean-css": "^5.3.2",
"dotenv": "^16.4.5",
"eslint": "^9.35.0",
"eslint-rule-composer": "^0.3.0",
"express": "^4.18.2",
"globals": "^15.15.0",
"html-entities": "^2.3.3",
"html-webpack-plugin": "^5.6.0",
"husky": "^9.1.7",
"license-check-and-add": "^4.0.5",
"mocha": "^7.2.0",
"nodemon": "^3.1.0",
"simple-git": "^3.32.3",
"ts-proto": "^2.8.0",
"typescript": "^5.4.5",
"uglify-js": "^3.17.4",
"vite-plugin-static-copy": "^3.3.0",
"vitest": "^4.1.0",
"webpack": "^5.88.2",
"webpack-cli": "^5.1.1",
"yaml": "^2.8.1"
},
"scripts": {
"test": "npx vitest run --config=src/backend/vitest.config.ts && node src/backend/tools/test.mjs",
"test:puterjs-api": "vitest run tests/puterJsApiTests",
"test:backend": "npm run build:ts; vitest run --config=src/backend/vitest.config.ts",
"test:backend-coverage": "npm run build:ts; vitest run --config=src/backend/vitest.config.ts",
"start=gui": "nodemon --exec \"node dev-server.js\" ",
"start": "node ./tools/run-selfhosted.js",
"prestart": "npm run build:ts",
"dev": "npm run build:ts && DEVCONSOLE=1 node ./tools/run-selfhosted.js",
"build": "npx eslint --quiet -c eslint/mandatory.eslint.config.js src/backend/src extensions && npm run build:ts && cd src/gui && node ./build.js",
"check-translations": "node tools/check-translations.js",
"prepare": "husky",
"build:ts": "tsc -p tsconfig.build.json",
"gen": "./scripts/gen.sh"
},
"workspaces": [
"src/*",
"tools/*",
"experiments/js-parse-and-output"
],
"nodemonConfig": {
"ext": "js, json, mjs, jsx, svg, css",
"ignore": [
"./dist/",
"./node_modules/"
]
},
"dependencies": {
"@ai-sdk/openai": "^3.0.25",
"@anthropic-ai/sdk": "^0.68.0",
"@aws-sdk/client-dynamodb": "^3.490.0",
"@aws-sdk/client-secrets-manager": "^3.879.0",
"@aws-sdk/client-sns": "^3.907.0",
"@aws-sdk/lib-dynamodb": "^3.490.0",
"@google/genai": "^1.19.0",
"@heyputer/putility": "^1.0.2",
"@paralleldrive/cuid2": "^2.2.2",
"@stylistic/eslint-plugin-js": "^4.4.1",
"ai": "^6.0.73",
"dedent": "^1.5.3",
"dynalite": "^4.0.0",
"express-xml-bodyparser": "^0.4.1",
"file-type": "21.3.3",
"javascript-time-ago": "^2.5.11",
"json-colorizer": "^3.0.1",
"music-metadata": "11.12.3",
"open": "^10.1.0",
"parse-domain": "^8.2.2",
"string-template": "^1.0.0",
"uuid": "^9.0.1"
},
"optionalDependencies": {
"sharp": "^0.34.4",
"sharp-bmp": "^0.1.5",
"sharp-ico": "^0.1.5"
},
"engines": {
"node": ">=24.0.0"
}
}
================================================
FILE: rust-toolchain.toml
================================================
[toolchain]
channel = "nightly"
components = [ "rustc", "rust-std" ]
targets = [ "wasm32-unknown-unknown", "i686-unknown-linux-gnu" ]
profile = "minimal"
================================================
FILE: scripts/gen.sh
================================================
#!/usr/bin/env bash
set -euo pipefail
protoc \
-I=src/backend/src/filesystem/definitions/proto \
--plugin=protoc-gen-ts_proto=$(npm root)/.bin/protoc-gen-ts_proto \
--ts_proto_out=src/backend/src/filesystem/definitions/ts \
--ts_proto_opt=esModuleInterop=true,outputServices=none,outputJsonMethods=true,useExactTypes=false,snakeToCamel=false \
src/backend/src/filesystem/definitions/proto/fsentry.proto
================================================
FILE: src/backend/.gitignore
================================================
# MAC OS hidden directory settings file
.DS_Store
# Created by https://www.toptal.com/developers/gitignore/api/node
# Edit at https://www.toptal.com/developers/gitignore?templates=node
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
lerna-debug.log*
.pnpm-debug.log*
# Diagnostic reports (https://nodejs.org/api/report.html)
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
*.lcov
# nyc test coverage
.nyc_output
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Snowpack dependency directory (https://snowpack.dev/)
web_modules/
# TypeScript cache
*.tsbuildinfo
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Microbundle cache
.rpt2_cache/
.rts2_cache_cjs/
.rts2_cache_es/
.rts2_cache_umd/
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
.env.test
.env.production
# parcel-bundler cache (https://parceljs.org/)
.cache
.parcel-cache
# Next.js build output
.next
out
# Nuxt.js build / generate output
.nuxt
dist
# Gatsby files
.cache/
# Comment in the public line in if your project uses Gatsby and not Next.js
# https://nextjs.org/blog/next-9-1#public-directory-support
# public
# vuepress build output
.vuepress/dist
# Serverless directories
.serverless/
# FuseBox cache
.fusebox/
# DynamoDB Local files
.dynamodb/
# TernJS port file
.tern-port
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
# yarn v2
.yarn/cache
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
# End of https://www.toptal.com/developers/gitignore/api/node
public/.DS_Store
*.zip
*.pem
public/.DS_Store
public/.DS_Store
public/.DS_Store
./build
build
# config file
volatile/
ssl
ssl/
keys
*.code-workspace
# credentials
creds*
# test-webhook persisted key
tools/.test-webhook-config.json
# thumbnai-service
thumbnail-service
# init sql generated from ./run.sh
init.sql
================================================
FILE: src/backend/CONTRIBUTING.md
================================================
# Contributing to Puter's Backend
## File Structure
## Architecture
- [boot sequence](./doc/contributors/boot-sequence.md)
- [modules and services](./doc/contributors/modules.md)
## Features
- [protected apps](./doc/features/protected-apps.md)
- [service scripts](./doc/features/service-scripts.md)
## Lists of Things
- [list of permissions](./doc/lists-of-things/list-of-permissions.md)
## Code-First Approach
If you prefer to understand a system by looking at the
first files which are invoked and starting from there,
here's a handy list!
- [Kernel](./src/Kernel.js), despite its intimidating name, is a
relatively simple (< 200 LOC) class which loads the modules
(modules register services), and then starts all the services.
- [RuntimeEnvironment](./src/boot/RuntimeEnvironment.js)
sets the configuration and runtime directories. It's invoked by Kernel.
- The default setup for running a self-hosted Puter loads these modules:
- [CoreModule](./src/CoreModule.js)
- [DatabaseModule](./src/DatabaseModule.js)
- [LocalDiskStorageModule](./src/LocalDiskStorageModule.js)
- HTTP endpoints are registered with
[WebServerService](./src/services/WebServerService.js)
by these services:
- [ServeGUIService](./src/services/ServeGUIService.js)
- [PuterAPIService](./src/services/PuterAPIService.js)
- [FilesystemAPIService](./src/services/FilesystemAPIService.js)
## Development Philosophies
### The copy-paste rule
If you're copying and pasting code, you need to ask this question:
- am I copying as a reference (i.e. how this function is used),
- or am I copying an implementation of actual behavior?
If your answer is the first, you should find more than one piece of
code that's doing the same thing you want to do and see if any of them
are doing it differently. One of the ways of doing this thing is going
to be more recent and/or (yes, potentially "or") more correct.
More correct approaches are ones which reduce
[coupling](https://en.wikipedia.org/wiki/Coupling_(computer_programming)),
move from legacy implementations to more recent ones, and are actually
more convenient for you to use. Whenever ever any of these three things
are in contention it's very important to communicate this to the
appropriate maintainers and contributors.
If your answer is the second, you should find a way to
[DRY that code](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself).
### Architecture Mistakes? You will make them and it will suck.
In my experience, the harder I think about the correct way to implement
something, the bigger a mistake I'm going to make; ***unless*** a big part
of the reason I'm thinking so hard is because I want to find a solution
that reduces complexity and has the right maintenance trade-off.
There's no easy solution for this so just keep it in mind; there are some
things we might write 2 times, 3 times, even more times over before we
really get it right and *that's okay*; sometimes part of doing useful work is
doing the useless work that reveals what the useful work is.
## Underlying Constructs
- [putility's README.md](../putility/README.md)
- Whenever you see `AdvancedBase`, that's from here
- Many things in backend extend this. Anything that doesn't only doesn't
because it was written before `AdvancedBase` existed.
- Allows adding "traits" to classes
- Have you ever wanted to wrap every method of a class with
common behavior? This can do that!
================================================
FILE: src/backend/README.md
================================================
# Puter Backend
_Part of a High-Level Distributed Operating System_
Whether or not you call Puter an operating system
(we call it a "high-level distributed operating system"),
**operating systems for devices**
are a useful reference point to describe the architecture of Puter.
If Puter's "hardware" is services, and Puter's "userspace" is the
client side of the API, then Puter's "kernel" is the backend.
Puter's backend is composed of:
- The **Kernel** class, which is responsible for initialization
- A number of **Modules** which are registered in **Kernel** for a customized
Puter instance.
- Many **Services** which are contained inside modules.
## Documentation
- [Backend File Structure](./doc/contributors/structure.md)
- [Boot Sequence](./doc/contributors/boot-sequence.md)
- [Kernel](./doc/Kernel.md)
- [Modules](./doc/contributors/modules.md)
## Can I use Puter's Backend Alone?
Puter's backend is not dependent on Puter's frontned. In fact, you could
prevent Puter's GUI from ever showing up by disabling PuterHomepageModule.
Similarly, you can run Puter's backend with no modules loaded for a completely
blank slate, or only include CoreModule and WebModule to quickly build your
own backend that's compatible with any of Puter's services.
## What can it do?
Puter's Kernel only initializes modules, nothing more. The modules bring a lot
of capabilities to the table, however. Within this directory you'll find modules that:
- coerce all the well-known AI services to a common interface
- manage authentication with Wisp servers (this brings TCP to the browser!)
- manage apps on Puter
- allow a user to host websites from Puter
- provide persistent key-value storage to Puter's desktop and apps
- provide a fast filesystem implementation
- communicate with other instances of Puter's backend,
secured with elliptic curve cryptography
- provide more services like converting files and compiling low-level code.
================================================
FILE: src/backend/doc/A-and-A/auth.md
================================================
# Authentication Documentation
## Concepts
### Actor
An "Actor" is an entity that can be authenticated. The following types of
actors are currently supported by Puter:
- **UserActorType** - represents a user and is identified by a user's UUID
- **AppUnderUserActorType** - represents an app running in an iframe from a
`puter.site` domain or another origin and is identified by a user's UUID
and an app's UUID together.
- **AccessTokenActorType** - not widely currently, but Puter supports
a concept called "access tokens". Any user can create an access token and
then grant any permissions they want to that access token. The access
token will have those permissions granted provided that the user who
created the access token does as well (via permission cascade)
- **SiteActorType** - represents a `puter.site` website accessing Puter's API.
- **SystemActorType** - internal representation of the actor during a privileged
backend operation. This actor cannot be authenticated in a request.
This actor does not represent the `system` user.
### Token
- **Legacy** - legacy tokens result in an error response
- **Session** - this token is a JWT with a claim for the UUID of an entry in
server memory or the database that we call a "session". This entry associates
the token to a user and some metadata for security auditing purposes.
Revoking the session entry disables the token.
This type of token resolves to an actor with **UserActorType**.
- **AppUnderUser** - this token is a JWT with a claim for an app UUID and a
claim for a session UUID.
Revoking the session entry disables the token.
This type of token resolves to an actor with **AppUnderUserActorType**.
- **AccessToken** - this token is a JWT with three claims:
- A session UUID
- An optional App UUID
- A UUID representing the access token for permission associations
The session or session+app creates a **UserActorType** or
**AppUnderUserActorType** actor respectively. This actor is called
the "authorizor". This actor is aggregated by an **AccessTokenActorType**
actor which becomes the effective actor for a request.
- **ActorSite** - this token is a JWT with a claim for a site UID.
The site UID is associated with an origin, generally a `puter.site`
subdomain.
## Components
### Auth Middleware
There have so far been three iterations of the authentication middleware:
- `src/backend/src/middleware/auth.js`
- `src/backend/src/middleware/auth2.js`
- `src/backend/src/middleware/configurable_auth.js`
The newest implementation is `configurable_auth` and eventually the other
two will be removed. There is no legacy behavior involved:
- `auth` was rewritten to use `auth2`
- `auth2` was rewritten to use `configurable_auth`
The `configurable_auth` middleware accepts a parameter that can be specified
if an endpoint is optionally authenticated. In this case, the request's
`actor` will be `undefined` if there was no information for authentication.
================================================
FILE: src/backend/doc/A-and-A/permission.md
================================================
# Permission Documentation
## Concepts
### Permission
A permission is a string composed of colon-delimited components which identifies
a resource or functionality to which access can be controlled.
For example, `fs:e8ac2973-287b-4121-a75d-7e0619eb8e87:read` is a permission which
represents reading the file or directory with UUID `e8ac2973-287b-4121-a75d-7e0619eb8e87`.
### Group
A group has an owner and several member users. An owner decides what users are in the
group and what users are not. Any user can grant permissions to the group.
### Granting & Revoking
Granting is the act of creating a permission association to a user or group from
the current user. A permission association also holds an object called `extra`
which holds additional claims associated with the permission association.
These are arbitrary and can be used in any way by the subsystem or extension that
is checking the permission. `extra` is usually just an empty object.
Revoking is the act of removing a permission association.
### Permission Options
Permission options are an association between a permission and an actor that can not
be revoked by another actor. For example, the user `ed` always has access to files
under `/ed`. The user `system` always has all permissions granted. These can also be
considered "terminals" because they will always be at
the end of a pathway through granted permissions between users.
This are also called "implied" permissions because they are implied by the system.
### Permission Pathways
A permission pathway is the path between users or groups that leads to a permission.
For example, `ed` can grant the permission `a:b` to `fred`, then `fred` can grant
that permission to the group `cool_group`, and then `alice` may be in the group
`cool_group`. Assuming `ed` holds the implied permission `a:b`, a permission path
exists between `alice` and `ed` via `cool_group` and `fred`:
```
alice <--<> cool_group <-- fred <-- ed (a:b)
```
If any link in this chain breaks the permission is effectively revoked from `alice`
unless there is another pathway leading to a valid permission option for `a:b`.
### Reading - AKA Permission Scan Result
A permission reading is a JSON-serializable object which contains all the pathways
a specified actor has to permissions options matching the specified permission strings.
The following is an example reading for the user `ed3` on the permission
`fs:24729b88-a4c5-4990-ad4e-272b87895732:read`. This file is owned by the
user `admin` who shared it with `ed3`.
```
[
{
"$": "explode",
"from": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read",
"to": [
"fs:24729b88-a4c5-4990-ad4e-272b87895732:read",
"fs:24729b88-a4c5-4990-ad4e-272b87895732:write",
"fs:24729b88-a4c5-4990-ad4e-272b87895732",
"fs"
]
},
{
"$": "path",
"via": "user",
"has_terminal": true,
"permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read",
"data": {},
"holder_username": "ed3",
"issuer_username": "admin",
"reading": [
{
"$": "explode",
"from": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read",
"to": [
"fs:24729b88-a4c5-4990-ad4e-272b87895732:read",
"fs:24729b88-a4c5-4990-ad4e-272b87895732:write",
"fs:24729b88-a4c5-4990-ad4e-272b87895732",
"fs"
]
},
{
"$": "option",
"permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732:read",
"source": "implied",
"by": "is-owner",
"data": {}
},
{
"$": "option",
"permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732:write",
"source": "implied",
"by": "is-owner",
"data": {}
},
{
"$": "option",
"permission": "fs:24729b88-a4c5-4990-ad4e-272b87895732",
"source": "implied",
"by": "is-owner",
"data": {}
},
{
"$": "time",
"value": 19
}
]
},
{
"$": "time",
"value": 20
}
]
```
Each object in the reading has a property named `$` which is the type for the object.
The most fundamental types for permission readings are `path` and `option`. A path
always contains another reading, which contains more paths or options. An option
specifies the permission string, the name of the rule that granted the permission,
and a data object which may hold additional claims.
Readings begin with an `explode` if there are multiple strings that may grant the
permission.
Readings end with a `time` that repots how long the reading took to help manage
the potential performance impact of complex permission graphs.
## Permission Service
### check(actor, permissions)
Returns true if the current actor has a path to any permission options matching
any of the permission strings specified by `permissions`. This is done by invoking
`scan()` and returning `true` if there are more than 0 permission options.
### scan(actor, permissions)
Returns a "reading". A permission reading is a JSON-serializable structure.
Readings are described above.
## Permission Scan Sequence
The `scan()` method of **PermissionService** invokes the permission scan sequence.
The permission scan sequence is a [Sequence](https://github.com/HeyPuter/puter/blob/0e0bfd6d7c92eed5080518a099c9a66a2f2dc9ec/src/backend/src/codex/Sequence.js)
that is defined in [scan-permission.js](src/backend/src/structured/sequence/scan-permission.js).
It invokes many "permission scanners" which are defined in
[permission-scanners.js](src/backend/src/unstructured/permission-scanners.js)
The Permission Scan Sequence is as follows:
- `grant_if_system` - if system user, push an option to the reading and stop
- `rewrite_permission` - process the permission through any permission string
rewriters that were registered with PermissionService by other services.
For example, since path-based file permissions aren't currently supported
the FilesystemService regsiters a rewriter that converts any `fs:/`
permission into a corresponding UUID permission.
- `explode_permission` - break the permission into multiple permissions
than are sufficient to grant the permission being scanned. For example if
there are multiple components, like `a.b.c`, having either permission `a.b` or
`a` granted implis having `a.b.c` granted. Other services can also register
"permission exploders" which handle non-hierarchical cases such as
`fs:AAAA:write` implying `fs:AAAA:read`.
- `run_scanners` - run the permission scanners.
Each permission scanner has a name, documentation text, and a scan function.
The scan function has access to the scan sequence's context and can push
objects onto the permission reading.
For information on individual scanners, refer to permission-scanners.js.
================================================
FILE: src/backend/doc/Kernel.md
================================================
# Puter Kernel Documentation
## Overview
The **Puter Kernel** is the core runtime component of the Puter system. It provides the foundational infrastructure for:
- Initializing the runtime environment
- Managing internal and external modules (extensions)
- Setting up and booting core services
- Configuring logging and debugging utilities
- Integrating with third-party modules and performing dependency installs at runtime
This kernel is responsible for orchestrating the startup sequence and ensuring that all necessary services, modules, and environmental configurations are properly loaded before the application enters its operational state.
---
## Features
1. **Modular Architecture**:
The Kernel supports both internal and external modules:
- **Internal Modules**: Provided to Kernel by an initializing script, such
as `tools/run-selfhosted.js`, via the `add_module()` method.
- **External Modules**: Discovered in configured module directories and installed
dynamically. This includes resolving and executing `package.json` entries and
running `npm install` as needed.
2. **Service Container & Registry**:
The Kernel initializes a service container that manages a wide range of services. Services can:
- Register modules
- Initialize dependencies
- Emit lifecycle events (`boot.consolidation`, `boot.activation`, `boot.ready`) to
orchestrate a stable and consistent environment.
3. **Runtime Environment Setup**:
The Kernel sets up a `RuntimeEnvironment` to determine configuration paths and environment parameters. It also provides global helpers like `kv` for key-value storage and `cl` for simplified console logging.
4. **Logging and Debugging**:
Uses a temporary `BootLogger` for the initialization phase until LogService is
initialized, at which point it will replace the boot logger. Debugging features
(`ll`, `xtra_log`) are enabled in development environments for convenience.
## Initialization & Boot Process
1. **Constructor**:
When a Kernel instance is created, it sets up basic parameters, initializes an empty
module list, and prepares `useapi()` integration.
2. **Booting**:
The `boot()` method:
- Parses CLI arguments using `yargs`.
- Calls `_runtime_init()` to set up the `RuntimeEnvironment` and boot logger.
- Initializes global debugging/logging utilities.
- Sets up the service container (usually called `services`c instance of **Container**).
- Invokes module installation and service bootstrapping processes.
3. **Module Installation**:
Internal modules are registered and installed first.
External modules are discovered, packaged, installed, and their code is executed.
External modules are given a special context with access to `useapi()`, a dynamic
import mechanism for Puter modules and extensions.
4. **Service Bootstrapping**:
After modules and extensions are installed, services are initialized and activated.
For more information about how this works, see [boot-sequence.md](./contributors/boot-sequence.md).
================================================
FILE: src/backend/doc/README.md
================================================
## Backend - Contributor Documentation
### Where to Start
Start with [Backend File Structure](./contributors/structure.md).
There also also some videos. In one of the videos Eric does a
Steve Ballmer impression so it's definitely worth it.
- [Services and Modules in Puter](https://www.youtube.com/watch?v=TOeS67QXMVU)
- [Puter's Boot Sequence](https://www.youtube.com/watch?v=a8bOLNnW1Uo)
- [Building a Driver on Puter](https://www.youtube.com/watch?v=8znQmrKgNxA)
### Index
- [Backend File Structure](./contributors/structure.md)
- [Boot Sequence](./contributors/boot-sequence.md)
- [Kernel](./Kernel.md)
- [Modules](./contributors/modules.md)
- [Configuring Logs](./log_config.md)
================================================
FILE: src/backend/doc/contributors/boot-sequence.md
================================================
# Puter Backend Boot Sequence
This document describes the boot sequence of Puter's backend.
**Runtime Environment**
- Configuration directory is determined
- Runtime directory is determined
- Mod directory is determined
- Services are instantiated
**Construction**
- Data structures are created
**Initialization**
- Registries are populated
- Services prepare for next phase
**Consolidation**
- Service event bus receives first event (`boot.consolidation`)
- Services perform coordinated setup behaviors
- Services prepare for next phase
**Activation**
- Blocking listeners of `boot.consolidation` have resolved
- HTTP servers start listening
**Ready**
- Services are informed that Puter is providing service
## Boot Phases
### Construction
Services implement a method called `construct` which initializes members
of an instance. Services do not override the class constructor of
**BaseService**. This makes it possible to use the `new` operator without
invoking a service's constructor behavior during debugging.
The first phase of the boot sequence, "construction", is simply a loop to
call `construct` on all registered services.
The `_construct` override should not:
- call other services
- emit events
### Initialization
At initialization, the `init()` method is called on all services.
The `_init` override can be used to:
- register information with other services, when services don't
need to register this information in a specific sequence.
An example of this is registering commands with CommandService.
- perform setup that is required before the consolidation phase starts.
### Consolidation
Consolidation is a phase where services should emit events that
are related to bringing up the system. For example, WebServerService
('web-server') emits an event telling services to install middlewares,
and later emits an event telling services to install routes.
Consolidation starts when Kernel emits `boot.consolidation` to the
services event bus, which happens after `init()` resolves for all
services.
### Activation
Activation is a phase where services begin listening on external
interfaces. For example, this is when the web server starts listening.
Activation starts when Kernel emits `boot.activation`.
### Ready
Ready is a phase where services are informed that everything is up.
Ready starts when Kernel emits `boot.ready`.
## Events and Asynchronous Execution
The services event bus is implemented so you can `await` a call to `.emit()`.
Event listeners can choose to have blocking behavior by returning a promise.
During emission of a particular event, listeners of this event will not
block each other, but all listeners must resolve before the call to
`.emit()` is resolved. (i.e. `emit` uses `Promise.all`)
## Legacy Services
Some services were implemented before the `BaseService` class - which
implements the `init` method - was created. These services are called
"legacy services" and they are instantiated _after_ initialization but
_before_ consolidation.
================================================
FILE: src/backend/doc/contributors/coding-style.md
================================================
# Backend Style
## File Structure
### Copyright Notice
All files should begin with the standard copyright notice:
```javascript
/*
* Copyright (C) 2025-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } The user object
* @throws {Error} If user not found
*/
async function getUserById(id) {
// ...
}
```
### Inline Comments
- Use inline comments to explain complex logic
- Prefix comments with tags like `track:` to indicate specific purposes
```javascript
// track: slice a prefix
const uid = uid_part.slice('uid#'.length);
```
================================================
FILE: src/backend/doc/contributors/modules.md
================================================
# Puter Kernel Moduels and Services
## Modules
A Puter kernel module is simply a collection of services that run when
the module is installed. You can find an example of this in the
`run-selfhosted.js` script at the root of the Puter monorepo.
Here is the relevant excerpt in `run-selfhosted.js` at the time of
writing this documentation:
```javascript
const {
Kernel,
CoreModule,
DatabaseModule,
LocalDiskStorageModule,
SelfHostedModule
} = (await import('@heyputer/backend')).default;
const k = new Kernel();
k.add_module(new CoreModule());
k.add_module(new DatabaseModule());
k.add_module(new LocalDiskStorageModule());
k.add_module(new SelfHostedModule());
k.boot();
```
A few modules are added to Puter before booting. If you want to install
your own modules into Puter you can edit this file for self-hosted runs
or create your own script that boots Puter. This makes it possible to
have deployments of Puter with custom functionality.
To function properly, Puter needs **CoreModule**, a database module,
and a storage module.
A module extends
[AdvancedBase](../../../putility/README.md)
and implements
an `install` method. The install method has one parameter, a
[Context](../../src/util/context.js)
object containing all the values kernel modules have access to. This
includes the `services`
[Container](../../src/services/Container.js`).
A module adds services to Puter.eA typical module may look something
like this:
```javascript
class MyPuterModule extends AdvancedBase {
async install (context) {
const services = context.get('services');
const MyService = require('./path/to/MyService.js');
services.registerService('my-service', MyService, {
some_options: 'for-my-service',
});
}
}
```
## Services
Services extend
[BaseService](../../src/services/BaseService.js)
and provide additional functionality for Puter. They can add HTTP
endpoints and register objects with other services.
When implementing a service it is important to understand
Puter's [boot sequence](./boot-sequence.md)
A typical service may look like this:
```javascript
class MyService extends BaseService {
static MODULES = {
// Use node's `require` function to populate this object;
// this makes these available to `this.require` and offers
// dependency-injection for unit testing.
['some-module']: require('some-module')
}
// Do not override the constructor of BaseService - use this instead!
async _construct () {
this.my_list = [];
}
// This method is called after _construct has been called on all
// other services.
async _init () {
const services = this.services;
// We can get the instances of other services here
const svc_otherService = services.get('other-service');
}
// The service container can listen on the "service event bus"
async ['__on_boot.consolidation'] () {}
async ['__on_boot.activation'] () {}
async ['__on_start.webserver'] () {}
async ['__on_install.routes'] () {}
}
```
================================================
FILE: src/backend/doc/contributors/structure.md
================================================
# Puter Backend - Directory Structure
## MFU - Most Frequently Used
These locations under `/src/backend/src` are the most important
to know about. Whether you're contributing a feature or fixing a bug,
you might only need to look at code in these locations.
### `modules` directory
The `modules` directory contains Puter backend kernel modules only.
Everything in here has a `Module.js` file and one or more
`Service.js` files.
> **Note:** A "backend kernel module" is simply a class understood by
[`src/backend/src/Kernel.js`](../../src/Kernel.js)
that registers a number of "Service" classes.
You can look at [Puter's init file](../../../../tools/run-selfhosted.js)
to see how modules are added to Puter.
The `README.md` file inside any module directory is generated with
the `module-docgen` script in the Puter repo's `/tools` directory.
The actual documentation for the module exists in jsdoc comments
in the source files.
Each module might contain these directories:
- `doc/` - additional module documentation, like sample requests
- `lib/` - utility code that isn't a Module or Service class.
This utility code may be exposed by a service in the module
to Puter's runtime import mechanism for extension support.
### `services` directory
This directory existed before the `modules` directory. Most of
the services here go on a module called **CoreModule**
(CoreModule.js is directly in `/src/backend/src`), but this
directory can be thought of as "services that are not yet
organized in a distinct module".
### `routers` directory
While routes are typically registered by Services, the implementation
of a route might be placed under `src/backend/src/routers` to keep the
service's code tidy or for legacy reasons.
These are some services that reference files under `src/backend/src/routers`:
- [PermissionAPIService](../../src/services/PermissionAPIService.js) -
This service registers routes that allow a user to configure permissions they
grant to apps and groups. This is a relatively recent case of using files under the
`routers` directory to clean up the service.
- [UserProtectedEndpointsService](../../src/services/web/UserProtectedEndpointsService.js) -
This service follows a slightly different approach where files under
`routers/user-protected` contain an "endpoint specification" instead of an express
handler function. This might be good inspiration for future routes.
- [PuterAPIService](../../src/services/PuterAPIService.js) -
This service is a catch-all for routes that existed before separation of concerns
into backend kernel modules.
### `filesystem` directory
The filesystem is likely the most complex portion of Puter's source code. This code
is in its own directory as a matter of circumstance more than intention. Ideally the
filesystem's concerns will be split across a few modules as we prepare to add
support for mounting different file systems and improved cache behavior.
For example, Puter's native filesystem implementation should be mostly moved to
`src/backend/src/modules/puterfs` as we continue this development.
Since this directory is in flux, don't trust this documentation completely.
If you're contributing to filesystem,
[tag @KernelDeimos on the community Discord](https://discord.gg/PQcx7Teh8u)
if you have questions.
These are the key locations in the `filesystem` directory:
- `FSNodeContext.js` - When you have a reference to a file or directory in backend code,
it is an instance of the FSNodeContext class.
- `ll_operations` - Runnables that implement the behavior of a filesystem operation.
These used to include the behavior of Puter's filesystem, but they now delegate
the actual behavior to the implementation in the `.provider` member of a
FSNodeContext (filesystem node / a file or directory) so that we can eventually
support "mountpoints" (multiple filesystem implementations).
- `hl_operations` - Runnables that implement the behavior of higher-level versions
of filesystem operations. For example, the high-level mkdir operation might create
multiple directories in chain; the high-level write might change the name of the
file to avoid conflicts if you specify the `dedupe_name` flag.
================================================
FILE: src/backend/doc/dev_socket.md
================================================
## Backend - dev socket
The "dev socket" allows you to interact with Puter's backend by running commands.
It's a UNIX socket that lets you run commands registered with
[CommandService](../../src/services/CommandService.js) (e.g. `help`, `logs:indent`, `params:get`, etc.).
### Enabling the dev socket
The dev socket is provided by the **dev-console extension** and is **opt-in**. To enable it:
1. Set the environment variable `DEVCONSOLE=1` when starting Puter (e.g. `npm run dev` already does this).
2. The extension lives in `extensions/dev-console/` and registers a `dev-socket` service when `DEVCONSOLE=1`.
### Socket location
The socket is created in a directory chosen as follows (in order):
- `PUTER_DEV_SOCKET_DIR` if set
- `./volatile/runtime` if it exists (typical local dev)
- otherwise the process current working directory
The socket file is named `dev.sock`.
### Connecting
When in that directory, connect with your tool of choice. For example, using `nc` and `rlwrap` for readline history:
```
rlwrap nc -U ./dev.sock
```
If it is successful you will see a message with instructions. Enter a command (e.g. `help`) and press Enter.
================================================
FILE: src/backend/doc/extensions/README.md
================================================
# Puter Backend Extensions
## What Are Extensions
Extensions can extend the functionality of Puter's backend by handling specific
events or importing/exporting runtime libraries.
## Creating an Extension
The easiest way to create an extension is to place a new file or directory under
the `extensions/` directory immediately under the root directory of the Puter
repository. If your extension is a single `.js` file called `my-extension.js` it
will be implicitly converted into a CJS module with the following structure:
```
extensions/
|
|- my-extension/
|
|- package.json
|- main.js
```
The location of the extensions directory can be changed in
[the config file](../../../../doc/self-hosters/config.md)
by setting `mod_directories` to an array of valid locations.
The `mod_directories` parameter has the following default value:
```json
["{repo}/mods/mods_enabled", "{repo}/extensions"]
```
### Events
The primary mechanism of communication between extensions and Puter,
and between different extensions, is through events. The `extension`
pseudo-global provides `.on(fn)` to add event listemers and
`.emit('name', { arbitrary: 'data' })` to emit events.
To try working with events, you could make a simple extension that
emits an event after adding a listener for its own event:
```javascript
// Listen to a test event called 'test-event'
extension.on('test-event', event => {
console.log(`We got the test event from ${sender}`);
});
// Listen to init; a good time to emit events
extension.on('init', event => {
extension.emit('test-event', { sender: 'Quinn' });
});
```
### Puter Extension Imports
Your extensions may need to invoke specific actions in Puter's backend
in response to an event. Puter provides libraries at runtime which you
can access via `extension.imports`:
```javascript
const { kv } = extension.imports('data');
kv.set('some-key', 'some value');
```
#### The `data` import
The data import makes it possible to access Puter's database, persistent
key-value store, and in-memory cache.
- [Read more about the 'data' import](./builtins/data.md)
### Adding Features to Puter
- [Implementing Drivers](./pages/drivers.md)
### Bundled extensions
- **dev-console** – When `DEVCONSOLE=1` is set (e.g. `npm run dev`), the dev-console extension registers a UNIX socket (`dev.sock`) so you can run backend commands (see [CommandService](../../src/services/CommandService.js)) from a terminal. See [Backend – dev socket](../dev_socket.md).
## Extensions - Planned Features
Extensions are under refactor currently. This is the checklist:
- [x] Add RuntimeModule construct for imports and exports
- [x] Add support to implement drivers in extensions
- [ ] Add the ability to target specific extensions when
emitting events
- [ ] Add event name aliasing and configurable import mapping
- [ ] Extract extension loading from the core
- [ ] List exports in console
================================================
FILE: src/backend/doc/extensions/builtins/data.md
================================================
## Extensions - the `data` extension
The `data` extension can be imported in custom extensions for access
to the database and key-value store.
You can import these from `'data'`:
- `db` - Puter's main SQL database
- `kv` - A persistent key-value store
- `cache` - In-memory [kv.js](https://github.com/HeyPuter/kv.js/) store
```javascript
const { db, kv, cache } = extension.import('data');
```
### Database (`db`)
Don't forget to import it first!
```javascript
const { db } = extension.import('data');
```
#### `db.read`
Usage:
```javascript
const rows = await db.read('SELECT * FROM apps WHERE `name` = ?', [
'editor'
]);
```
#### `db.write`
Usage:
```javascript
const {
insertId, // internal ID of new row (if this is an INSERT)
anyRowsAffected, // true if 1 or more rows were affected
} = await db.write(
// A query like INSERT, UPDATE, DELETE, etc...
'INSERT INTO example_table (a, b, c) VALUES (?, ?, ?)',
// Parameters (all user input should go here)
[
"Value for column a",
"Value for column b",
"Value for column c",
]
);
```
### Persistent KV Store (`kv`)
Don't forget to import it first!
```javascript
const { kv } = extension.import('data');
```
#### `kv.get({ key })`
```javascript
// Short-Form (like kv.js)
const someValue = kv.get('some-key');
// Long-Form (the `puter-kvstore` driver interface)
const someValue = kv.get({ key: 'some-key' });
```
#### `kv.set({ key, value })`
```javascript
await kv.set('some-key', 'some value');
// or...
await kv.set({
key: 'some-key',
value: 'some value',
});
```
#### `kv.expire({ key, ttl })`
This key will persist for 20 minutes, even if the server restarts.
```javascript
kv.expire({
key: 'some-key',
ttl: 1000 * 60 * 20, // 20 minutes
});
```
### `kv.expireAt({ key, timestamp })`
The following example expires a key 1 second before
["the apocalypse"](https://en.wikipedia.org/wiki/Year_2038_problem).
(don't worry, KV won't break in 2038)
```javascript
kv.expireAt(
key: 'some-key',
// Expires Jan 19 2038 3:14:07 GMT
timestamp: 2147483647,
);
```
### In-Memory Cache (`cache`)
Don't forget to import it first!
```javascript
const { cache } = extension.import('data');
```
The in-memory cache is provided by [kv.js](https://github.com/HeyPuter/kv.js).
Below is a simple example.
For comprehensive documentation, see the [kv.js repository's readme](https://github.com/HeyPuter/kv.js/blob/main/README.md).
```javascript
const { cache } = extension.require('data');
cache.set('some-key', 'some value');
const value = cache.get('some-key'); // some value
// This value only exists for 5 minutes
cache.set('temporary', 'abcdefg', { EX: 5 * 60 });
cache.incr('qwerty'); // cache.get('qwerty') is now: 1
cache.incr('qwerty'); // cache.get('qwerty') is now: 2
```
================================================
FILE: src/backend/doc/extensions/pages/core-devs.md
================================================
## Extensions - Technical Context for Core Devs
This document provides technical context for extensions from the perspective of
core backend modules and services, including the backend kernel.
### Lifecycle
For extensions, the concept of an "init" event handler is different from core.
This is because a developer of an extension expects `init` to occur after core
modules and services have been initialized. For this reason, extensions receive
`init` when backend services receive `boot.consolidation`.
It is still possible to handle core's `init` event in an extension. This is done
using the `preinit` event.
```
Backend Core Lifecycle
Modules -> Construction -> Initialization -> Consolidation -> Activation -> Ready
Extension Lifecycle
index.js executed -> (no event) -> 'preinit' -> 'init' -> (no event) -> 'ready'
```
Extensions have an implicit Service instance that needs to listen for events on
the **Service Event Bus** such as `install.routes` (emitted by WebServerService).
Since extensions need to affect the behavior of the service when these events
occur (for example using `extension.post()` to add a POST handler) it is necessary
for their entry files to be loaded during a module installation phase, when
services are being registered and `_construct()` has not yet been called on any
service.
Kernel.js loads all core modules/services before any extensions. This allows
core modules and services to create [runtime modules](./runtime-modules.md)
which can be imported by services.
### How Extensions are Loaded
Before extensions are loaded, all of Puter's core modules have their `.install()`
methods called. The core modules are the ones added with `kernel.add_module`,
for example in [run-selfhosted.js](../../../../../tools/run-selfhosted.js).
Then, `Kernel.install_extern_mods_` is called. This is where a `readdir` is
performed on each directory listed in the `"mod_directories"` configuration
parameter, which has a default value of `["{repo}/extensions"]` (the
placeholder `{repo}` is automatically replaced with the path to the Puter
repository).
For each item in each mod directory, except for ignored items like `.git`
directories, a mod is installed. First a directory is created in Puter's
runtime directory (`volatile/runtime` locally, `/var/puter` on a server).
If the item is a file then a `package.json` will be created for it after
`//@extension` directives are processed. If the item is a directory then
it is copied as is and `//@extension` directives are not supported
(`puter.json` is used instead). Source files for the mod are copied to
the mod directory under the runtime directory.
It is at this point the pseudo-globals are added be prepending `cost`
declarations at the top of `.js` files in the extension. This is not
a great way to do this, but there is a severe lack of options here.
See the heading below - "Extension Pseudo-Globals" - for details.
Before the entry file for the extension is `require()`'d a couple of
objects are created: an `ExtensionModule` and an `Extension`.
The `ExtensionModule` is a Puter module just like any of the Puter core
modules, so it has an `.install()` method that installs services before
Puter's kernel starts the initialization sequence. In this case it will
install the implied service that an extension creates if it registers
routes or performs any other action that's typically done inside services
in core modules.
A RuntimeModule is also created. This could be thought of as analygous
to node's own `Module` class, but instead of being for imports/exports
between npm modules it's for imports/exports between Puter extensions
loaded at runtime. (see [runtime modules](./runtime-modules.md))
### Extension Pseudo-Globals
The `extension` global is a different object per extension, which will
make it possible to develop "remapping" for imports/exports when
extension names collide among other functions that need context about
which extension is calling them. Implementing this per-extension global
was very tricky and many solutions were considered, including using the
`node:vm` builtin module to run the extension in a different instance.
Unfortunately `node:vm` support for EMCAScript Modules is lacking;
`vm.Module` has a drastically different API from `vm.Script`, requires
an experimental feature flag to be passed to node, and does not provide
any alternative to `createRequire` to make a valid linker for the
dependencies of a package being run in `node:vm`.
The current solution - which sucks - is as follows: prepend `const`
definitions to the top of every `.js` file in the extension's installation
directory unless it's under a directory called `node_modules` or `gui`.
This type of "pseudo-global" has a quirk when compared to real globals,
which is that they can't be shadowed at the root scope without an error
being thrown. The naive solution of wrapping the rest of the file's
contents in a scope limiter (`{ ... }`) would break ES Module support
because `import` directives must be in the top-level scope, and the naive
solution to that problem of moving imports to the top of the file after
adding the scope limiter requires invoking a javascript parser do
determine the difference between a line starting with `import` because
it's actually an import and this unholy abomination of a situation:
```
console.log(`
import { me, and, everything, breaks } from 'lackOfLexicalAnalysis';
`);
```
Exposing the same instance for `extension` to all extensions with a
real global and using AsyncLocalStorage to get the necessary information
about the calling extension on each of `extension`'s methods was another
idea. This would cause surprising behavior for extension developers when
calling methods on `extension` in callbacks that lose the async context
fail because of missing extension information.
Eventually a better compromise will be to have commonjs extensions
run using `vm.Script` and ESM extensions continue to run using this hack.
### Event Listener Sub-Context
In extensions, event handlers are registered using `extension.on`. These
handlers, when called, are supplemented with identifying information for
the extension through AsyncLocalStorage. This means any methods called
on the object passed from the event (usually just called `event`) will
be able to access the extension's name.
This is used by CommandService's `create.commands` event. For example
the following extension code will register the command `utils:say-hello`
if it is invoked form an extension named `utils`:
```javascript
extension.on('create.commands', event => {
event.createCommand('say-hello', async (args, console) => {
console.log('Hello,', ...args);
});
});
```
================================================
FILE: src/backend/doc/extensions/pages/drivers.md
================================================
## Extensions - Implementing Drivers
Puter's concept of drivers has existed long before the extension system
was refined, and to keep things moving forward it has become easier to
develop Puter drivers in extensions than anywhere else in Puter's source.
If you want to build a driver, an extension is the recommended way to do it.
### What are Puter drivers?
Puter drivers are all called through the `/drivers/call` endpoint, so they
can be thought of as being "above" the HTTP layer. When a method on a driver
throws an error you will still receive a `200` HTTP status response because
the the invocation - from the HTTP layer - was successful.
A driver response follows this structure:
```json
{
"success": true,
"service": {
"name": "implementation-name"
},
"result": "any type of value goes here",
"metadata": {}
}
```
There exists an example driver called `hello-world`. This driver implements
a method called `greet` with the optional parameter `subject` which returns
a string greeting either `World` (default) or the specified subject.
```javascript
await puter.call('hello-world', 'no-frills', 'greet', { subject: 'Dave' });
```
Let's break it down:
#### `'hello-world'`
`'hello-world'` is the name of an "interface". An interface can be thought of
a contract of what inputs are allowed and what outputs are expected. For
example the `hello-world` interface specifies that there must be a method
called `greet` and it should return a string representing a greeting.
To add another example, an interface called `weather` specify a method called
`forcast5day` that always returns a list of 5 objects with a particular
structure.
#### `no-frills`
`'no-frills'` is a simple - "no frills" (nothing extra) - implementation of
the `hello-world` interface. All it does is return the string:
```javascript
`Hello, ${subject ?? 'World'}!`
```
#### `'greet'`
`greet` is the method being called. It's the only method on the `hello-world`
interface.
#### `{ subject: 'Dave' }`
These are the arguments to the `greet` method. The arguments specify that we
want to say "Hello" to Dave. Hopefully he doesn't ask us to open the pod bay
doors, or if he does we hopefully have extensions to add a driver interface
and driver implementation for the pod bay doors so that we can interact with
them.
### Drivers in Extensions
The `hellodriver` extension adds the `hello-world` interface like this:
```javascript
extension.on('create.interfaces', event => {
// createInterface is the only method on this `event`
event.createInterface('hello-world', {
description: 'Provides methods for generating greetings',
methods: {
greet: {
description: 'Returns a greeting',
parameters: {
subject: {
type: 'string',
optional: true
},
locale: {
type: 'string',
optional: true
},
}
}
}
})
});
```
The `hellodriver` extension adds the `no-frills` implementation for
`hello-world` like this:
```javascript
extension.on('create.drivers', event => {
event.createDriver('hello-world', 'no-frills', {
greet ({ subject }) {
return `Hello, ${subject ?? 'World'}!`;
}
});
});`
```
You can pass an instance of a class for a driver implementation as well:
```javascript
class Greeter {
greet ({ subject }) {
return `Hello, ${subject ?? 'World'}!`;
}
}
extension.on('create.drivers', event => {
event.createDriver('hello-world', 'no-frills', new Greeter());
});`
```
Instances of classes being supported
may seem to be implied by the example before this
one, but that is not the case. What's shown here is that function members
of the object passed to `createDriver` will not be "bound" (have their
`.bind()` method called with a different object as the instance variable).
### Permission Denied
When you try to access a driver as any user other than the default
`admin` user, it will not work unless permission has been granted.
The `hellodriver` extension grants permission to all clients using
the following snippet:
```javascript
extension.on('create.permissions', event => {
event.grant_to_everyone('service:no-frills:ii:hello-world');
});
```
The `create.permissions` event's `event` object has a few methods
you can use depending on the desired granularity:
- `grant_to_everyone` - grants permission to all users
- `grant_to_users` - grants permission to only registered users
(i.e. not to temporary/guest users)
================================================
FILE: src/backend/doc/extensions/pages/import-and-export.md
================================================
## Extensions - Importing & Exporting
Here are two extensions. One extension has an "extension export" (an export to
other extensions) and an "extension import" (an import from another extension).
This is different from regular `import` or `require()` because it resolves to
a Puter extension loaded at runtime rather than an `npm` module.
To import and export in Puter extensions, we use `extension.import()` and `extension.exports`.
`exports-something.js`
```javascript
//@puter priority -1
// ^ setting load priority to "-1" allows other extensions to import
// this extension's exports before the initialization event occurs
// Just like "module.exports", but for extensions!
extension.exports = {
test_value: 'Hello, extensions!',
};
```
`imports-something.js`
```javascript
const { test_value } = extension.import('exports-something');
console.log(test_value); // 'Hello, extensions!'
```
================================================
FILE: src/backend/doc/extensions/pages/runtime-modules.md
================================================
## Extensions - Runtime Modules
Runtime modules are modules that extensions can import with tihs syntax:
```javascript
const somelib = extension.import('somelib');
```
These modules are registered in the [runtime module registry](../../../src/extension/RuntimeModuleRegistry.js)
which is instantiated by [Kernel.js](../../../src/Kernel.js).
All extensions implicitly have a Runtime Module. The runtime module shares the name
of the extension that it corresponds to. Extensions can export to their module by
using `extension.exports`:
```javascript
extension.exports = { /* ... */ };
```
The [Extension](../../../src/Extension.js) object proxies this call to the
runtime module (called `this.runtime` in the snippet):
```javascript
class Extension extends AdvancedBase {
// ...
set exports (value) {
this.runtime.exports = value;
}
// ...
}
```
You may be wondering why RuntimeModule is a separate class from Extension,
rather than just registering extensions into this registry.
Separating RuntimeModule allows core code that has not yet been migrated
to extensions to export values as if they came from extensions.
Since core modules are loaded before extensions, this allows any legacy
`useapi` definitions be be exported where modules are installed.
For example, in [CoreModule.js](../../../src/CoreModule.js) this snippet
of code is used to add a runtime module called `core`:
```javascript
// Extension compatibility
const runtimeModule = new RuntimeModule({ name: 'core' });
context.get('runtime-modules').register(runtimeModule);
runtimeModule.exports = useapi.use('core');
```
================================================
FILE: src/backend/doc/features/batch-and-symlinks.md
================================================
# Batch and Symlinks
2024-10-08
### Batch and Symlinks
All filesystem operations will eventually be available through batch requests.
Since batch requests can also handle the cases for single files, it seems silly
to support those endpoints too, so eventually most calls will be done through
`/batch`. Puter's legacy filesystem endpoints will always be supported, but a
future `api.___/fs/v2.0` urlspace for the filesystem API might not include them.
This is batch:
```javascript
await (async () => {
const endpoint = 'http://api.puter.localhost:4100/batch';
const ops = [
{
op: 'mkdir',
path: '/default_user/Desktop/some-dir',
},
{
op: 'write',
path: '/default_user/Desktop/some-file.txt',
}
];
const blob = new Blob(["12345678"], { type: 'text/plain' });
const formData = new FormData();
for ( const op of ops ) {
formData.append('operation', JSON.stringify(op));
}
formData.append('fileinfo', JSON.stringify({
name: 'file.txt',
size: 8,
mime: 'text/plain',
}));
formData.append('file', blob, 'hello.txt');
const response = await fetch(endpoint, {
method: 'POST',
headers: { 'Authorization': `Bearer ${puter.authToken}` },
body: formData
});
return await response.json();
})();
```
Symlinks are also created via `/batch`
```javascript
await (async () => {
const endpoint = 'http://api.puter.localhost:4100/batch';
const ops = [
{
op: 'symlink',
path: '~/Desktop',
name: 'link',
target: '/bb/Desktop/some'
},
];
const formData = new FormData();
for ( const op of ops ) {
formData.append('operation', JSON.stringify(op));
}
const response = await fetch(endpoint, {
method: 'POST',
headers: { 'Authorization': `Bearer ${puter.authToken}` },
body: formData
});
return await response.json();
})();
```
================================================
FILE: src/backend/doc/features/protected-apps.md
================================================
# Protected Apps and Subdomains
## Protected Sites
If a site is not protected, anyone can access the site.
When a site is protected, the following changes:
- The site can only be accessed inside a Puter app iframe
- Only users with explicit permission will be able to load
the page associated with the site.
## Protected Apps
If an app is not protected, anyone with the name of the
app or its UUID will be able to access the app.
If the app is **approved for listing** (todo: doc this)
all users can access the app.
If an app is protected, the following changes:
- The app can only be "seen" (listed) by users
with explicit permission.
- App metadata can only be accessed by users
with explicit permission.
Note that an app being protected does not imply that the
site is protected. If a user action results in an app
being protected it should also result in the site (subdomain)
being protected **if they own it**. If the site will not
be protected the user should have some indication.
================================================
FILE: src/backend/doc/features/service-scripts.md
================================================
> **NOTICE:** This documentation is new and might contain errors.
> Feel free to open a Github issue if you run into any problems.
# Service Scripts
## What is a Service Script?
Service scripts allow backend services to provide client-side code that
runs in Puter's GUI. This is useful if you want to make a mod or plugin
for Puter that has backend functionality. For example, you might want
to add a tab to the settings panel to make use of or configure the service.
Service scripts are made possible by the `puter-homepage` service, which
allows you to register URLs for additional javascript files Puter's
GUI should load.
## ES Modules - A Problem of Ordering
In browsers, script tags with `type=module` implicitly behave according
to those with the `defer` attribute. This means after the DOM is loaded
the scripts will run in the order in which they appear in the document.
Relying on this execution order however does not work. This is because
`import` is implicitly asynchronous. Effectively, this means these
scripts will execute in arbitrary order if they all have imports.
In a situation where all the client-side code is bundled with rollup
or webpack this is not an issue as you typically only have one
entry script. To facilitate loading service scripts, which are not
bundled with the GUI, we require that service scripts call the global
`service_script` function to access the API for service scripts.
## Providing a Service Script
For a service to provide a service script, it simply needs to serve
static files (the "service script") on some URL, and register that
URL with the `puter-homepage` service.
In this example below we use builtin functionality of express to serve
static files.
```javascript
class MyService extends BaseService {
async _init () {
// First we tell `puter-homepage` that we're going to be serving
// a javascript file which we want to be included when the GUI
// loads.
const svc_puterHomepage = this.services.get('puter-homepage');
svc_puterHomepage.register_script('/my-service-script/main.js');
}
async ['__on_install.routes'] (_, { app }) {
// Here we ask express to serve our script. This is made possible
// by WebServerService which provides the `app` object when it
// emits the 'install.routes` event.
app.use('/my-service-script',
express.static(
PathBuilder.add(__dirname).add('gui').build()
)
);
}
}
```
## A Simple Service Script
```javascript
import SomeModule from "./SomeModule.js";
service_script(api => {
api.on_ready(() => {
// This callback is invoked when the GUI is ready
// We can use api.get() to import anything exposed to
// service scripts by Puter's GUI; for example:
const Button = api.use('ui.components.Button');
// ^ Here we get Puter's Button component, which is made
// available to service scripts.
});
});
```
## Adding a Settings Tab
Starting with the following example:
```javascript
import MySettingsTab from "./MySettingsTab.js";
globalThis.service_script(api => {
api.on_ready(() => {
const svc_settings = globalThis.services.get('settings');
svc_settings.register_tab(MySettingsTab(api));
});
});
```
The module **MySettingsTab** exports a function for scoping the `api`
object, and that function returns a settings tab. The settings tab is
an object with a specific format that Puter's settings window understands.
Here are the contents of `MySettingsTab.js`:
```javascript
import MyWindow from "./MyWindow.js";
export default api => ({
id: 'my-settings-tab',
title_i18n_key: 'My Settings Tab',
icon: 'shield.svg',
factory: () => {
const NotifCard = api.use('ui.component.NotifCard');
const ActionCard = api.use('ui.component.ActionCard');
const JustHTML = api.use('ui.component.JustHTML');
const Flexer = api.use('ui.component.Flexer');
const UIAlert = api.use('ui.window.UIAlert');
// The root component for our settings tab will be a "flexer",
// which by default displays its child components in a vertical
// layout.
const component = new Flexer({
children: [
// We can insert raw HTML as a component
new JustHTML({
no_shadow: true, // use CSS for settings window
html: 'Some Heading ',
}),
new NotifCard({
text: 'I am a card with some text',
style: 'settings-card-success',
}),
new ActionCard({
title: 'Open an Alert',
button_text: 'Click Me',
on_click: async () => {
// Here we open an example window
await UIAlert({
message: 'Hello, Puter!',
});
}
})
]
});
return component;
}
});
```
================================================
FILE: src/backend/doc/howto_make_driver.md
================================================
# How to Make a Puter Driver
## What is a Driver?
A driver can be one of two things depending on what you're
talking about:
- a **driver interface** describes a general type of service
and what its parameters and result look like.
For example, `puter-chat-completion` is a driver interface
for AI Chat services, and it specifies that any service
on Puter for AI Chat needs a method called `complete` that
accepts a JSON parameter called `messages`.
- a **driver implementation** exists when a **Service** on
Puter implements a **trait** with the same name as a
driver interface.
## Part 1: Choose or Create a Driver Interface
Available driver interfaces exist at this location in the repo:
[/src/backend/src/services/drivers/interfaces.js](../src/services/drivers/interfaces.js).
When creating a new Puter driver implementation, you should check
this file to see if there's an appropriate interface. We're going
to make a driver that returns greeting strings, so we can use the
existing `hello-world` interface. If there wasn't an existing
interface, it would need to be created. Let's break down this
interface:
```javascript
'hello-world': {
description: 'A simple driver that returns a greeting.',
methods: {
greet: {
description: 'Returns a greeting.',
parameters: {
subject: {
type: 'string',
optional: true,
},
},
result: { type: 'string' },
}
}
},
```
The **description** describes what the interface is for. This
should be provided that both driver developers and users can
quickly identify what types of services should use it.
The **methods** object should have at least one entry, but it
may have more. The key of each entry is the name of a method;
in here we see `greet`. Each method also has a description,
a **parameters** object, and a **result** object.
The **parameters** object has an entry for each parameter that
may be passed to the method. Each entry is an object with a
`type` property specifying what values are allowed, and possibly
an `optional: true` entry.
All methods for Puter drivers use _named parameters_. There are no
positional parameters in Puter driver methods.
The **result** object specifies the type of the result. A service
called DriverService will use this to determine the response format
and headers of the response.
## Part 2: Create a Service
Creating a service is very easy, provided the service doesn't do
anything. Simply add a class to `src/backend/src/services` or into
the module of your choice (`src/backend/src/modules/`)
that looks like this:
```javascript
const BaseService = require('./BaseService')
// NOTE: the path specified ^ HERE might be different depending
// on the location of your file.
class PrankGreetService extends BaseService {
}
```
Notice I called the service "PrankGreet". This is a good service
name because you already know what the service is likely to
implement: this service generates a greeting, but it is a greeting
that intends to play a prank on whoever is beeing greeted.
Then, register the service into a module. If you put the service
under `src/backend/src/services`, then it goes in
[CoreModule](..//src/CoreModule.js) somewhere near the end of
the `install()` method. Otherwise, it will go in the `*Module.js`
file in the module where you placed your service.
The code to register the service is two lines of code that will
look something like this:
```javascript
const { PrankGreetServie } = require('./path/to/PrankGreetServie.js');
services.registerService('prank-greet', PrankGreetServie);
```
## Part 3: Verify that the Service is Registered
It's always a good idea to verify that the service is loaded
when starting Puter. Otherwise, you might spend time trying to
determine why your code doesn't work, when in fact it's not
running at all to begin with.
To do this, we'll add an `_init` handler to the service that
logs a message after a few seconds. We wait a few seconds so that
any log noise from boot won't bury our message.
```javascript
class PrankGreetService extends BaseService {
async _init () {
// Wait for 5 seconds
await new Promise(rslv => setTimeout(rslv), 5000);
// Display a log message
console.debug('Hello from PrankGreetService!');
}
}
```
## Part 4: Implement the Driver Interface in your Service
Now that it has been verified that the service is loaded, we can
start implementing the driver interface we chose eralier.
```javascript
class PrankGreetService extends BaseService {
async _init () {
// ... same as before
}
// Now we add this:
static IMPLEMENTS = {
['hello-world']: {
async greet ({ subject }) {
if ( subject ) {
return `Hello ${subject}, tell me about updog!`;
}
return `Hello, tell me about updog!`;
}
}
}
}
```
## Part 5: Test the Driver Implementation
We have now created the `prank-greet` implementation of `hello-world`.
Let's make a request in the browser to check it out. The example below
is a `fetch` call using `http://api.puter.localhost:4100` as the API
origin, which is the default when you're running Puter's backend locally.
Also, in this request I refer to `puter.authToken`. If you run this
snippet in the Dev Tools window of your browser from a tab with Puter
open (your local Puter, to be precise), this should contain the current
value for your auth token.
```javascript
await (await fetch("http://api.puter.localhost:4100/drivers/call", {
"headers": {
"Content-Type": "application/json",
"Authorization": `Bearer ${puter.authToken}`,
},
"body": JSON.stringify({
interface: 'hello-world',
service: 'prank-greet',
method: 'greet',
args: {
subject: 'World',
},
}),
"method": "POST",
})).json();
```
**You might see a permissions error!** Don't worry, this is expected;
in the next step we'll add the required permissions.
## Part 6: Permissions
In the previous step, you will only have gotten a successful response
if you're logged in as the `admin` user. If you're logged in as another
user you won't have access to the service's driver implementations be
default.
To grant permission for all users, update
[hardcoded-permissions.js](../src/data/hardcoded-permissions.js).
First, look for the constant `hardcoded_user_group_permissions`.
Whereever you see an entry for `service:hello-world:ii:hello-world`, add
the corresponding entry for your service, which will be called
```
service:prank-greet:ii:hello-world
```
To help you remember the permission string, its helpful to know that
`ii` in the string stands for "invoke interface". i.e. the scope of the
permission is under `service:prank-greet` (the `prank-greet` service)
and we want permission to invoke the interface `hello-world` on that
service.
You'll notice each entry in `hardcoded_user_group_permissions` has a value
determined by a call to the utility function `policy_perm(...)`. The policy
called `user.es` is a permissive policy for storage drivers, and we can
re-purpose it for our greeting implementor.
The policy of a permission determines behavior like rate limiting. This is
an advanced topic that is not covered in this guide.
If you want apps to be able to access the driver implementation without
explicit permission from a user, you will need to also register it in the
`default_implicit_user_app_permissions` constant. Additionally, you can
use the `implicit_user_app_permissions` constant to grant implicit
permission to the builtin Puter apps only.
Permissions to implementations on services can also be granted at runtime
to a user or group of users using the permissions API. This is beyond the
scope of this guide.
## Part 7: Verify Successful Response
If all went well, you should see the response in your console when you
try the request from Part 5. Try logging into a user other than `admin`
to verify permisison is granted.
```json
"Hello World, tell me about updog!"
```
## Part 8: Next Steps
- [Access Configuration](./services/config.md)
- [Output Logs](./services/log.md)
- [Add HTTP Routes](./services/http.md)
================================================
FILE: src/backend/doc/license_header.txt
================================================
Copyright (C) 2024 Puter Technologies Inc.
This file is part of Puter.
Puter is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see :`
- `` specifies the file that this permission
is associated with.
The ACL service
(which checks filesystem permissions)
knows if the value is a path or UUID based on the presence
of a leading slash; if it starts with `"/"` it's a path.
- `` specifies one of:
`write`, `read`, `list`, `see`; where each item in that
list implies all the access levels which follow.
- A permission that grants access to a directory,
such as `/user/shared`, implies access
of the same **access level** to all child file or directory
nodes under that location, **recursively**;
`fs:/user/shared:read` implies `fs:/user/shared/nested/file.txt:read`
- The "real" permission is `fs::`;
whenever path is specified the permission is rewritten.
**note:** future support for other filesystems
could make this rewrite rule conditional.
## App and Subdomain permissions
### `site::access`
- `` specifies the subdomain that this
permission is associated with.
Here, "subdomain" means the **"name of the subdomain"**,
which means a site accessed via `my-name.example.site`
will be specified here with `my-name`.
- This permission is always rewritten as the permission
described below (backend does this automatically).
### `site:uid#:access`
- If the subdomain is **not** [protected](../features/protected-apps.md),
this permission is ignored by the system.
- If the subdomain **is** protected, this permission will
allow access to the site via a Puter app iframe with
a token for the entity to which permission was granted
### `app::access`
- `` specifies the app that this
permission is associated with.
- This permission is always rewritten as the permission
described below (backend does this automatically).
### `app:uid#:access`
- If the app is **not** [protected](../features/protected-apps.md),
this permission is ignored by the system.
- If the app **is** protected, this permission will
allow reading the app's metadata and seeing that the app exists.
================================================
FILE: src/backend/doc/lists-of-things/list-of-tto-types.md
================================================
# Types for Type-Tagged Objects
## Internal Use
### `{ $: 'share-intent' }`
- Used in the `/share` endpoint
- Permissions get applied to existing users
- For email shares, is trasnformed into a `token:share`
which is stored in the `share` database table.
- **variants:**
- `share-intent:file`
- `share-intent:app`
- **properties:**
- `permissions` - a list of permissions to grant
### `{ $: 'internal:share' }`
- Stored in the `share` database table
- **properties:**
- `permissions` - a list of permissions to grant
### `{ $: 'token:share }`
- Stored in a JWT called the "share token"
- Contains only the share UUID
- **properties:**
- `uid` - UUID of a share
================================================
FILE: src/backend/doc/log_config.md
================================================
## Backend - Configuring Logs
### Log visibility specified by configuration file
The configuration file can define an array parameter called `logging`.
This configures the visibility of specific logs in core areas based on
which string flags are present.
For example, the following configuration enables HTTP request logs:
```json
{
"logging": ['http']
}
```
Sometimes "enabling" a log means moving its log level from `debug` to `info`.
#### Available logging flags:
- `http`: http requests
- `fsentries-not-found`: information about files that were stat'd but weren't there
#### Other log options
- Setting `log_upcoming_alarms` to `true` will log alarms before they are created.
This would be useful if AlarmService itself is failing.
- Setting `trace_logs` to `true` will display a stack trace below every log message.
This can be useful if you don't know where a particular log is coming from and
want to track it down.
#### Service-level log configuration
Services can be configured to change their logging behavior. Services will have one of
two behaviors:
1. **info logging** - `log.info` can be used to create an `[INFO]` log message
2. **debug logging only** - `log.info` is redirected to `log.debug`
Services will have **info logging** enabled by default, unless the class definition
has the static member `static LOG_DEBUG = true` (in which case **debug logging only**
is the default).
In a service's configuration block the desired behavior can be specified by setting
either `"log_debug": true` or `"log_info": true`
================================================
FILE: src/backend/doc/modules/filesystem/API_SPEC.md
================================================
# Filesystem API
Filesystem endpoints allow operations on files and directories in the Puter filesystem.
## POST `/mkdir` (auth required)
### Description
Creates a new directory in the filesystem. Currently support 2 formats:
- Full path: `{"path": "/foo/bar", args ...}` — this API is used by apitest (`./tools/api-tester/apitest.js`) and aligns more closely with the POSIX spec (https://linux.die.net/man/3/mkdir)
- Parent + path: `{"parent": "/foo", "path": "bar", args ...}` — this API is used by `puter-js` via `puter.fs.mkdir`
A future work would be use a unified format for all filesystem operations.
### Parameters
- **path** _- required_
- **accepts:** `string`
- **description:** The path where the directory should be created
- **notes:** Cannot be empty, null, or undefined
- **parent** _- optional_
- **accepts:** `string | UUID`
- **description:** The parent directory path or UUID
- **notes:** If not provided, path is treated as full path
- **overwrite** _- optional_
- **accepts:** `boolean`
- **default:** `false`
- **description:** Whether to overwrite existing files/directories
- **dedupe_name** _- optional_
- **accepts:** `boolean`
- **default:** `false`
- **description:** Whether to automatically rename if name exists
- **create_missing_parents** _- optional_
- **accepts:** `boolean`
- **default:** `false`
- **description:** Whether to create parent directories if they don't exist
- **aliases:** `create_missing_ancestors`
- **shortcut_to** _- optional_
- **accepts:** `string | UUID`
- **description:** Creates a shortcut/symlink to the specified target
### Example
```json
{
"path": "/user/Desktop/new-directory"
}
```
```json
{
"parent": "/user",
"path": "Desktop/new-directory"
}
```
### Response
Returns the created directory's metadata including name, path, uid, and any parent directories created.
## Other Filesystem Endpoints
[Additional endpoints would be documented here...]
================================================
FILE: src/backend/doc/modules/puterai/README.md
================================================
# PuterAI Module
The PuterAI module provides AI capabilities to Puter through various services including:
- Text generation and chat completion
- Text-to-speech synthesis
- Image generation
- Document analysis
## Metered Services
All AI services in this module are metered using Puter's MeteringService. This allows us to charge per `unit` usage, where a `unit` is defined by the specific service:
for example, most LLMs will charge per token, AWS Polly charges per character, and AWS Textract charges per page. the metering service tracks usage units, and relies on its centralized cost maps to determine if a user has enough credits to perform an operation, and to record usage after the operation is complete.
see [MeteringService](../../../src/services/MeteringService/MeteringService.ts) for more details on how metering works.
================================================
FILE: src/backend/doc/notes/2024-10-03_email_in_use_checks.md
================================================
## 2024-10-03
### Plan (constantly changing as per what's below)
- `signup.js` only says "email already used" if the one that's
already been used is confirmed.
- "change email" needs to follow the same logic; show an error when
an email already exists on an account with a confirmed email.
Then, upon confirming the update, Ensure that in the meanwhile no
new account came up with that email set.
- ensure `clean_email` is updated whenever the email is updated
### Email duplicate check on confirmation
- signup.js:149 -> this is where email dupe is currently checked
- signup.js:290 -> This is where we send the confirmation email.
There is also a branch that sends a "confirm token".
I don't recall what this is for.
### Investigating the "confirm token"
- email template is `email_verification_code`
instead of `email_verification_link`
- This happens when either:
- user.requires_email_confirmation is TRUE
- send_confirmation_code is TRUE in REQUEST
### Figuring out when `requires_email_confirmation` is TRUE
I'm mostly curious about this state on a user.
It's strange that `signup.js` would do anything on EXISTING users.
1. `pseudo_user` may be populated if `req.body.email` exists
AND a user with no password exists with that email
2. `uuid_user` may be populated if a user exists with the specified
UUID, but it has no usefulness unless `uuid_user` has the same
id as `pseudo_user`.
`uuid_user` is only used to set `email_confirmation_required` to 0
IFF `pseudo_user` has same id as `uuid_user`
AND `psuedo_user` has an email
When does `pseudo_user` have an email?
### Figuring out when a pseudo user can have an email
- asking NJ, I'm at a loss on this one for the moment
### Figuring out if account takeover is possible on signup.js with a uuid
- Nope, looks like `uuid_user` is only used to set
`email_confirmation_required = 0`
### Figuring out when `send_confirmation_code` is TRUE in REQUEST
- IFF `require_email_verification_to_publish_website` is TRUE
- it's not currently, but we need this to be possible to enable
- ^ That seems to be the ONLY place when this matters
### Current Thoughts
- `email_verification_code` will be difficult to test because there is
nothing currently in the system that's using it. However, I could try
enabling `require_email_verification_to_publish_website` locally and
see if this behavior begins to work as expected.
- `email_verification_link` where we can confirm an email. If another email
was already confirmed since the time the link was sent, we need to display
an error message to the user.
### Find places where (on backend) email change process is triggered
Right now there are two handlers:
- `/user-protected/change-email` (UserProtectedEndpointsService)
- Invokes the process (sends confirmation email)
- `/change_email/confirm` (PuterAPIService)
- Endpoint that the email link points to
================================================
FILE: src/backend/doc/services/config.md
================================================
# Service Configuration
To locate your configuration file, see [Configuring Puter](https://github.com/HeyPuter/puter/wiki/self_hosters-config).
### Accessing Service Configuration
Service configuration appears under the `"services"` property in the
configuration file for Puter. If Puter's configuration had no other
values except for a service config with one key, it might look like
this:
```json
{
"services": {
"my-service": {
"somekey": "some value"
}
}
}
```
Services have their configuration object assigned to `this.config`.
```javascript
class MyService extends BaseService {
async _init () {
// You can access configuration for a service like this
this.log.info('value of my key is: ' + this.config.somekey);
}
}
```
### Accessing Global Configuration
Services can access global configuration. This can be useful for knowing how
Puter itself is configured, but using this global config object for service
configuration is discouraged as it could create conflicts between services.
```javascript
class MyService extends BaseService {
async _init () {
// You can access configuration for a service like this
this.log.info('Puter is hosted on: ' + this.global_config.domain);
}
}
```
================================================
FILE: src/backend/doc/services/event_buses.md
================================================
# Event Buses
Puter's backend has two event buses:
- Service Event Bus
- Application Event Bus
## Service Event Bus
This is a simple event bus that lives in the [Container](../../src/services/Container.js)
class. There is only one instance of **Container** and it is called the "services container".
When Puter boots, all the services registered by modules are registered into the services
container.
Services handle events from the Service Event Bus by implementing methods which are named
with the prefix `__on_`. This prefix looks a little strange at first so it's worth
breaking it down:
- `__` (two underscores) prevents collision with common method names, and also
common conventions like beginning a method name with a single underscore
to indicate a method that should be overridden.
- `on` is the meaningful name.
- `_`, the last underscore, is for readability, as the event name conventionally
begins with a lowercase letter.
Note that you will need to use the
Example:
```javascript
class MyService extends BaseService {
['__on_boot.ready'] () {
//
}
}
```
================================================
FILE: src/backend/doc/services/http.md
================================================
# Adding HTTP Routes to Services
Services can serve HTTP routes when the [WebModule](../../src/modules/web/WebModule.js)
is enabled by listening for the `install.routes` event on the [Service Event Bus](./)
================================================
FILE: src/backend/doc/services/log.md
================================================
# Logging in Services
# NOTE: You can, and maybe should, just use console log methods, as they are overriden to log through our logger
Services all have a logger available at `this.log`.
```javascript
class MyService extends BaseService {
async init () {
this.log.info('Hello, Logger!');
}
}
```
There are multiple "log levels", similar to `logrus` or other common logging
libraries.
```javascript
class MyService extends BaseService {
async init () {
this.log.info('I\'m just a regular log.');
this.log.debug('I\'m only for developers.');
this.log.warn('It is statistically unlikely I will be awknowledged.');
this.log.error('Something is broken! Pay attention!');
this.log.noticeme('This will be noticed, unlike warnings. Use sparingly.');
this.log.system('I am a system event, like shutdown.');
this.log.tick('A periodic behavior like cache pruning is occurring.');
}
}
```
Log methods can take a second parameter, an object specifying fields.
```javascript
class MyService extends BaseService {
async init () {
this.log.info('I have fields!', {
why: "why not",
random_number: 1, // chosen by coin toss, guarenteed to be random
});
}
}
```
================================================
FILE: src/backend/exports.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } Resolves when legacy services are installed
*/
async install_legacy (context) {
const services = context.get('services');
await install_legacy({ services });
}
}
module.exports = CoreModule;
================================================
FILE: src/backend/src/DatabaseModule.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see >();
const getDynalitePathKey = (path?: string) => {
if ( path === ':memory:' ) return LOCAL_DYNAMO_PATH_KEY;
return path || './puter-ddb';
};
const getOrCreateLocalDynaliteEndpoint = async (pathKey: string) => {
let endpointPromise = localDynaliteEndpointPromises.get(pathKey);
if ( endpointPromise ) return endpointPromise;
endpointPromise = (async () => {
const dynaliteOptions = pathKey === LOCAL_DYNAMO_PATH_KEY
? { createTableMs: 0 }
: { createTableMs: 0, path: pathKey };
const dynaliteInstance = dynalite(dynaliteOptions);
const dynaliteServer = dynaliteInstance.listen(0, '127.0.0.1');
// Don't keep test workers alive just because dynalite is still open.
dynaliteServer.unref?.();
await once(dynaliteServer, 'listening');
const address = dynaliteServer.address();
const port = (typeof address === 'object' && address ? address.port : undefined) || 4567;
return `http://127.0.0.1:${port}`;
})();
localDynaliteEndpointPromises.set(pathKey, endpointPromise);
endpointPromise.catch(() => {
if ( localDynaliteEndpointPromises.get(pathKey) === endpointPromise ) {
localDynaliteEndpointPromises.delete(pathKey);
}
});
return endpointPromise;
};
export class DDBClient {
ddbClientPromise: Promise;
#documentClient!: DynamoDBDocumentClient;
config?: DBClientConfig;
constructor (config?: DBClientConfig) {
this.config = config;
this.ddbClientPromise = this.#getClient();
this.ddbClientPromise.then(client => {
this.#documentClient = DynamoDBDocumentClient.from(client, {
marshallOptions: {
removeUndefinedValues: true,
} });
});
}
async recreateClient () {
this.ddbClientPromise = this.#getClient();
this.#documentClient = DynamoDBDocumentClient.from(await this.ddbClientPromise, {
marshallOptions: {
removeUndefinedValues: true,
} });
}
async #getClient () {
if ( ! this.config?.aws ) {
console.warn('No config for DynamoDB, will fall back on local dynalite');
const pathKey = getDynalitePathKey(this.config?.path);
const dynamoEndpoint = await getOrCreateLocalDynaliteEndpoint(pathKey);
const client = new DynamoDBClient({
credentials: {
accessKeyId: 'fake',
secretAccessKey: 'fake',
},
maxAttempts: 3,
requestHandler: new NodeHttpHandler({
connectionTimeout: 5000,
requestTimeout: 5000,
httpsAgent: new httpsAgent({ keepAlive: true }),
}),
endpoint: dynamoEndpoint,
region: 'us-west-2',
});
console.log(`Dynalite client created within instance for region: ${await client.config.region()}`);
return client;
}
const client = new DynamoDBClient({
credentials: {
accessKeyId: this.config.aws.access_key,
secretAccessKey: this.config.aws.secret_key,
},
maxAttempts: 3,
requestHandler: new NodeHttpHandler({
connectionTimeout: 5000,
requestTimeout: 5000,
httpsAgent: new httpsAgent({ keepAlive: true }),
}),
...(this.config.endpoint ? { endpoint: this.config.endpoint } : {}),
region: this.config.aws.region || 'us-west-2',
});
console.log(`DynamoDB client created with region ${await client.config.region()}`);
return client;
}
async get >(table: string, key: T, consistentRead = false) {
const command = new GetCommand({
TableName: table,
Key: key,
ConsistentRead: consistentRead,
ReturnConsumedCapacity: 'TOTAL',
});
const response = await this.#documentClient.send(command);
return response;
}
async put >(table: string, item: T) {
const command = new PutCommand({
TableName: table,
Item: item,
ReturnConsumedCapacity: 'TOTAL',
});
const response = await this.#documentClient.send(command);
return response;
}
async batchGet (params: { table: string, items: Record }[], consistentRead = false) {
// TODO DS: implement chunking for more than 100 items or more than allowed req size
const allRequestItemsPerTable = params.reduce((acc, curr) => {
if ( ! acc[curr.table] ) acc[curr.table] = [];
acc[curr.table].push(curr.items);
return acc;
}, {} as Record[]>);
const RequestItems: BatchGetCommandInput['RequestItems'] = Object.entries(allRequestItemsPerTable).reduce(
(acc, [table, keyList]) => {
const Keys = keyList;
acc[table] = {
Keys,
ConsistentRead: consistentRead,
};
return acc;
},
{} as NonNullable,
);
const command = new BatchGetCommand({
RequestItems,
ReturnConsumedCapacity: 'TOTAL',
});
return this.#documentClient.send(command);
}
async del> (table: string, key: T) {
const command = new DeleteCommand({
TableName: table,
Key: key,
ReturnConsumedCapacity: 'TOTAL',
});
return this.#documentClient.send(command);
}
async query> (
table: string,
keys: T,
limit = 0,
pageKey?: Record,
index = '',
consistentRead = false,
options?: { beginsWith?: { key: string; value: string } },
) {
const keyExpressionParts = Object.keys(keys).map(key => `#${key} = :${key}`);
const expressionAttributeValues = Object.entries(keys).reduce((acc, [key, value]) => {
acc[`:${key}`] = value;
return acc;
}, {});
const expressionAttributeNames = Object.keys(keys).reduce((acc, key) => {
acc[`#${key}`] = key;
return acc;
}, {});
if ( options?.beginsWith?.key && typeof options.beginsWith.value === 'string' && options.beginsWith.value !== '' ) {
const beginsKey = options.beginsWith.key;
const beginsValueToken = `:${beginsKey}_begins_with`;
keyExpressionParts.push(`begins_with(#${beginsKey}, ${beginsValueToken})`);
expressionAttributeValues[beginsValueToken] = options.beginsWith.value;
expressionAttributeNames[`#${beginsKey}`] = beginsKey;
}
const keyExpression = keyExpressionParts.join(' AND ');
const command = new QueryCommand({
TableName: table,
...(!index ? {} : { IndexName: index }),
KeyConditionExpression: keyExpression,
ExpressionAttributeValues: expressionAttributeValues,
ExpressionAttributeNames: expressionAttributeNames,
ConsistentRead: consistentRead,
...(!pageKey ? {} : { ExclusiveStartKey: pageKey }),
...(!limit ? {} : { Limit: limit }),
ReturnConsumedCapacity: 'TOTAL',
});
return await this.#documentClient.send(command);
}
async update> (
table: string,
key: T,
expression: string,
expressionValues?: Record,
expressionNames?: Record,
) {
const hasValues = !!expressionValues && Object.keys(expressionValues).length > 0;
const hasNames = !!expressionNames && Object.keys(expressionNames).length > 0;
const command = new UpdateCommand({
TableName: table,
Key: key,
UpdateExpression: expression,
...(hasValues ? { ExpressionAttributeValues: expressionValues } : {}),
...(hasNames ? { ExpressionAttributeNames: expressionNames } : {}),
ReturnValues: 'ALL_NEW',
ReturnConsumedCapacity: 'TOTAL',
});
try {
return await this.#documentClient.send(command);
} catch ( e ) {
console.error('DDB Update Error', e);
throw e;
}
}
async createTableIfNotExists (params: CreateTableCommandInput, ttlAttribute?: string) {
if ( this.config?.aws ) {
console.warn('Creating DynamoDB tables in AWS is disabled by default, but if you need to enable it, modify the DDBClient class');
return;
}
try {
await this.#documentClient.send(new CreateTableCommand(params));
} catch ( e ) {
if ( (e as Error)?.name !== 'ResourceInUseException' ) {
throw e;
}
setTimeout(async () => {
if ( ttlAttribute ) {
// ensure TTL is set
await this.#documentClient.send(new UpdateTimeToLiveCommand({
TableName: params.TableName!,
TimeToLiveSpecification: {
AttributeName: ttlAttribute,
Enabled: true,
},
}));
}
}, 5000); // wait 5 seconds to ensure table is active
}
}
}
================================================
FILE: src/backend/src/clients/dynamodb/DDBClientWrapper.ts
================================================
import { BaseService } from '@heyputer/backend/src/services/BaseService.js';
import { DDBClient } from './DDBClient.js';
/** Wrapping actual implementation to be usable through our core structure */
class DDBClientServiceWrapper extends BaseService {
ddbClient!: DDBClient;
async _construct () {
this.ddbClient = new DDBClient(this.config as unknown as ConstructorParameters[0]);
await this.ddbClient.ddbClientPromise; // ensure client is ready
Object.getOwnPropertyNames(DDBClient.prototype).forEach(fn => {
if ( fn === 'constructor' ) return;
this[fn] = (...args: unknown[]) => this.ddbClient[fn](...args);
});
}
}
export const DDBClientWrapper = DDBClientServiceWrapper as unknown as DDBClient;
================================================
FILE: src/backend/src/clients/redis/.gitignore
================================================
*.js
*.js.map
================================================
FILE: src/backend/src/clients/redis/cacheUpdate.ts
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see => {
const flattened: Array = [];
for ( const input of inputs ) {
if ( Array.isArray(input) ) {
flattened.push(...flattenCacheKeys(input));
continue;
}
flattened.push(input);
}
return flattened;
};
export const normalizeCacheKeys = (cacheKey: CacheKeyInput | CacheKeyInput[]): string[] => {
const arr = Array.isArray(cacheKey) ? cacheKey : [cacheKey];
return [...new Set(flattenCacheKeys(arr)
.map(key => key === null || key === undefined ? '' : String(key))
.filter(Boolean))];
};
const getEventService = (eventService?: CacheUpdateOptions['eventService']) => {
if ( eventService?.emit ) return eventService;
const contextServices = Context.get('services', { allow_fallback: true });
if ( contextServices?.get ) {
try {
return contextServices.get('event');
} catch (e) {
// no-op
}
}
const globalServices = (globalThis)[SERVICES_KEY]?.services as typeof contextServices;
if ( globalServices?.get ) {
try {
return globalServices.get('event');
} catch (e) {
// no-op
}
}
return null;
};
export const emitOuterCacheUpdate = (
{
cacheKey,
data,
ttlSeconds,
}: {
cacheKey: CacheKeyInput | CacheKeyInput[],
data?: unknown,
ttlSeconds?: number,
},
{
eventService,
emitEvent = true,
}: CacheUpdateOptions = {},
) => {
if ( ! emitEvent ) return;
const keys = normalizeCacheKeys(cacheKey);
if ( ! keys.length ) return;
const svc_event = getEventService(eventService);
if ( ! svc_event ) return;
const payload: Record = { cacheKey: keys };
if ( data !== undefined ) payload.data = data;
if ( ttlSeconds !== undefined && ttlSeconds !== null ) {
payload.ttlSeconds = ttlSeconds;
}
svc_event.emit('outer.cacheUpdate', payload);
};
export const setRedisCacheValue = async (
key: string,
value: string | number,
{
ttlSeconds,
}: {
ttlSeconds?: number,
eventData?: unknown,
eventService?: CacheUpdateOptions['eventService'],
emitEvent?: boolean,
} = {},
) => {
if ( ttlSeconds ) {
await redisClient.set(key, value, 'EX', ttlSeconds);
} else {
await redisClient.set(key, value);
}
};
================================================
FILE: src/backend/src/clients/redis/deleteRedisKeys.ts
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see => {
const flattened: Array = [];
for ( const input of inputs ) {
if ( Array.isArray(input) ) {
flattened.push(...flattenInputs(input));
continue;
}
flattened.push(input);
}
return flattened;
};
export const deleteRedisKeys = async (...inputs: (DeleteRedisKeysInput | DeleteRedisKeysOptions)[]) => {
const keysInput = [...inputs];
if ( isDeleteOptions(keysInput[keysInput.length - 1]) ) {
keysInput.pop() as DeleteRedisKeysOptions;
}
const keys = flattenInputs(keysInput as DeleteRedisKeysInput[])
.map(key => key === null || key === undefined ? '' : String(key))
.filter(Boolean);
if ( keys.length === 0 ) {
return 0;
}
const uniqueKeys = [...new Set(keys)];
const deleteResults = await Promise.allSettled(uniqueKeys.map(key => redisClient.del(key)));
const deleted = deleteResults.reduce((sum, promiseCount) => sum + (promiseCount.status === 'fulfilled' ? promiseCount.value : 0), 0);
return deleted;
};
================================================
FILE: src/backend/src/clients/redis/redisSingleton.test.ts
================================================
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const redisMocks = vi.hoisted(() => {
const redisClusterInstances: Array<{
on: ReturnType;
once: ReturnType;
}> = [];
return {
redisClusterInstances,
redisClusterConstructorMock: vi.fn(),
mockRedisClusterConstructorMock: vi.fn(),
};
});
vi.mock('ioredis', () => {
class RedisClusterMock {
on = vi.fn().mockReturnThis();
once = vi.fn().mockReturnThis();
constructor (...args: unknown[]) {
redisMocks.redisClusterConstructorMock(...args);
redisMocks.redisClusterInstances.push(this);
}
}
return {
default: {
Cluster: RedisClusterMock,
},
};
});
vi.mock('ioredis-mock', () => {
class MockRedisClusterMock {
constructor (...args: unknown[]) {
redisMocks.mockRedisClusterConstructorMock(...args);
}
}
return {
default: {
Cluster: MockRedisClusterMock,
},
};
});
describe('redisSingleton', () => {
const initialRedisConfig = process.env.REDIS_CONFIG;
beforeEach(() => {
vi.resetModules();
redisMocks.redisClusterInstances.length = 0;
redisMocks.redisClusterConstructorMock.mockReset();
redisMocks.mockRedisClusterConstructorMock.mockReset();
process.env.REDIS_CONFIG = JSON.stringify([{ host: '127.0.0.1', port: 6379 }]);
vi.spyOn(console, 'log').mockImplementation(() => undefined);
vi.spyOn(console, 'warn').mockImplementation(() => undefined);
vi.spyOn(console, 'error').mockImplementation(() => undefined);
});
afterEach(() => {
if ( initialRedisConfig === undefined ) {
delete process.env.REDIS_CONFIG;
} else {
process.env.REDIS_CONFIG = initialRedisConfig;
}
vi.restoreAllMocks();
});
it('uses resilient cluster options and registers startup-safe listeners', async () => {
const singletonModule = await import('./redisSingleton.ts');
expect(redisMocks.redisClusterConstructorMock).toHaveBeenCalledTimes(1);
const [startupNodes, clusterOptions] = redisMocks.redisClusterConstructorMock.mock.calls[0];
expect(startupNodes).toEqual([{ host: '127.0.0.1', port: 6379 }]);
expect(clusterOptions).toEqual(expect.objectContaining({
enableOfflineQueue: true,
retryDelayOnFailover: 500,
retryDelayOnClusterDown: 1000,
retryDelayOnTryAgain: 300,
slotsRefreshTimeout: 5000,
clusterRetryStrategy: expect.any(Function),
dnsLookup: expect.any(Function),
redisOptions: expect.objectContaining({
connectTimeout: 10000,
maxRetriesPerRequest: null,
tls: {},
}),
}));
expect(clusterOptions.clusterRetryStrategy(1)).toBe(200);
expect(clusterOptions.clusterRetryStrategy(100)).toBe(2000);
const clusterInstance = redisMocks.redisClusterInstances[0];
expect(singletonModule.redisClient).toBe(clusterInstance);
expect(clusterInstance.once).toHaveBeenCalledWith('connect', expect.any(Function));
expect(clusterInstance.once).toHaveBeenCalledWith('ready', expect.any(Function));
expect(clusterInstance.on).toHaveBeenCalledWith('error', expect.any(Function));
expect(clusterInstance.on).toHaveBeenCalledWith('node error', expect.any(Function));
});
});
================================================
FILE: src/backend/src/clients/redis/redisSingleton.ts
================================================
import Redis, { Cluster } from 'ioredis';
import MockRedis from 'ioredis-mock';
const redisStartupRetryMaxDelayMs = 2000;
const redisSlotsRefreshTimeoutMs = 5000;
const redisConnectTimeoutMs = 10000;
const redisBootRetryRegex = /Cluster(All)?FailedError|None of startup nodes is available/i;
const formatRedisError = (error: unknown): string => {
if ( error instanceof Error ) {
return `${error.name}: ${error.message}`;
}
return String(error);
};
const attachClusterEventHandlers = (clusterClient: Cluster): void => {
clusterClient.once('connect', () => {
console.log('[redis] cluster transport connected');
});
clusterClient.once('ready', () => {
console.log('[redis] cluster ready');
});
clusterClient.on('error', (error: unknown) => {
const errorText = formatRedisError(error);
if ( redisBootRetryRegex.test(errorText) ) {
console.warn(`[redis] startup issue while connecting to cluster; retrying automatically (${errorText})`);
return;
}
console.error('[redis] cluster error', error);
});
clusterClient.on('node error', (error: unknown, nodeKey: string) => {
const errorText = formatRedisError(error);
if ( redisBootRetryRegex.test(errorText) ) {
console.warn(`[redis] startup issue for cluster node ${nodeKey}; retrying automatically (${errorText})`);
return;
}
console.error(`[redis] cluster node error (${nodeKey})`, error);
});
};
let redisOpt: Cluster;
if ( process.env.REDIS_CONFIG ) {
const redisConfig = JSON.parse(process.env.REDIS_CONFIG);
redisOpt = new Redis.Cluster(redisConfig, {
dnsLookup: (address, callback) => callback(null, address),
clusterRetryStrategy: (attempts) => Math.min(100 + (attempts * 100), redisStartupRetryMaxDelayMs),
retryDelayOnFailover: 500,
retryDelayOnClusterDown: 1000,
retryDelayOnTryAgain: 300,
slotsRefreshTimeout: redisSlotsRefreshTimeoutMs,
enableOfflineQueue: true,
redisOptions: {
tls: {},
connectTimeout: redisConnectTimeoutMs,
maxRetriesPerRequest: null,
},
});
attachClusterEventHandlers(redisOpt);
console.log('connecting to redis from config');
} else {
redisOpt = new MockRedis.Cluster(['redis://localhost:7001']);
console.log('connected to local redis mock');
}
export const redisClient = redisOpt;
================================================
FILE: src/backend/src/codex/CodeUtil.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } SequenceCallable
* A callable function returned by the Sequence constructor.
* @param {Object|Sequence.SequenceState} [opt_values] - Initial values for the sequence scope, or a SequenceState.
* @returns {Promise} The return value of the last step in the sequence.
*/
/**
* Sequence is a callable object that executes a series of functions in order.
* The functions are expected to be asynchronous; if they're not it might still
* work, but it's neither tested nor supported.
*
* Note: arrow functions are supported, but they are not recommended;
* using keyword functions allows each step to be named.
*
* Example usage:
*
* const seq = new Sequence([
* async function set_foo (a) {
* a.set('foo', 'bar')
* },
* async function print_foo (a) {
* console.log(a.get('foo'));
* },
* async function third_step (a) {
* // do something
* },
* ]);
*
* await seq();
*
* Example with controlled conditional branches:
*
* const seq = new Sequence([
* async function first_step (a) {
* // do something
* },
* {
* condition: async a => a.get('foo') === 'bar',
* fn: async function second_step (a) {
* // do something
* }
* },
* async function third_step (a) {
* // do something
* },
* ]);
*
* If it is called with an argument, it must be an object containing values
* which will populate the "sequence scope".
*
* If it is called on an instance with a member called `values`
* (i.e. if `this.values` is defined), then these values will populate the
* sequence scope. This is to maintain compatibility for Sequence to be used
* as an implementation of a runnable class. (See CodeUtil.mrwrap or BaseOperation)
*
* The object returned by the constructor is a function, which is used to
* make the object callable. The callable object will execute the sequence
* when called. The return value of the sequence is the return value of the
* last function in the sequence.
*
* Each function in the sequence is passed a SequenceState object
* as its first argument. Conventionally, this argument is called `a`,
* which is short for either "API", "access", or "the `a` variable"
* depending on which you prefer. Sequence provides methods for accessing
* the sequence scope.
*
* By accessing the sequence scope through the `a` variable, changes to the
* sequence scope can be monitored and recorded. (TODO: implement observe methods)
*/
/**
* Sequence is a callable object that executes a series of asynchronous functions in order.
* Each function receives a SequenceState instance for accessing and mutating the sequence scope.
* Supports conditional steps, deferred steps, and can be used as a runnable implementation for classes.
* @class @extends Function
*/
class Sequence {
/**
* SequenceState represents the state of a Sequence execution.
* Provides access to the sequence scope, step control, and utility methods for step functions.
*/
static SequenceState = class SequenceState {
/**
* Create a new SequenceState.
* @param {Sequence|function} sequence - The Sequence instance or its callable function.
* @param {Object} [thisArg] - The instance to bind as `this` for step functions.
*/
constructor (sequence, thisArg) {
if ( typeof sequence === 'function' ) {
sequence = sequence.sequence;
}
this.sequence_ = sequence;
this.thisArg = thisArg;
this.steps_ = null;
this.value_history_ = [];
this.scope_ = {};
this.last_return_ = undefined;
this.i = 0;
this.stopped_ = false;
this.defer_ptr_ = undefined;
this.defer = this.constructor.defer_0;
}
/**
* Get the current steps array for this sequence execution.
* @returns {Array} The steps to execute.
*/
get steps () {
return this.steps_ ?? this.sequence_?.steps_;
}
/**
* Run the sequence from the current step index.
* @param {Object} [values] - Initial values for the sequence scope.
* @returns {Promise}
*/
async run (values) {
// Initialize scope
values = values || this.thisArg?.values || {};
Object.setPrototypeOf(this.scope_, values);
// Run sequence
for ( ; this.i < this.steps.length ; this.i++ ) {
let step = this.steps[this.i];
if ( typeof step !== 'object' ) {
step = {
name: step.name,
fn: step,
};
}
if ( step.condition && !await step.condition(this) ) {
continue;
}
const parent_scope = this.scope_;
this.scope_ = {};
// We could do Object.assign(this.scope_, parent_scope), but
// setting the prototype should be faster (in theory)
Object.setPrototypeOf(this.scope_, parent_scope);
if ( this.sequence_.options_.record_history ) {
this.value_history_.push(this.scope_);
}
if ( this.sequence_.options_.before_each ) {
await this.sequence_.options_.before_each(this, step);
}
this.last_return_ = await step.fn.call(this.thisArg, this);
if ( this.last_return_ instanceof Sequence.SequenceState ) {
this.scope_ = this.last_return_.scope_;
}
if ( this.sequence_.options_.after_each ) {
await this.sequence_.options_.after_each(this, step);
}
if ( this.stopped_ ) {
break;
}
}
}
// Why check a condition every time code is called,
// when we can check it once and then replace the code?
/**
* The first time defer is called, clones the steps and sets up for deferred insertion.
* @param {function(Sequence.SequenceState): Promise} fn - The function to defer.
*/
static defer_0 = function (fn) {
this.steps_ = [...this.sequence_.steps_];
this.defer = this.constructor.defer_1;
this.defer_ptr_ = this.steps_.length;
this.defer(fn);
};
/**
* Subsequent calls to defer insert the function before the deferred pointer.
* @param {function(Sequence.SequenceState): Promise} fn - The function to defer.
*/
static defer_1 = function (fn) {
// Deferred functions don't affect the return value
const real_fn = fn;
fn = async () => {
await real_fn(this);
return this.last_return_;
};
// Insert deferred step before the pointer
this.steps_.splice(this.defer_ptr_, 0, fn);
};
/**
* Get a value from the sequence scope.
* @param {string} k - The key to retrieve.
* @returns {any} The value associated with the key.
*/
get (k) {
// TODO: record read1
return this.scope_[k];
}
/**
* Set a value in the sequence scope.
* @param {string} k - The key to set.
* @param {any} v - The value to assign.
*/
set (k, v) {
// TODO: record mutation
this.scope_[k] = v;
}
/**
* Get or set multiple values in the sequence scope.
* @param {Object} [opt_itemsToSet] - Optional object of key-value pairs to set.
* @returns {Object} Proxy to the current scope for value access.
*/
values (opt_itemsToSet) {
if ( opt_itemsToSet ) {
for ( const k in opt_itemsToSet ) {
this.set(k, opt_itemsToSet[k]);
}
}
return new Proxy(this.scope_, {
get: (target, property) => {
if ( property in target ) {
// TODO: record read
return target[property];
}
return undefined;
},
});
}
/**
* Get a value from the instance (`thisArg`).
* @param {string} [k] - The property name to retrieve. If omitted, returns the instance.
* @returns {any} The value from the instance or the instance itself.
*/
iget (k) {
if ( k === undefined ) return this.thisArg;
return this.thisArg?.[k];
}
// Instance call: call a method on the instance
/**
* Call a method on the instance (`thisArg`).
* @param {string} k - The method name.
* @param {...any} args - Arguments to pass to the method.
* @returns {any} The result of the method call.
*/
icall (k, ...args) {
return this.thisArg?.[k]?.call(this.thisArg, ...args);
}
// Instance dynamic call: call a method on the instance,
// passing the sequence state as the first argument
/**
* Call a method on the instance, passing the sequence state as the first argument.
* @param {string} k - The method name.
* @param {...any} args - Arguments to pass after the sequence state.
* @returns {any} The result of the method call.
*/
idcall (k, ...args) {
return this.thisArg?.[k]?.call(this.thisArg, this, ...args);
}
/**
* Get the logger from the instance, if available.
* @returns {Object|undefined} The logger object.
*/
get log () {
return this.iget('log');
}
/**
* Stop the sequence early and optionally return a value.
* @param {any} [return_value] - Value to return from the sequence.
* @returns {any} The provided return value.
*/
stop (return_value) {
this.stopped_ = true;
return return_value;
}
};
/**
*
* @param {Array | {condition: (a: A) => boolean | Promise, fn: function(A): Promise}> | function(A): Promise | Object} args
* @returns {Sequence}
*/
/**
* Create a new Sequence.
* @param {...(Array|Object>|function(Sequence.SequenceState): Promise|Object)} args
* - Arrays of step functions or step objects, individual step functions, or options objects.
* - Step objects may have a `condition` property (function) and a `fn` property (function).
* - Options object may include `name`, `record_history`, `before_each`, `after_each`.
* @returns {SequenceCallable} A callable function that runs the sequence.
*/
constructor (...args) {
const sequence = this;
const steps = [];
const options = {};
for ( const arg of args ) {
if ( Array.isArray(arg) ) {
steps.push(...arg);
} else if ( typeof arg === 'object' ) {
Object.assign(options, arg);
} else if ( typeof arg === 'function' ) {
steps.push(arg);
} else {
throw new TypeError(`Invalid argument to Sequence constructor: ${arg}`);
}
}
/**
* Callable function to execute the sequence.
* @param {Object|Sequence.SequenceState} [opt_values] - Initial values or a SequenceState.
* @returns {Promise} The return value of the last step.
*/
const fn = async function (opt_values) {
if ( opt_values && opt_values instanceof Sequence.SequenceState ) {
opt_values = opt_values.scope_;
}
// eslint-disable-next-line no-invalid-this
const state = new Sequence.SequenceState(sequence, this); // TODO: fix this odd structure, what is this even bound to ?
await state.run(opt_values ?? undefined);
return state.last_return_;
};
this.steps_ = steps;
this.options_ = options || {};
Object.defineProperty(fn, 'name', {
value: options.name || 'Sequence',
});
Object.defineProperty(fn, 'sequence', { value: this });
return fn;
}
}
module.exports = {
Sequence,
};
================================================
FILE: src/backend/src/config/ConfigLoader.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see ;
export interface IConfig extends ConfigRecord {
load_config: (o: ConfigRecord) => void;
__set_config_object__: (
object: ConfigRecord,
options?: { replacePrototype?: boolean; useInitialPrototype?: boolean }
) => void;
}
declare const config: IConfig;
export = config;
================================================
FILE: src/backend/src/config.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see = {
encode(message: FSEntry, writer: BinaryWriter = new BinaryWriter()): BinaryWriter {
if (message.uuid !== "") {
writer.uint32(10).string(message.uuid);
}
if (message.uid !== "") {
writer.uint32(18).string(message.uid);
}
if (message.name !== "") {
writer.uint32(26).string(message.name);
}
if (message.path !== "") {
writer.uint32(34).string(message.path);
}
if (message.parent_uuid !== "") {
writer.uint32(42).string(message.parent_uuid);
}
if (message.parent_uid !== "") {
writer.uint32(50).string(message.parent_uid);
}
if (message.parent_id !== "") {
writer.uint32(58).string(message.parent_id);
}
if (message.is_dir !== false) {
writer.uint32(64).bool(message.is_dir);
}
if (message.created !== 0) {
writer.uint32(72).int64(message.created);
}
if (message.modified !== 0) {
writer.uint32(80).int64(message.modified);
}
if (message.accessed !== 0) {
writer.uint32(88).int64(message.accessed);
}
if (message.size !== 0) {
writer.uint32(96).int64(message.size);
}
return writer;
},
decode(input: BinaryReader | Uint8Array, length?: number): FSEntry {
const reader = input instanceof BinaryReader ? input : new BinaryReader(input);
const end = length === undefined ? reader.len : reader.pos + length;
const message = createBaseFSEntry();
while (reader.pos < end) {
const tag = reader.uint32();
switch (tag >>> 3) {
case 1: {
if (tag !== 10) {
break;
}
message.uuid = reader.string();
continue;
}
case 2: {
if (tag !== 18) {
break;
}
message.uid = reader.string();
continue;
}
case 3: {
if (tag !== 26) {
break;
}
message.name = reader.string();
continue;
}
case 4: {
if (tag !== 34) {
break;
}
message.path = reader.string();
continue;
}
case 5: {
if (tag !== 42) {
break;
}
message.parent_uuid = reader.string();
continue;
}
case 6: {
if (tag !== 50) {
break;
}
message.parent_uid = reader.string();
continue;
}
case 7: {
if (tag !== 58) {
break;
}
message.parent_id = reader.string();
continue;
}
case 8: {
if (tag !== 64) {
break;
}
message.is_dir = reader.bool();
continue;
}
case 9: {
if (tag !== 72) {
break;
}
message.created = longToNumber(reader.int64());
continue;
}
case 10: {
if (tag !== 80) {
break;
}
message.modified = longToNumber(reader.int64());
continue;
}
case 11: {
if (tag !== 88) {
break;
}
message.accessed = longToNumber(reader.int64());
continue;
}
case 12: {
if (tag !== 96) {
break;
}
message.size = longToNumber(reader.int64());
continue;
}
}
if ((tag & 7) === 4 || tag === 0) {
break;
}
reader.skip(tag & 7);
}
return message;
},
fromJSON(object: any): FSEntry {
return {
uuid: isSet(object.uuid) ? globalThis.String(object.uuid) : "",
uid: isSet(object.uid) ? globalThis.String(object.uid) : "",
name: isSet(object.name) ? globalThis.String(object.name) : "",
path: isSet(object.path) ? globalThis.String(object.path) : "",
parent_uuid: isSet(object.parent_uuid) ? globalThis.String(object.parent_uuid) : "",
parent_uid: isSet(object.parent_uid) ? globalThis.String(object.parent_uid) : "",
parent_id: isSet(object.parent_id) ? globalThis.String(object.parent_id) : "",
is_dir: isSet(object.is_dir) ? globalThis.Boolean(object.is_dir) : false,
created: isSet(object.created) ? globalThis.Number(object.created) : 0,
modified: isSet(object.modified) ? globalThis.Number(object.modified) : 0,
accessed: isSet(object.accessed) ? globalThis.Number(object.accessed) : 0,
size: isSet(object.size) ? globalThis.Number(object.size) : 0,
};
},
toJSON(message: FSEntry): unknown {
const obj: any = {};
if (message.uuid !== "") {
obj.uuid = message.uuid;
}
if (message.uid !== "") {
obj.uid = message.uid;
}
if (message.name !== "") {
obj.name = message.name;
}
if (message.path !== "") {
obj.path = message.path;
}
if (message.parent_uuid !== "") {
obj.parent_uuid = message.parent_uuid;
}
if (message.parent_uid !== "") {
obj.parent_uid = message.parent_uid;
}
if (message.parent_id !== "") {
obj.parent_id = message.parent_id;
}
if (message.is_dir !== false) {
obj.is_dir = message.is_dir;
}
if (message.created !== 0) {
obj.created = Math.round(message.created);
}
if (message.modified !== 0) {
obj.modified = Math.round(message.modified);
}
if (message.accessed !== 0) {
obj.accessed = Math.round(message.accessed);
}
if (message.size !== 0) {
obj.size = Math.round(message.size);
}
return obj;
},
create(base?: DeepPartial): FSEntry {
return FSEntry.fromPartial(base ?? {});
},
fromPartial(object: DeepPartial): FSEntry {
const message = createBaseFSEntry();
message.uuid = object.uuid ?? "";
message.uid = object.uid ?? "";
message.name = object.name ?? "";
message.path = object.path ?? "";
message.parent_uuid = object.parent_uuid ?? "";
message.parent_uid = object.parent_uid ?? "";
message.parent_id = object.parent_id ?? "";
message.is_dir = object.is_dir ?? false;
message.created = object.created ?? 0;
message.modified = object.modified ?? 0;
message.accessed = object.accessed ?? 0;
message.size = object.size ?? 0;
return message;
},
};
type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
export type DeepPartial = T extends Builtin ? T
: T extends globalThis.Array ? globalThis.Array>
: T extends ReadonlyArray ? ReadonlyArray>
: T extends {} ? { [K in keyof T]?: DeepPartial }
: Partial;
function longToNumber(int64: { toString(): string }): number {
const num = globalThis.Number(int64.toString());
if (num > globalThis.Number.MAX_SAFE_INTEGER) {
throw new globalThis.Error("Value is larger than Number.MAX_SAFE_INTEGER");
}
if (num < globalThis.Number.MIN_SAFE_INTEGER) {
throw new globalThis.Error("Value is smaller than Number.MIN_SAFE_INTEGER");
}
return num;
}
function isSet(value: any): boolean {
return value !== null && value !== undefined;
}
export interface MessageFns {
encode(message: T, writer?: BinaryWriter): BinaryWriter;
decode(input: BinaryReader | Uint8Array, length?: number): T;
fromJSON(object: any): T;
toJSON(message: T): unknown;
create(base?: DeepPartial): T;
fromPartial(object: DeepPartial): T;
}
================================================
FILE: src/backend/src/filesystem/hl_operations/definitions.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } created or existing directory node
*/
async function createDirOrUseExisting ({ parent, selector, actor, fs }) {
try {
const ll_mkdir = new LLMkdir();
return await ll_mkdir.run({
parent,
name: selector.name,
actor,
});
} catch ( error ) {
// This "error" can occur when multiple `hl_mkdir` operations are being
// run at the same time with the `createMissingParents` option enabled.
const errorCode = error.code || error.fields?.code;
if ( errorCode === 'item_with_same_name_exists' ) {
const existing_node = await fs.node(selector);
// Wait for the entry to be stable (it might still be in the process
// of being created by another parallel request)
await existing_node.awaitStableEntry();
await existing_node.fetchEntry();
// If this is a file we need to re-throw the error
if ( await existing_node.get('type') !== FSNodeContext.TYPE_DIRECTORY ) {
throw error;
}
return existing_node;
}
throw error;
}
}
class MkTree extends HLFilesystemOperation {
static DESCRIPTION = `
High-level operation for making directory trees
The following input for 'tree':
['a/b/c', ['i/j/k'], ['p', ['q'], ['r/s']]]]
Would create a directory tree like this:
a
└── b
└── c
├── i
│ └── j
│ └── k
└── p
├── q
└── r
└── s
`;
static PARAMETERS = {
parent: new FSNodeParam('parent', { optional: true }),
};
static PROPERTIES = {
leaves: () => [],
directories_created: () => [],
};
async _run () {
const { values, context } = this;
const fs = context.get('services').get('filesystem');
await this.create_branch_({
parent_node: values.parent || await fs.node(new RootNodeSelector()),
tree: values.tree,
parent_exists: true,
});
}
async create_branch_ ({ parent_node, tree, parent_exists }) {
const { context } = this;
const fs = context.get('services').get('filesystem');
const actor = context.get('actor');
const trunk = tree[0];
const branches = tree.slice(1);
let current = parent_node.selector;
// trunk = a/b/c
const dirs = trunk === '.' ? []
: trunk.split('/').filter(Boolean);
// dirs = [a, b, c]
let parent_did_exist = parent_exists;
// This is just a loop that goes through each part of the path
// until it finds the first directory that doesn't exist yet.
let i = 0;
if ( parent_exists ) {
for ( ; i < dirs.length ; i++ ) {
const dir = dirs[i];
const currentParent = current;
current = new NodeChildSelector(current, dir);
const maybe_dir = await fs.node(current);
if ( maybe_dir.isRoot ) continue;
if ( await maybe_dir.isUserDirectory() ) continue;
if ( await maybe_dir.exists() ) {
if ( await maybe_dir.get('type') !== FSNodeContext.TYPE_DIRECTORY ) {
throw APIError.create('dest_is_not_a_directory');
}
continue;
}
current = currentParent;
parent_exists = false;
break;
}
}
if ( parent_did_exist && !parent_exists ) {
const node = await fs.node(current);
const has_perm = await chkperm(await node.get('entry'), actor.type.user.id, 'write');
if ( ! has_perm ) throw APIError.create('permission_denied');
}
// This next loop creates the new directories
// We break into a second loop because we know none of these directories
// exist yet. If we continued those checks each child operation would
// wait for the previous one to complete because FSNodeContext::fetchEntry
// will notice ResourceService has a lock on the previous operation
// we started.
// In this way it goes nyyyoooom because all the database inserts
// happen concurrently (and probably end up in the same batch).
for ( ; i < dirs.length ; i++ ) {
const dir = dirs[i];
const currentParent = current;
current = new NodeChildSelector(current, dir);
const node = await createDirOrUseExisting({
parent: await fs.node(currentParent),
selector: current,
actor,
fs,
});
current = node.selector;
this.directories_created.push(node);
}
const bottom_parent = await fs.node(current);
if ( branches.length === 0 ) {
this.leaves.push(bottom_parent);
}
for ( const branch of branches ) {
await this.create_branch_({
parent_node: bottom_parent,
tree: branch,
parent_exists,
});
}
}
}
class QuickMkdir extends HLFilesystemOperation {
async _run () {
const { context, values } = this;
let { parent, path } = values;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
const actor = context.get('actor');
parent = parent || await fs.node(new RootNodeSelector());
let current = parent.selector;
const dirs = path === '.' ? []
: path.split('/').filter(Boolean);
const api = require('@opentelemetry/api');
const currentSpan = api.trace.getSpan(api.context.active());
if ( currentSpan ) {
currentSpan.setAttribute('path', path);
currentSpan.setAttribute('dirs', dirs.join('/'));
currentSpan.setAttribute('parent', parent.selector.describe());
}
for ( let i = 0 ; i < dirs.length ; i++ ) {
const dir = dirs[i];
const currentParent = current;
current = new NodeChildSelector(current, dir);
const node = await createDirOrUseExisting({
parent: await fs.node(currentParent),
selector: current,
actor,
fs,
});
current = node.selector;
// this.directories_created.push(node);
}
this.created = await fs.node(current);
}
}
class HLMkdir extends HLFilesystemOperation {
static DESCRIPTION = `
High-level mkdir operation.
This operation is a wrapper around the low-level mkdir operation.
It provides the following features:
- create missing parent directories
- overwrite existing files
- dedupe names
- create shortcuts
`;
static PARAMETERS = {
parent: new FSNodeParam('parent', { optional: true }),
path: new StringParam('path'),
overwrite: new FlagParam('overwrite', { optional: true }),
create_missing_parents: new FlagParam('create_missing_parents', { optional: true }),
user: new UserParam(),
shortcut_to: new FSNodeParam('shortcut_to', { optional: true }),
};
static MODULES = {
_path: require('path'),
};
static PROPERTIES = {
parent_directories_created: () => [],
};
static FEATURES = [
new OtelFeature([
'_get_existing_parent',
'_create_parents',
]),
];
async _run () {
const { context, values } = this;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
if ( ! is_valid_path(values.path, {
no_relative_components: true,
allow_path_fragment: true,
}) ) {
throw APIError.create('field_invalid', null, {
key: 'path',
expected: 'valid path',
got: 'invalid path',
});
}
// Unify the following formats:
// - full path: {"path":"/foo/bar", args...}, used by apitest (./tools/api-tester/apitest.js)
// - parent + path: {"parent": "/foo", "path":"bar", args...}, used by puter-js (puter.fs.mkdir("/foo/bar"))
if ( !values.parent && values.path ) {
values.parent = await fs.node(new NodePathSelector(_path.dirname(values.path)));
values.path = _path.basename(values.path);
}
let parent_node = values.parent || await fs.node(new RootNodeSelector());
let target_basename = _path.basename(values.path);
// "top_parent" is the immediate parent of the target directory
// (e.g: /home/foo/bar -> /home/foo)
const top_parent = values.create_missing_parents
? await this._create_dir(parent_node)
: await this._get_existing_top_parent({ top_parent: parent_node })
;
// TODO: this can be removed upon completion of: https://github.com/HeyPuter/puter/issues/1352
if ( top_parent.isRoot ) {
// root directory is read-only
throw APIError.create('forbidden', null, {
message: 'Cannot create directories in the root directory.',
});
}
// `parent_node` becomes the parent of the last directory name
// specified under `path`.
parent_node = await this._create_parents({
parent_node: top_parent,
actor: values.actor,
});
const user_id = values.actor.type.user.id;
const has_perm = await chkperm(await parent_node.get('entry'), user_id, 'write');
if ( ! has_perm ) throw APIError.create('permission_denied');
const existing = await fs.node(new NodeChildSelector(parent_node.selector, target_basename));
await existing.fetchEntry();
if ( existing.found ) {
const { overwrite, dedupe_name, create_missing_parents } = values;
if ( overwrite ) {
// TODO: tag rm operation somehow
const has_perm = await chkperm(await existing.get('entry'), user_id, 'write');
if ( ! has_perm ) throw APIError.create('permission_denied');
const hl_remove = new HLRemove();
await hl_remove.run({
target: existing,
actor: values.actor,
recursive: true,
});
}
else if ( dedupe_name ) {
const fs = context.get('services').get('filesystem');
const parent_selector = parent_node.selector;
for ( let i = 1 ;; i++ ) {
let try_new_name = `${target_basename} (${i})`;
const selector = new NodeChildSelector(parent_selector, try_new_name);
const exists = await parent_node.provider.quick_check({
selector,
});
if ( ! exists ) {
target_basename = try_new_name;
break;
}
}
}
else if ( create_missing_parents ) {
if ( ! existing.entry.is_dir ) {
throw APIError.create('dest_is_not_a_directory');
}
this.created = existing;
this.used_existing = true;
return await this.created.getSafeEntry();
} else {
throw APIError.create('item_with_same_name_exists', null, {
entry_name: target_basename,
});
}
}
if ( values.shortcut_to ) {
const shortcut_to = values.shortcut_to;
if ( ! await shortcut_to.exists() ) {
throw APIError.create('shortcut_to_does_not_exist');
}
if ( ! shortcut_to.entry.is_dir ) {
throw APIError.create('shortcut_target_is_a_directory');
}
const has_perm = await chkperm(shortcut_to.entry, user_id, 'read');
if ( ! has_perm ) throw APIError.create('forbidden');
this.created = await fs.mkshortcut({
parent: parent_node,
name: target_basename,
actor: values.actor,
target: shortcut_to,
});
await this.created.awaitStableEntry();
return await this.created.getSafeEntry();
}
let created_node;
try {
const ll_mkdir = new LLMkdir();
created_node = await ll_mkdir.run({
parent: parent_node,
name: target_basename,
actor: values.actor,
});
} catch ( error ) {
// This "error" can occur when multiple `hl_mkdir` operations are being
// run at the same time with the `createMissingParents` option enabled.
const errorCode = error.code || error.fields?.code;
if ( errorCode === 'item_with_same_name_exists' ) {
const existing_node = await fs.node(new NodeChildSelector(parent_node.selector, target_basename));
// Wait for the entry to be stable (it might still be in the process
// of being created by another parallel request)
await existing_node.awaitStableEntry();
await existing_node.fetchEntry();
// If this is a file we need to re-throw the error
if ( await existing_node.get('type') !== FSNodeContext.TYPE_DIRECTORY ) {
throw error;
}
created_node = existing_node;
} else {
throw error;
}
}
this.created = created_node;
const all_nodes = [
...this.parent_directories_created,
this.created,
];
await Promise.all(all_nodes.map(node => node.awaitStableEntry()));
const response = await this.created.getSafeEntry();
response.parent_dirs_created = [];
for ( const node of this.parent_directories_created ) {
response.parent_dirs_created.push(await node.getSafeEntry());
}
response.requested_path = values.path;
return response;
}
async _create_parents ({ parent_node }) {
const { context, values } = this;
const { _path } = this.modules;
const fs = context.get('services').get('filesystem');
// Determine the deepest existing node
let deepest_existing = parent_node;
let remaining_path = _path.dirname(values.path).split('/').filter(Boolean);
{
const parts = remaining_path.slice();
for ( ;; ) {
if ( remaining_path.length === 0 ) {
return deepest_existing;
}
const component = remaining_path[0];
const next_selector = new NodeChildSelector(deepest_existing.selector, component);
const next_node = await fs.node(next_selector);
if ( ! await next_node.exists() ) {
break;
}
deepest_existing = next_node;
remaining_path.shift();
}
}
const tree_op = new MkTree();
await tree_op.run({
parent: deepest_existing,
tree: [remaining_path.join('/')],
});
this.parent_directories_created = tree_op.directories_created;
return tree_op.leaves[0];
}
/**
* Creates a directory and all its ancestors.
*
* @param {FSNodeContext} dir - The directory to create.
* @returns {Promise} The created directory.
*/
async _create_dir (dir) {
if ( await dir.exists() ) {
if ( ! dir.entry.is_dir ) {
throw APIError.create('dest_is_not_a_directory');
}
return dir;
}
const maybe_path_selector =
dir.get_selector_of_type(NodePathSelector);
if ( ! maybe_path_selector ) {
throw APIError.create('dest_does_not_exist', null, { what_dest: 'path from selector' });
}
const path = maybe_path_selector.value;
const fs = this.context.get('services').get('filesystem');
const tree_op = new MkTree();
await tree_op.run({
parent: await fs.node(new RootNodeSelector()),
tree: [path],
});
return tree_op.leaves[0];
}
async _get_existing_top_parent ({ top_parent }) {
if ( ! await top_parent.exists() ) {
throw APIError.create('dest_does_not_exist', null, {
// This seems verbose, but is necessary information when creating
// shortcuts, otherwise the developer doesn't know if we're talking
// about the shortcut's target directory or this parent directory.
what_dest: 'parent directory of the new directory being created',
});
}
if ( ! top_parent.entry.is_dir ) {
throw APIError.create('dest_is_not_a_directory');
}
return top_parent;
}
}
module.exports = {
QuickMkdir,
HLMkdir,
MkTree,
};
================================================
FILE: src/backend/src/filesystem/hl_operations/hl_mklink.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } Result of the write operation
* @throws {APIError} When the target node does not exist
*/
async _run () {
const node = this.values.node;
// Embed fields into this.context
this.context.set('immutable', this.values.immutable);
this.context.set('tmp', this.values.tmp);
this.context.set('fsentry_tmp', this.values.fsentry_tmp);
this.context.set('message', this.values.message);
this.context.set('actor', this.values.actor);
this.context.set('app_id', this.values.app_id);
// TODO: Add symlink write
if ( ! await node.exists() ) {
// TODO: different class of errors for low-level operations
throw APIError.create('subject_does_not_exist');
}
return await node.provider.write_overwrite({
context: this.context,
node: node,
file: this.values.file,
});
}
}
/**
* The "non-overwrite" write operation.
*
* This operation is used to write a file to a non-existent path.
*
* @extends LLFilesystemOperation
*/
class LLCWrite extends LLFilesystemOperation {
static MODULES = {
_path: require('path'),
uuidv4: require('uuid').v4,
config: require('../../config.js'),
};
/**
* Executes the create operation by writing a new file to the parent directory.
* @returns {Promise} Result of the write operation
* @throws {APIError} When the parent directory does not exist
*/
async _run () {
const parent = this.values.parent;
// Embed fields into this.context
this.context.set('immutable', this.context.get('immutable') ?? this.values.immutable);
this.context.set('tmp', this.context.get('tmp') ?? this.values.tmp);
this.context.set('fsentry_tmp', this.context.get('fsentry_tmp') ?? this.values.fsentry_tmp);
this.context.set('message', this.context.get('message') ?? this.values.message);
this.context.set('actor', this.context.get('actor') ?? this.values.actor);
this.context.set('app_id', this.context.get('app_id') ?? this.values.app_id);
if ( ! await parent.exists() ) {
throw APIError.create('subject_does_not_exist');
}
return await parent.provider.write_new({
context: this.context,
parent,
name: this.values.name,
file: this.values.file,
});
}
}
module.exports = {
LLCWrite,
LLOWrite,
};
================================================
FILE: src/backend/src/filesystem/node/selectors.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see _`,
where `` specifies broadly what that strategies contained within
the directory are concerned with (storage, fsentry, etc), and ``
is a letter from A-Z indicating the layer/level of concern.
The class **A** indicates that this is the highest level of swappable
behaviour, which generally means there will be two strategies:
- one which supports legacy behaviour that is coupled with multiple concerns
- one which adapts more cohesive strategies to an interface which
supports the case above.
================================================
FILE: src/backend/src/filesystem/strategies/storage_a/LocalDiskStorageStrategy.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } Resolves when the upload is complete
*/
async _run () {
const { uid, file, storage_api } = this.values;
const { progress_tracker } = storage_api;
if ( file.buffer ) {
await this.parent.svc_localDiskStorage.store_buffer({
key: uid,
buffer: file.buffer,
});
progress_tracker.set_total(file.buffer.length);
progress_tracker.set(file.buffer.length);
} else {
await this.parent.svc_localDiskStorage.store_stream({
key: uid,
stream: file.stream,
size: file.size,
on_progress: evt => {
progress_tracker.set_total(file.size);
progress_tracker.set(evt.uploaded);
},
});
}
}
/**
* Hook called after the operation is inserted into the trace.
*/
post_insert () {
}
}
/**
* Handles file copy operations within local disk storage.
* Extends BaseOperation to provide copy functionality with progress tracking.
*/
class LocalDiskCopyStrategy extends BaseOperation {
/**
* Creates a new LocalDiskCopyStrategy instance.
* @param {Object} parent - The parent storage strategy instance
*/
constructor (parent) {
super();
this.parent = parent;
}
/**
* Executes the copy operation by duplicating a file from source to destination.
* Updates progress tracker to indicate completion.
* @returns {Promise} Resolves when the copy is complete
*/
async _run () {
const { src_node, dst_storage, storage_api } = this.values;
const { progress_tracker } = storage_api;
await this.parent.svc_localDiskStorage.copy({
src_key: await src_node.get('uid'),
dst_key: dst_storage.key,
});
// for now we just copy the file, we don't care about the progress
progress_tracker.set_total(1);
progress_tracker.set(1);
}
/**
* Hook called after the operation is inserted into the trace.
*/
post_insert () {
}
}
/**
* Handles file deletion operations from local disk storage.
* Extends BaseOperation to provide delete functionality.
*/
class LocalDiskDeleteStrategy extends BaseOperation {
/**
* Creates a new LocalDiskDeleteStrategy instance.
* @param {Object} parent - The parent storage strategy instance
*/
constructor (parent) {
super();
this.parent = parent;
}
/**
* Executes the delete operation by removing a file from local disk storage.
* @returns {Promise} Resolves when the deletion is complete
*/
async _run () {
const { node } = this.values;
await this.parent.svc_localDiskStorage.delete({
key: await node.get('uid'),
});
}
}
/**
* Main strategy class for managing local disk storage operations.
* Provides factory methods for creating upload, copy, and delete operations.
*/
class LocalDiskStorageStrategy {
/**
* Creates a new LocalDiskStorageStrategy instance.
* @param {Object} config - Configuration object
* @param {Object} config.services - Services container for dependency injection
*/
constructor ({ services }) {
this.svc_localDiskStorage = services.get('local-disk-storage');
}
/**
* Creates a new upload operation instance.
* @returns {LocalDiskUploadStrategy} A new upload strategy instance
*/
create_upload () {
return new LocalDiskUploadStrategy(this);
}
/**
* Creates a new copy operation instance.
* @returns {LocalDiskCopyStrategy} A new copy strategy instance
*/
create_copy () {
return new LocalDiskCopyStrategy(this);
}
/**
* Creates a new delete operation instance.
* @returns {LocalDiskDeleteStrategy} A new delete strategy instance
*/
create_delete () {
return new LocalDiskDeleteStrategy(this);
}
/**
* Creates a readable stream for accessing file data from local disk storage.
* @param {string} uid - The unique identifier of the file to read
* @param {Object} [options={}] - Optional parameters for stream creation
* @returns {Promise} A readable stream for the file data
*/
async create_read_stream (uid, options = {}) {
return await this.svc_localDiskStorage.create_read_stream(uid, options);
}
}
module.exports = {
LocalDiskStorageStrategy,
};
================================================
FILE: src/backend/src/filesystem/strategies/storage_a/README.md
================================================
## Class A Storage Strategies
This is the broadest definition of storage strategies.
This is to allow swapping between the behaviour of the original
Puter storage logic, and Class B storage strategies.
- they know the UID of the file
- they can perform post-operations after the fsentry is inserted
- they can access the Puter database
================================================
FILE: src/backend/src/filesystem/validation.bench.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see >}
*/
export const get_apps = spanify('get_apps', async (specifiers, options = {}) => {
if ( ! Array.isArray(specifiers) ) {
specifiers = [specifiers];
}
const decorateApp = (app) => {
if ( ! app ) return app;
const icon_url = get_app_icon_url(app.uid ?? app.uuid);
if ( ! icon_url ) return { ...app };
return { ...app, icon: icon_url };
};
const normalizeAppForCache = (app) => {
if ( ! app ) return app;
const normalized = { ...app };
delete normalized.icon_is_base64;
return normalized;
};
const isDecoratedAppCacheEntry = (app) => (
!!app &&
typeof app === 'object' &&
Object.prototype.hasOwnProperty.call(app, 'icon_is_base64')
);
const cacheApp = async (app) => {
if ( ! app ) return;
AppRedisCacheSpace.setCachedApp(app, {
ttlSeconds: 60,
});
};
const normalized = specifiers.map(spec => spec ? { ...spec } : {});
if ( options.follow_old_names ) {
const svc_oldAppName = servicesContainer.services.get('old-app-name');
for ( const spec of normalized ) {
if ( spec.uid || !spec.name ) continue;
const old_name = await svc_oldAppName.check_app_name(spec.name);
if ( old_name ) {
spec.uid = old_name.app_uid;
delete spec.name;
}
}
}
const appByUid = new Map();
const appByName = new Map();
const appById = new Map();
const addApp = (app) => {
if ( ! app ) return;
appByUid.set(app.uid, app);
appByName.set(app.name, app);
appById.set(app.id, app);
};
const pendingLookups = new Map();
const pendingToResolve = new Map();
const queryUids = new Set();
const queryNames = new Set();
const queryIds = new Set();
const queueMissing = (type, value) => {
const queryKey = `${type}:${value}`;
if ( pendingToResolve.has(queryKey) || pendingLookups.has(queryKey) ) {
return;
}
const separatorIndex = queryKey.indexOf(':');
const lookup = queryKey.slice(0, separatorIndex);
value = queryKey.slice(separatorIndex + 1);
const pendingKey = AppRedisCacheSpace.pendingKey({
lookup,
value,
});
const pending = kv.get(pendingKey);
if ( pending ) {
pendingLookups.set(queryKey, pending);
return;
}
let resolveQuery;
let rejectQuery;
const queryPromise = new Promise((resolve, reject) => {
resolveQuery = resolve;
rejectQuery = reject;
});
kv.set(pendingKey, queryPromise, { 'EX': PENDING_QUERY_TTL });
pendingToResolve.set(queryKey, { resolveQuery, rejectQuery, pendingKey });
if ( type === 'uid' ) {
queryUids.add(value);
} else if ( type === 'name' ) {
queryNames.add(value);
} else if ( type === 'id' ) {
queryIds.add(value);
}
};
const cacheLookupPlan = normalized.map((spec) => {
if ( spec.uid ) {
return {
lookup: 'uid',
value: spec.uid,
cacheKey: AppRedisCacheSpace.key({
lookup: 'uid',
value: spec.uid,
}),
};
}
if ( spec.name ) {
return {
lookup: 'name',
value: spec.name,
cacheKey: AppRedisCacheSpace.key({
lookup: 'name',
value: spec.name,
}),
};
}
if ( spec.id ) {
return {
lookup: 'id',
value: spec.id,
cacheKey: AppRedisCacheSpace.key({
lookup: 'id',
value: spec.id,
}),
};
}
return null;
});
const cachedAppsByKey = await redisGetJsonMany(
cacheLookupPlan.filter(Boolean).map(item => item.cacheKey),
);
for ( const plannedLookup of cacheLookupPlan ) {
if ( ! plannedLookup ) continue;
let cached = cachedAppsByKey.get(plannedLookup.cacheKey);
if ( isDecoratedAppCacheEntry(cached) ) {
AppRedisCacheSpace.invalidateCachedApp(cached);
cached = null;
}
if ( cached ) {
addApp(decorateApp(cached));
} else {
queueMissing(plannedLookup.lookup, plannedLookup.value);
}
}
const pendingResultsPromise = pendingLookups.size
? Promise.all(Array.from(pendingLookups.values()))
: Promise.resolve([]);
if ( queryUids.size || queryNames.size || queryIds.size ) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'apps');
const clauses = [];
const params = [];
if ( queryUids.size ) {
const uids = Array.from(queryUids);
clauses.push(`uid IN (${uids.map(() => '?').join(', ')})`);
params.push(...uids);
}
if ( queryNames.size ) {
const names = Array.from(queryNames);
clauses.push(`name IN (${names.map(() => '?').join(', ')})`);
params.push(...names);
}
if ( queryIds.size ) {
const ids = Array.from(queryIds);
clauses.push(`id IN (${ids.map(() => '?').join(', ')})`);
params.push(...ids);
}
let rows = [];
const resolvedKeys = new Set();
try {
rows = await db.read(
`SELECT *, CASE WHEN icon LIKE 'data:%' THEN 1 ELSE 0 END AS icon_is_base64 FROM \`apps\` WHERE ${clauses.join(' OR ')}`,
params,
);
for ( const app of rows ) {
const appForCache = normalizeAppForCache(app);
cacheApp(appForCache);
const decorated_app = decorateApp(appForCache);
addApp(decorated_app);
const uidKey = `uid:${appForCache.uid}`;
const nameKey = `name:${appForCache.name}`;
const idKey = `id:${appForCache.id}`;
if ( pendingToResolve.has(uidKey) ) {
pendingToResolve.get(uidKey).resolveQuery(appForCache);
resolvedKeys.add(uidKey);
}
if ( pendingToResolve.has(nameKey) ) {
pendingToResolve.get(nameKey).resolveQuery(appForCache);
resolvedKeys.add(nameKey);
}
if ( pendingToResolve.has(idKey) ) {
pendingToResolve.get(idKey).resolveQuery(appForCache);
resolvedKeys.add(idKey);
}
}
for ( const [key, { resolveQuery }] of pendingToResolve.entries() ) {
if ( ! resolvedKeys.has(key) ) {
resolveQuery(null);
}
}
} catch ( err ) {
for ( const { rejectQuery } of pendingToResolve.values() ) {
rejectQuery(err);
}
throw err;
} finally {
for ( const { pendingKey } of pendingToResolve.values() ) {
kv.del(pendingKey);
}
}
}
const pendingResults = await pendingResultsPromise;
for ( const app of pendingResults ) {
addApp(decorateApp(app));
}
return normalized.map(spec => {
let app;
if ( spec.uid ) {
app = appByUid.get(spec.uid);
} else if ( spec.name ) {
app = appByName.get(spec.name);
} else if ( spec.id ) {
app = appById.get(spec.id);
}
if ( ! app ) return null;
const result = { ...app };
delete result.icon_is_base64;
return result;
});
});
/**
* Checks to see if an app exists
*
* @param {string} options - `options`
* @returns {Promise}
*/
export async function app_exists (options) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'apps');
let app;
if ( options.uid )
{
app = await db.read('SELECT `id` FROM `apps` WHERE `uid` = ? LIMIT 1', [options.uid]);
}
else if ( options.name )
{
app = await db.read('SELECT `id` FROM `apps` WHERE `name` = ? LIMIT 1', [options.name]);
}
else if ( options.id )
{
app = await db.read('SELECT `id` FROM `apps` WHERE `id` = ? LIMIT 1', [options.id]);
}
return app[0];
}
/**
* change username
*
* @param {string} options - `options`
* @returns {Promise}
*/
export async function change_username (user_id, new_username) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_WRITE, 'auth');
const old_username = (await get_user({ id: user_id })).username;
// update username
await db.write('UPDATE `user` SET username = ? WHERE `id` = ? LIMIT 1', [new_username, user_id]);
// update root directory name for this user
await db.write(
'UPDATE `fsentries` SET `name` = ?, `path` = ? ' +
'WHERE `user_id` = ? AND parent_uid IS NULL LIMIT 1',
[new_username, `/${ new_username}`, user_id],
);
console.log(`User ${old_username} changed username to ${new_username}`);
await servicesContainer.services.get('filesystem').update_child_paths(`/${old_username}`, `/${new_username}`, user_id);
invalidate_cached_user_by_id(user_id);
}
/**
* Find a FSEntry by its uuid
*
* @param {integer} id - `id` of FSEntry
* @returns {Promise} Promise object represents the UUID of the FileSystem Entry
* @deprecated Use fs middleware instead
*/
export async function uuid2fsentry (uuid, return_thumbnail) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
// todo optim, check if uuid is not exactly 36 characters long, if not it's invalid
// and we can avoid one unnecessary DB lookup
let fsentry = await db.requireRead(
`SELECT
id,
associated_app_id,
uuid,
public_token,
bucket,
bucket_region,
file_request_token,
user_id,
parent_uid,
is_dir,
is_public,
is_shortcut,
shortcut_to,
sort_by,
${return_thumbnail ? 'thumbnail,' : ''}
immutable,
name,
metadata,
modified,
created,
accessed,
size
FROM fsentries WHERE uuid = ? LIMIT 1`,
[uuid],
);
if ( ! fsentry[0] )
{
return false;
}
else
{
return fsentry[0];
}
}
/**
* Find a FSEntry by its id
*
* @param {integer} id - `id` of FSEntry
* @returns {Promise} Promise object represents the UUID of the FileSystem Entry
*/
export async function id2fsentry (id, return_thumbnail) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
// todo optim, check if uuid is not exactly 36 characters long, if not it's invalid
// and we can avoid one unnecessary DB lookup
let fsentry = await db.requireRead(
`SELECT
id,
uuid,
public_token,
file_request_token,
associated_app_id,
user_id,
parent_uid,
is_dir,
is_public,
is_shortcut,
shortcut_to,
sort_by,
${return_thumbnail ? 'thumbnail,' : ''}
immutable,
name,
metadata,
modified,
created,
accessed,
size
FROM fsentries WHERE id = ? LIMIT 1`,
[id],
);
if ( ! fsentry[0] ) {
return false;
} else
{
return fsentry[0];
}
}
/**
* Takes a an absolute path and returns its corresponding FSEntry.
*
* @param {string} path - absolute path of the filesystem entry to be resolved
* @param {boolean} return_content - if FSEntry is a file, determines whether its content should be returned
* @returns {false|object} - `false` if path could not be resolved, otherwise an object representing the FSEntry
* @deprecated Use fs middleware instead
*/
export async function convert_path_to_fsentry (path) {
// todo optim, check if path is valid (e.g. contaisn valid characters)
// if syntactical errors are found we can potentially avoid some expensive db lookups
// '/' means that parent_uid is null
// TODO: facade fsentry for root (devlog:2023-06-01)
if ( path === '/' )
{
return null;
}
//first slash is redundant
path = path.substr(path.indexOf('/') + 1);
//last slash, if existing is redundant
if ( path[path.length - 1] === '/' )
{
path = path.slice(0, -1);
}
//split path into parts
const fsentry_names = path.split('/');
// if no parts, return false
if ( fsentry_names.length === 0 )
{
return false;
}
let parent_uid = null;
let final_res = null;
let is_public = false;
let result;
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
// Try stored path first
result = await db.read(
'SELECT * FROM fsentries WHERE path=? LIMIT 1',
[`/${ path}`],
);
if ( result[0] ) {
return result[0];
}
for ( let i = 0; i < fsentry_names.length; i++ ) {
if ( parent_uid === null ) {
result = await db.read(
'SELECT * FROM fsentries WHERE parent_uid IS NULL AND name=? LIMIT 1',
[fsentry_names[i]],
);
}
else {
result = await db.read(
'SELECT * FROM fsentries WHERE parent_uid = ? AND name=? LIMIT 1',
[parent_uid, fsentry_names[i]],
);
}
if ( result[0] ) {
parent_uid = result[0].uuid;
// is_public is either directly specified or inherited from parent dir
if ( result[0].is_public === null )
{
result[0].is_public = is_public;
}
else
{
is_public = result[0].is_public;
}
} else {
return false;
}
final_res = result;
}
return final_res[0];
}
/**
*
* @param {integer} bytes - size in bytes
* @returns {string} bytes in human-readable format
*/
export function byte_format (bytes) {
// calculate and return bytes in human-readable format
const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
if ( typeof bytes !== 'number' || bytes < 1 ) {
return '0 B';
}
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)));
return `${Math.round(bytes / Math.pow(1024, i), 2) } ${ sizes[i]}`;
};
export const get_descendants = spanify('get_descendants', async (...args) => {
return await getDescendantsHelper(...args);
});
/**
*
* @param {integer} entry_id
* @returns
*/
export const id2path = spanify('helpers:id2path', async (entry_uid) => {
if ( entry_uid == null ) {
throw new Error('got null or undefined entry id');
}
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
const log = servicesContainer.services.get('log-service').create('helpers.id2path');
log.traceOn();
const errors = servicesContainer.services.get('error-service').create(log);
log.called();
let result;
log.debug(`entry id: ${entry_uid}`);
if ( typeof entry_uid === 'number' ) {
const old = entry_uid;
entry_uid = await id2uuid(entry_uid);
log.debug(`entry id resolved: resolved ${old} ${entry_uid}`);
}
try {
result = await db.read(`
WITH RECURSIVE cte AS (
SELECT uuid, parent_uid, name, name AS path
FROM fsentries
WHERE uuid = ?
UNION ALL
SELECT e.uuid, e.parent_uid, e.name, ${
db.case({
sqlite: 'e.name || \'/\' || cte.path',
otherwise: 'CONCAT(e.name, \'/\', cte.path)',
})
}
FROM fsentries e
INNER JOIN cte ON cte.parent_uid = e.uuid
)
SELECT *
FROM cte
WHERE parent_uid IS NULL
`, [entry_uid]);
} catch (e) {
errors.report('id2path.select', {
alarm: true,
source: e,
message: `error while resolving path for ${entry_uid}: ${e.message}`,
extra: {
entry_uid,
},
});
throw new ManagedError(`cannot create path for ${entry_uid}`);
}
if ( !result || !result[0] ) {
errors.report('id2path.select', {
alarm: true,
message: `no result for ${entry_uid}`,
extra: {
entry_uid,
},
});
throw new ManagedError(`cannot create path for ${entry_uid}`);
}
return `/${ result[0].path}`;
});
/**
* Recursively retrieve all files, directories, and subdirectories under `path`.
* Optionally the `depth` can be set.
*
* @param {string} path
* @param {object} user
* @param {integer} depth
* @returns
*/
async function getDescendantsHelper (path, user, depth, return_thumbnail = false) {
const log = servicesContainer.services.get('log-service').create('get_descendants');
log.called();
// decrement depth if it's set
depth !== undefined && depth--;
// turn path into absolute form
path = _resolve('/', path);
// get parent dir
const parent = await convert_path_to_fsentry(path);
// holds array that will be returned
const ret = [];
// holds immediate children of this path
let children;
// try to extract username from path
let username;
let split_path = path.split('/');
if ( split_path.length === 2 && split_path[0] === '' )
{
username = split_path[1];
}
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
// -------------------------------------
// parent is root ('/')
// -------------------------------------
if ( parent === null ) {
path = '';
// direct children under root
children = await db.read(
`SELECT
id, uuid, parent_uid, name, metadata, is_dir, bucket, bucket_region,
modified, created, immutable, shortcut_to, is_shortcut, sort_by, associated_app_id,
${return_thumbnail ? 'thumbnail, ' : ''}
accessed, size
FROM fsentries
WHERE user_id = ? AND parent_uid IS NULL`,
[user.id],
);
// users that have shared files/dirs with this user
const sharing_users = await db.read(
`SELECT DISTINCT(owner_user_id), user.username
FROM share
INNER JOIN user ON user.id = share.owner_user_id
WHERE share.recipient_user_id = ?`,
[user.id],
);
if ( sharing_users.length > 0 ) {
for ( let i = 0; i < sharing_users.length; i++ ) {
let dir = {};
dir.id = null;
dir.uuid = null;
dir.parent_uid = null;
dir.name = sharing_users[i].username;
dir.is_dir = true;
dir.immutable = true;
children.push(dir);
}
}
}
// -------------------------------------
// parent doesn't exist
// -------------------------------------
else if ( parent === false ) {
return [];
}
// -------------------------------------
// Parent is a shared-user directory: /[some_username](/)
// but make sure `[some_username]` is not the same as the requester's username
// -------------------------------------
else if ( username && username !== user.username ) {
children = [];
let sharing_user;
sharing_user = await get_user({ username: username });
if ( ! sharing_user )
{
return [];
}
// shared files/dirs with this user
const shared_fsentries = await db.read(
`SELECT
fsentries.id, fsentries.user_id, fsentries.uuid, fsentries.parent_uid, fsentries.bucket, fsentries.bucket_region,
fsentries.name, fsentries.shortcut_to, fsentries.is_shortcut, fsentries.metadata, fsentries.is_dir, fsentries.modified,
fsentries.created, fsentries.accessed, fsentries.size, fsentries.sort_by, fsentries.associated_app_id,
fsentries.is_symlink, fsentries.symlink_path,
fsentries.immutable ${return_thumbnail ? ', fsentries.thumbnail' : ''}
FROM share
INNER JOIN fsentries ON fsentries.id = share.fsentry_id
WHERE share.recipient_user_id = ? AND owner_user_id = ?`,
[user.id, sharing_user.id],
);
// merge `children` and `shared_fsentries`
if ( shared_fsentries.length > 0 ) {
for ( let i = 0; i < shared_fsentries.length; i++ ) {
shared_fsentries[i].path = await id2path(shared_fsentries[i].id);
children.push(shared_fsentries[i]);
}
}
}
// -------------------------------------
// All other cases
// -------------------------------------
else {
children = [];
let temp_children = await db.read(
`SELECT
id, user_id, uuid, parent_uid, name, metadata, is_shortcut,
shortcut_to, is_dir, modified, created, accessed, size, sort_by, associated_app_id,
is_symlink, symlink_path,
immutable ${return_thumbnail ? ', thumbnail' : ''}
FROM fsentries
WHERE parent_uid = ?`,
[parent.uuid],
);
// check if user has access to each file, if yes add it
if ( temp_children.length > 0 ) {
for ( let i = 0; i < temp_children.length; i++ ) {
const tchild = temp_children[i];
if ( await chkperm(tchild, user.id) )
{
children.push(tchild);
}
}
}
}
// shortcut on empty result set
if ( children.length === 0 ) return [];
const ids = children.map(child => child.id);
const qmarks = ids.map(() => '?').join(',');
let rows = await db.read(
`SELECT root_dir_id FROM subdomains WHERE root_dir_id IN (${qmarks}) AND user_id=?`,
[...ids, user.id],
);
const websiteMap = {};
for ( const row of rows ) websiteMap[row.root_dir_id] = true;
for ( let i = 0; i < children.length; i++ ) {
const contentType = _contentType(children[i].name);
// has_website
let has_website = false;
if ( children[i].is_dir ) {
has_website = websiteMap[children[i].id];
}
// object to return
// TODO: DRY creation of response fsentry from db fsentry
ret.push({
path: children[i].path ?? (`${path }/${ children[i].name}`),
name: children[i].name,
metadata: children[i].metadata,
_id: children[i].id,
id: children[i].uuid,
uid: children[i].uuid,
is_shortcut: children[i].is_shortcut,
shortcut_to: (children[i].shortcut_to ? await id2uuid(children[i].shortcut_to) : undefined),
shortcut_to_path: (children[i].shortcut_to ? await id2path(children[i].shortcut_to) : undefined),
is_symlink: children[i].is_symlink,
symlink_path: children[i].symlink_path,
immutable: children[i].immutable,
is_dir: children[i].is_dir,
modified: children[i].modified,
created: children[i].created,
accessed: children[i].accessed,
size: children[i].size,
sort_by: children[i].sort_by,
thumbnail: children[i].thumbnail,
associated_app_id: children[i].associated_app_id,
type: contentType ? contentType : null,
has_website: has_website,
});
if ( children[i].is_dir &&
(depth === undefined || (depth !== undefined && depth > 0))
) {
ret.push(await get_descendants(`${path }/${ children[i].name}`, user, depth));
}
}
return ret.flat();
};
export const get_dir_size = async (path, user) => {
let size = 0;
const descendants = await get_descendants(path, user);
for ( let i = 0; i < descendants.length; i++ ) {
if ( ! descendants[i].is_dir ) {
size += descendants[i].size;
}
}
return size;
};
/**
*
* @param {string} glob
* @param {object} user
* @returns
*/
export async function resolve_glob (glob, user) {
//turn glob into abs path
glob = _resolve('/', glob);
//get base of glob
const base = micromatch.scan(glob).base;
//estimate needed depth
let depth = 1;
const dirs = glob.split('/');
for ( let i = 0; i < dirs.length; i++ ) {
if ( dirs[i].includes('**') ) {
depth = undefined;
break;
} else {
depth++;
}
}
const descendants = await get_descendants(base, user, depth);
return descendants.filter((fsentry) => {
return fsentry.path && micromatch.isMatch(fsentry.path, glob);
});
}
function isString (variable) {
return typeof variable === 'string' || variable instanceof String;
}
export const body_parser_error_handler = (err, req, res, next) => {
if ( err instanceof SyntaxError && err.status === 400 && 'body' in err ) {
return res.status(400).send(err); // Bad request
}
next();
};
/**
* Given a uid, returns a file node.
*
* TODO (xiaochen): It only works for MemoryFSProvider currently.
*
* @param {string} uid - The uid of the file to get.
* @returns {Promise} The file node, or null if the file does not exist.
*/
async function get_entry (uid) {
const svc_mountpoint = Context.get('services').get('mountpoint');
const uid_selector = new NodeUIDSelector(uid);
const provider = await svc_mountpoint.get_provider(uid_selector);
// NB: We cannot import MemoryFSProvider here because it will cause a circular dependency.
if ( provider.constructor.name !== 'MemoryFSProvider' ) {
return null;
}
return provider.stat({
selector: uid_selector,
});
}
export async function is_ancestor_of (ancestor_uid, descendant_uid) {
const ancestor = await get_entry(ancestor_uid);
const descendant = await get_entry(descendant_uid);
if ( ancestor && descendant ) {
return descendant.path.startsWith(ancestor.path);
}
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
// root is an ancestor to all FSEntries
if ( ancestor_uid === null )
{
return true;
}
// root is never a descendant to any FSEntries
if ( descendant_uid === null )
{
return false;
}
if ( typeof ancestor_uid === 'number' ) {
ancestor_uid = await id2uuid(ancestor_uid);
}
if ( typeof descendant_uid === 'number' ) {
descendant_uid = await id2uuid(descendant_uid);
}
let parent = await db.read('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]);
if ( parent[0] === undefined )
{
parent = await db.pread('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]);
}
if ( parent[0].uuid === ancestor_uid || parent[0].parent_uid === ancestor_uid ) {
return true;
}
// keep checking as long as parent of parent is not root
while ( parent[0].parent_uid !== null ) {
parent = await db.read('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [parent[0].parent_uid]);
if ( parent[0] === undefined ) {
parent = await db.pread('SELECT `uuid`, `parent_uid` FROM `fsentries` WHERE `uuid` = ? LIMIT 1', [descendant_uid]);
}
if ( parent[0].uuid === ancestor_uid || parent[0].parent_uid === ancestor_uid ) {
return true;
}
}
return false;
}
export async function sign_file (fsentry, action) {
// fsentry not found
if ( fsentry === false ) {
throw { message: 'No entry found with this uid' };
}
const uid = fsentry.uuid ?? (fsentry.uid ?? fsentry._id);
const ttl = 9999999999999;
const secret = config.url_signature_secret;
const expires = Math.ceil(Date.now() / 1000) + ttl;
const signature = sha256(`${uid}/${action}/${secret}/${expires}`);
const contentType = _contentType(fsentry.name);
// return
return {
uid: uid,
expires: expires,
signature: signature,
url: `${config.api_base_url}/file?uid=${uid}&expires=${expires}&signature=${signature}`,
read_url: `${config.api_base_url}/file?uid=${uid}&expires=${expires}&signature=${signature}`,
write_url: `${config.api_base_url}/writeFile?uid=${uid}&expires=${expires}&signature=${signature}`,
metadata_url: `${config.api_base_url}/itemMetadata?uid=${uid}&expires=${expires}&signature=${signature}`,
fsentry_type: contentType,
fsentry_is_dir: !!fsentry.is_dir,
fsentry_name: fsentry.name,
fsentry_size: fsentry.size,
fsentry_accessed: fsentry.accessed,
fsentry_modified: fsentry.modified,
fsentry_created: fsentry.created,
};
}
export async function gen_public_token (file_uuid) {
// get fsentry
let fsentry = await uuid2fsentry(file_uuid);
// fsentry not found
if ( fsentry === false ) {
throw { message: 'No entry found with this uid' };
}
const uid = fsentry.uuid;
const token = v4();
const contentType = _contentType(fsentry.name);
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_WRITE, 'filesystem');
// insert into DB
try {
await db.write(
'UPDATE fsentries SET public_token = ? WHERE id = ?',
[
//token
token,
//fsentry_id
fsentry.id,
],
);
} catch (e) {
console.log(e);
return false;
}
// return
return {
uid: uid,
token: token,
url: `${config.api_base_url}/pubfile?token=${token}`,
fsentry_type: contentType,
fsentry_is_dir: fsentry.is_dir,
fsentry_name: fsentry.name,
};
}
export async function deleteUser (user_id) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
const svc_fs = servicesContainer.services.get('filesystem');
// get a list of up to 5000 files owned by this user
// eslint-disable-next-line no-constant-condition
for ( let offset = 0; true; offset += 5000 ) {
let files = await db.read(
`SELECT uuid, bucket, bucket_region FROM fsentries WHERE user_id = ? AND is_dir = 0 LIMIT 5000 OFFSET ${ offset}`,
[user_id],
);
if ( !files || files.length == 0 ) break;
// delete all files from S3
if ( files !== null && files.length > 0 ) {
for ( let i = 0; i < files.length; i++ ) {
const node = await svc_fs.node(new NodeUIDSelector(files[i].uuid));
await node.provider.unlink({
context: Context.get(),
override_immutable: true,
node,
});
}
}
}
// delete all fsentries from DB
await db.write('DELETE FROM fsentries WHERE user_id = ?', [user_id]);
// delete user
await db.write('DELETE FROM user WHERE id = ?', [user_id]);
}
export function subdomain (req) {
if ( config.experimental_no_subdomain ) return 'api';
return req.hostname.slice(0, -1 * (config.domain.length + 1));
}
export async function jwt_auth (req, authService) {
let token;
// HTTML Auth header
if ( req.header && req.header('Authorization') )
{
token = req.header('Authorization');
}
// Cookie
else if ( req.cookies && req.cookies[config.cookie_name] )
{
token = req.cookies[config.cookie_name];
}
// Auth token in URL
else if ( req.query && req.query.auth_token )
{
token = req.query.auth_token;
}
// Socket
else if ( req.handshake && req.handshake.auth && req.handshake.auth.auth_token )
{
token = req.handshake.auth.auth_token;
}
if ( !token || token === 'null' )
{
throw ('No auth token found');
}
else if ( typeof token !== 'string' )
{
throw ('token must be a string.');
}
else
{
token = token.replace('Bearer ', '');
}
try {
if ( ! authService ) {
throw new Error('jwt_auth requires authService');
}
const actor = await authService.authenticate_from_token(token);
if ( !actor.type?.constructor?.name === 'UserActorType' ) {
throw ({
message: APIError.create('token_unsupported')
.serialize(),
});
}
return {
actor,
user: actor.type.user,
token: token,
};
} catch (e) {
if ( ! (e instanceof APIError) ) {
console.log('ERROR', e);
}
throw (e.message);
}
}
/**
* returns all ancestors of an fsentry
*
* @param {*} fsentry_id
*/
export async function ancestors (fsentry_id) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
const ancestors = [];
// first parent
let parent = await db.read('SELECT * FROM `fsentries` WHERE `id` = ? LIMIT 1', [fsentry_id]);
if ( parent.length === 0 ) {
return ancestors;
}
// get all subsequent parents
while ( parent[0].parent_uid !== null ) {
const parent_fsentry = await uuid2fsentry(parent[0].parent_uid);
parent = await db.read('SELECT * FROM `fsentries` WHERE `id` = ? LIMIT 1', [parent_fsentry.id]);
if ( parent[0].length !== 0 ) {
ancestors.push(parent[0]);
}
}
return ancestors;
}
export function hyphenize_confirm_code (email_confirm_code) {
email_confirm_code = email_confirm_code.toString();
email_confirm_code =
`${email_confirm_code[0] +
email_confirm_code[1] +
email_confirm_code[2]
}-${
email_confirm_code[3]
}${email_confirm_code[4]
}${email_confirm_code[5]}`;
return email_confirm_code;
}
export async function username_exists (username) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
let rows = await db.read('SELECT EXISTS(SELECT 1 FROM user WHERE username=?) AS username_exists', [username]);
if ( rows[0].username_exists )
{
return true;
}
}
export async function generate_random_username () {
let username;
do {
username = generate_identifier();
} while ( await username_exists(username) );
return username;
}
export async function app_name_exists (name) {
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_READ, 'filesystem');
let rows = await db.read('SELECT EXISTS(SELECT 1 FROM apps WHERE apps.name=?) AS app_name_exists', [name]);
if ( rows[0].app_name_exists )
{
return true;
}
const svc_oldAppName = servicesContainer.services.get('old-app-name');
const name_info = await svc_oldAppName.check_app_name(name);
if ( name_info ) return true;
}
export function send_email_verification_code (email_confirm_code, email) {
const svc_email = Context.get('services').get('email');
svc_email.send_email({ email }, 'email_verification_code', {
code: hyphenize_confirm_code(email_confirm_code),
});
}
export function send_email_verification_token (email_confirm_token, email, user_uuid) {
const svc_email = Context.get('services').get('email');
const link = `${config.origin}/confirm-email-by-token?user_uuid=${user_uuid}&token=${email_confirm_token}`;
svc_email.send_email({ email }, 'email_verification_link', { link });
}
export function generate_random_str (length) {
let result = '';
const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
const charactersLength = characters.length;
for ( let i = 0; i < length; i++ ) {
result += characters.charAt(Math.floor(Math.random() *
charactersLength));
}
return result;
}
/**
* Converts a given number of seconds into a human-readable string format.
*
* @param {number} seconds - The number of seconds to be converted.
* @returns {string} The time represented in the format: 'X years Y days Z hours A minutes B seconds'.
* @throws {TypeError} If the `seconds` parameter is not a number.
*/
export function seconds_to_string (seconds) {
const numyears = Math.floor(seconds / 31536000);
const numdays = Math.floor((seconds % 31536000) / 86400);
const numhours = Math.floor(((seconds % 31536000) % 86400) / 3600);
const numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60);
const numseconds = (((seconds % 31536000) % 86400) % 3600) % 60;
return `${numyears } years ${ numdays } days ${ numhours } hours ${ numminutes } minutes ${ numseconds } seconds`;
}
/**
* returns a list of apps that could open the fsentry, ranked by relevance
* @param {*} fsentry
* @param {*} options
*/
const SUGGEST_APP_CODE_EXTS = [
'.asm',
'.asp',
'.aspx',
'.bash',
'.c',
'.cpp',
'.css',
'.csv',
'.dhtml',
'.f',
'.go',
'.h',
'.htm',
'.html',
'.html5',
'.java',
'.jl',
'.js',
'.jsa',
'.json',
'.jsonld',
'.jsf',
'.jsp',
'.kt',
'.log',
'.lock',
'.lua',
'.md',
'.perl',
'.phar',
'.php',
'.pl',
'.py',
'.r',
'.rb',
'.rdata',
'.rda',
'.rdf',
'.rds',
'.rs',
'.rlib',
'.rpy',
'.scala',
'.sc',
'.scm',
'.sh',
'.sol',
'.sql',
'.ss',
'.svg',
'.swift',
'.toml',
'.ts',
'.wasm',
'.xhtml',
'.xml',
'.yaml',
];
const buildSuggestedAppSpecifiers = async (fsentry) => {
const name_specifiers = [];
let content_type = _contentType(fsentry.name);
if ( ! content_type ) content_type = '';
// IIFE just so fsname can stay `const`
const fsname = (() => {
if ( ! fsentry.name ) {
return 'missing-fsentry-name';
}
let fsname = fsentry.name.toLowerCase();
// We add `.directory` so that this works as a file association
if ( fsentry.is_dir ) fsname += '.directory';
return fsname;
})();
const file_extension = extname(fsname).toLowerCase();
const any_of = (list, name) => list.some(v => name.endsWith(v));
//---------------------------------------------
// Code
//---------------------------------------------
if ( any_of(SUGGEST_APP_CODE_EXTS, fsname) || !fsname.includes('.') ) {
name_specifiers.push({ name: 'code' });
name_specifiers.push({ name: 'editor' });
}
//---------------------------------------------
// Editor
//---------------------------------------------
if (
fsname.endsWith('.txt') ||
// files with no extension
!fsname.includes('.')
) {
name_specifiers.push({ name: 'editor' });
name_specifiers.push({ name: 'code' });
}
//---------------------------------------------
// Markus
//---------------------------------------------
if ( fsname.endsWith('.md') ) {
name_specifiers.push({ name: 'markus' });
}
//---------------------------------------------
// Viewer
//---------------------------------------------
if (
fsname.endsWith('.jpg') ||
fsname.endsWith('.png') ||
fsname.endsWith('.webp') ||
fsname.endsWith('.svg') ||
fsname.endsWith('.bmp') ||
fsname.endsWith('.jpeg')
) {
name_specifiers.push({ name: 'viewer' });
}
//---------------------------------------------
// Draw
//---------------------------------------------
if (
fsname.endsWith('.bmp') ||
content_type.startsWith('image/')
) {
name_specifiers.push({ name: 'draw' });
}
//---------------------------------------------
// PDF
//---------------------------------------------
if ( fsname.endsWith('.pdf') ) {
name_specifiers.push({ name: 'pdf' });
}
//---------------------------------------------
// Player
//---------------------------------------------
if (
fsname.endsWith('.mp4') ||
fsname.endsWith('.webm') ||
fsname.endsWith('.mpg') ||
fsname.endsWith('.mpv') ||
fsname.endsWith('.mp3') ||
fsname.endsWith('.m4a') ||
fsname.endsWith('.ogg')
) {
name_specifiers.push({ name: 'player' });
}
//---------------------------------------------
// 3rd-party apps
//---------------------------------------------
const apps = safe_json_parse(await redisClient.get(
AppRedisCacheSpace.associationAppsKey(file_extension.slice(1)),
), []);
/** @type {{id:string}[]} */
const id_specifiers = apps.map(app_id => ({ id: app_id }));
return { name_specifiers, id_specifiers };
};
const buildSuggestedAppsFromResolved = (resolved, name_specifier_count, options) => {
const suggested_apps = [];
const name_apps = resolved.slice(0, name_specifier_count);
suggested_apps.push(...name_apps);
const third_party_apps = resolved.slice(name_specifier_count);
for ( const third_party_app of third_party_apps ) {
if ( ! third_party_app ) continue;
if ( third_party_app.approved_for_opening_items ||
(options?.user && options.user.id === third_party_app.owner_user_id) )
{
suggested_apps.push(third_party_app);
}
}
const needs_codeapp = suggested_apps.some(app => app && app.name === 'editor');
return { suggested_apps, needs_codeapp };
};
const normalizeSuggestedApps = (suggested_apps) => (
suggested_apps.filter((suggested_app, pos, self) => {
// Remove any null values caused by calling `get_app()` for apps that don't exist.
// This happens on self-host because we don't include `code`, among others.
if ( ! suggested_app ) {
return false;
}
// Remove any duplicate entries
return self.indexOf(suggested_app) === pos;
})
);
const buildSuggestedAppsCacheKey = (fsentry, options) => {
const user_id = options?.user?.id ?? '';
const entry_id = fsentry?.uuid ?? fsentry?.uid ?? fsentry?.id ?? fsentry?.path ?? '';
const entry_name = fsentry?.name ?? '';
const entry_type = fsentry?.is_dir ? 'd' : 'f';
return `${user_id}:${entry_id}:${entry_type}:${entry_name}`;
};
const cloneSuggestedApps = (suggested_apps) => (
Array.isArray(suggested_apps)
? suggested_apps.map(app => (app ? { ...app } : app))
: suggested_apps
);
export async function suggestedAppsForFsEntries (fsentries, options) {
if ( ! Array.isArray(fsentries) ) {
fsentries = [fsentries];
}
const batches = [];
const specifiers = [];
const results = new Array(fsentries.length);
const cacheKeysByIndex = new Map();
for ( let index = 0; index < fsentries.length; index++ ) {
const fsentry = fsentries[index];
if ( ! fsentry ) {
results[index] = [];
continue;
}
const cache_key = buildSuggestedAppsCacheKey(fsentry, options);
const cached = suggestedAppsCache.get(cache_key);
if ( cached !== undefined ) {
results[index] = cloneSuggestedApps(cached);
continue;
}
const { name_specifiers, id_specifiers } = await buildSuggestedAppSpecifiers(fsentry);
const entry_specifiers = [...name_specifiers, ...id_specifiers];
if ( entry_specifiers.length === 0 ) {
results[index] = [];
cacheKeysByIndex.set(index, cache_key);
continue;
}
const offset = specifiers.length;
specifiers.push(...entry_specifiers);
batches.push({
index,
offset,
count: entry_specifiers.length,
name_count: name_specifiers.length,
suggested_apps: [],
needs_codeapp: false,
});
cacheKeysByIndex.set(index, cache_key);
}
let resolved = [];
if ( specifiers.length > 0 ) {
resolved = await get_apps(specifiers);
}
let any_needs_codeapp = false;
for ( const batch of batches ) {
const slice = resolved.slice(batch.offset, batch.offset + batch.count);
const { suggested_apps, needs_codeapp } = buildSuggestedAppsFromResolved(
slice,
batch.name_count,
options,
);
batch.suggested_apps = suggested_apps;
batch.needs_codeapp = needs_codeapp;
if ( needs_codeapp ) any_needs_codeapp = true;
}
let codeapp;
if ( any_needs_codeapp ) {
[codeapp] = await get_apps([{ name: 'codeapp' }]);
}
for ( const batch of batches ) {
let suggested_apps = batch.suggested_apps;
if ( batch.needs_codeapp && codeapp ) {
suggested_apps = [...suggested_apps, codeapp];
}
results[batch.index] = normalizeSuggestedApps(suggested_apps);
}
// Deduplicate results by ID
const deduplicatedResults = results.map(apps => {
if ( ! Array.isArray(apps) ) return apps;
const seen = new Set();
return apps.filter(app => {
if ( !app || !app.id ) return true;
if ( seen.has(app.id) ) return false;
seen.add(app.id);
return true;
});
});
for ( const [index, cache_key] of cacheKeysByIndex ) {
const apps = deduplicatedResults[index];
if ( apps !== undefined ) {
suggestedAppsCache.set(cache_key, cloneSuggestedApps(apps));
}
}
return deduplicatedResults;
}
export async function suggestedAppForFsEntry (fsentry, options) {
const [result] = await suggestedAppsForFsEntries([fsentry], options);
return result;
}
export async function get_taskbar_items (user, {
icon_size: iconSizeFromSnake,
iconSize: iconSizeFromCamel,
no_icons,
} = {}) {
const iconSize = iconSizeFromCamel ?? iconSizeFromSnake;
/** @type BaseDatabaseAccessService */
const db = servicesContainer.services.get('database').get(DB_WRITE, 'filesystem');
let taskbar_items_from_db = [];
// If taskbar items don't exist (specifically NULL)
// add default apps.
if ( ! user.taskbar_items ) {
taskbar_items_from_db = [
{ name: 'app-center', type: 'app' },
{ name: 'dev-center', type: 'app' },
{ name: 'editor', type: 'app' },
{ name: 'code', type: 'app' },
{ name: 'camera', type: 'app' },
{ name: 'recorder', type: 'app' },
];
await db.write(
'UPDATE user SET taskbar_items = ? WHERE id = ?',
[
JSON.stringify(taskbar_items_from_db),
user.id,
],
);
invalidate_cached_user(user);
}
// there are items from before
else {
try {
taskbar_items_from_db = JSON.parse(user.taskbar_items);
} catch (e) {
// ignore errors
}
}
const app_specifiers = taskbar_items_from_db.map((taskbar_item_from_db) => {
if ( taskbar_item_from_db.type !== 'app' ) return {};
if ( taskbar_item_from_db.name === 'explorer' ) return {};
if ( taskbar_item_from_db.name ) {
return { name: taskbar_item_from_db.name };
}
if ( taskbar_item_from_db.id ) {
return { id: taskbar_item_from_db.id };
}
if ( taskbar_item_from_db.uid ) {
return { uid: taskbar_item_from_db.uid };
}
return {};
});
const taskbar_apps = await get_apps(app_specifiers);
// get apps that these taskbar items represent
let taskbar_items = [];
for ( let index = 0; index < taskbar_items_from_db.length; index++ ) {
const taskbar_item_from_db = taskbar_items_from_db[index];
if ( taskbar_item_from_db.type !== 'app' ) continue;
if ( taskbar_item_from_db.name === 'explorer' ) continue;
const item = taskbar_apps[index];
// if item not found, skip it
if ( ! item ) continue;
// delete sensitive attributes
delete item.id;
delete item.owner_user_id;
delete item.timestamp;
// delete item.godmode;
delete item.approved_for_listing;
delete item.approved_for_opening_items;
if ( no_icons ) {
delete item.icon;
} else {
item.icon = get_app_icon_url(item, iconSize);
}
// add to final object
taskbar_items.push(item);
}
return taskbar_items;
}
export function validate_signature_auth (url, action, options = {}) {
const query = new URL(url).searchParams;
if ( ! query.get('uid') )
{
throw { message: '`uid` is required for signature-based authentication.' };
}
else if ( ! action )
{
throw { message: '`action` is required for signature-based authentication.' };
}
else if ( ! query.get('expires') )
{
throw { message: '`expires` is required for signature-based authentication.' };
}
else if ( ! query.get('signature') )
{
throw { message: '`signature` is required for signature-based authentication.' };
}
if ( options.uid ) {
if ( query.get('uid') !== options.uid ) {
throw { message: 'Authentication failed. `uid` does not match.' };
}
}
const expired = query.get('expires') && (query.get('expires') < Date.now() / 1000);
// expired?
if ( expired )
{
throw { message: 'Authentication failed. Signature expired.' };
}
const uid = query.get('uid');
const secret = config.url_signature_secret;
// before doing anything, see if this signature is valid for 'write' action, if yes that means every action is allowed
if ( !expired && query.get('signature') === sha256(`${uid}/write/${secret}/${query.get('expires')}`) )
{
return true;
}
// if not, check specific actions
else if ( !expired && query.get('signature') === sha256(`${uid}/${action}/${secret}/${query.get('expires')}`) )
{
return true;
}
// auth failed
else
{
throw { message: 'Authentication failed' };
}
}
export function get_url_from_req (req) {
return `${req.protocol }://${ req.get('host') }${req.originalUrl}`;
}
/**
* Formats a number with grouped thousands.
*
* @param {number|string} number - The number to be formatted. If a string is provided, it must only contain numerical characters, plus and minus signs, and the letter 'E' or 'e' (for scientific notation).
* @param {number} decimals - The number of decimal points. If a non-finite number is provided, it defaults to 0.
* @param {string} [dec_point='.'] - The character used for the decimal point. Defaults to '.' if not provided.
* @param {string} [thousands_sep=','] - The character used for the thousands separator. Defaults to ',' if not provided.
* @returns {string} The formatted number with grouped thousands, using the specified decimal point and thousands separator characters.
* @throws {TypeError} If the `number` parameter cannot be converted to a finite number, or if the `decimals` parameter is non-finite and cannot be converted to an absolute number.
*/
export function number_format (number, decimals, dec_point, thousands_sep) {
// Strip all characters but numerical ones.
number = (`${number }`).replace(/[^0-9+\-Ee.]/g, '');
let n = !isFinite(+number) ? 0 : +number,
prec = !isFinite(+decimals) ? 0 : Math.abs(decimals),
sep = (typeof thousands_sep === 'undefined') ? ',' : thousands_sep,
dec = (typeof dec_point === 'undefined') ? '.' : dec_point,
s = '',
toFixedFix = function (n, prec) {
const k = Math.pow(10, prec);
return `${ Math.round(n * k) / k}`;
};
// Fix for IE parseFloat(0.55).toFixed(0) = 0;
s = (prec ? toFixedFix(n, prec) : `${ Math.round(n)}`).split('.');
if ( s[0].length > 3 ) {
s[0] = s[0].replace(/\B(?=(?:\d{3})+(?!\d))/g, sep);
}
if ( (s[1] || '').length < prec ) {
s[1] = s[1] || '';
s[1] += new Array(prec - s[1].length + 1).join('0');
}
return s.join(dec);
}
================================================
FILE: src/backend/src/index.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } An object containing:
* - {Object} open_count - Open counts for different time periods
* - {Object} user_count - Uniqu>e user counts for different time periods
* - {number|null} referral_count - The number of referrals (all-time only)
*/
async get_stats (app_uid, options = {}) {
let period = options.period ?? 'all';
let stats_grouping = options.grouping;
let app_creation_ts = options.created_at;
const parse_cached_int = (value) => {
if ( value === null || value === undefined ) return null;
const parsed = parseInt(value, 10);
return Number.isNaN(parsed) ? null : parsed;
};
// Check cache first if period is 'all' and no grouping is requested
if ( period === 'all' && !stats_grouping ) {
const key_open_count = AppRedisCacheSpace.openCountKey(app_uid);
const key_user_count = AppRedisCacheSpace.userCountKey(app_uid);
const key_referral_count = AppRedisCacheSpace.referralCountKey(app_uid);
const [cached_open_count, cached_user_count, cached_referral_count] = await Promise.all([
redisClient.get(key_open_count),
redisClient.get(key_user_count),
redisClient.get(key_referral_count),
]);
const cached_open_count_parsed = parse_cached_int(cached_open_count);
const cached_user_count_parsed = parse_cached_int(cached_user_count);
if ( cached_open_count_parsed !== null && cached_user_count_parsed !== null ) {
return {
open_count: cached_open_count_parsed,
user_count: cached_user_count_parsed,
referral_count: parse_cached_int(cached_referral_count),
};
}
}
const db = this.services.get('database').get(DB_READ, 'apps');
const getTimeRange = (period) => {
const now = new Date();
const today = new Date(now.getFullYear(), now.getMonth(), now.getDate());
switch ( period ) {
case 'today':
return {
start: today.getTime(),
end: now.getTime(),
};
case 'yesterday': {
const yesterday = new Date(today);
yesterday.setDate(yesterday.getDate() - 1);
return {
start: yesterday.getTime(),
end: today.getTime() - 1,
};
}
case '7d': {
const weekAgo = new Date(now);
weekAgo.setDate(weekAgo.getDate() - 7);
return {
start: weekAgo.getTime(),
end: now.getTime(),
};
}
case '30d': {
const monthAgo = new Date(now);
monthAgo.setDate(monthAgo.getDate() - 30);
return {
start: monthAgo.getTime(),
end: now.getTime(),
};
}
case 'this_week': {
const firstDayOfWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay());
return {
start: firstDayOfWeek.getTime(),
end: now.getTime(),
};
}
case 'last_week': {
const firstDayOfLastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay() - 7);
const firstDayOfThisWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - now.getDay());
return {
start: firstDayOfLastWeek.getTime(),
end: firstDayOfThisWeek.getTime() - 1,
};
}
case 'this_month': {
const firstDayOfMonth = new Date(now.getFullYear(), now.getMonth(), 1);
return {
start: firstDayOfMonth.getTime(),
end: now.getTime(),
};
}
case 'last_month': {
const firstDayOfLastMonth = new Date(now.getFullYear(), now.getMonth() - 1, 1);
const firstDayOfThisMonth = new Date(now.getFullYear(), now.getMonth(), 1);
return {
start: firstDayOfLastMonth.getTime(),
end: firstDayOfThisMonth.getTime() - 1,
};
}
case 'this_year': {
const firstDayOfYear = new Date(now.getFullYear(), 0, 1);
return {
start: firstDayOfYear.getTime(),
end: now.getTime(),
};
}
case 'last_year': {
const firstDayOfLastYear = new Date(now.getFullYear() - 1, 0, 1);
const firstDayOfThisYear = new Date(now.getFullYear(), 0, 1);
return {
start: firstDayOfLastYear.getTime(),
end: firstDayOfThisYear.getTime() - 1,
};
}
case '12m': {
const twelveMonthsAgo = new Date(now);
twelveMonthsAgo.setMonth(twelveMonthsAgo.getMonth() - 12);
return {
start: twelveMonthsAgo.getTime(),
end: now.getTime(),
};
}
case 'all': {
const start = new Date(app_creation_ts);
return {
start: start.getTime(),
end: now.getTime(),
};
}
default:
return null;
}
};
const timeRange = getTimeRange(period);
// Handle time-based grouping if stats_grouping is specified
if ( stats_grouping ) {
const timeFormat = this.mysqlDateFormats[stats_grouping];
if ( ! timeFormat ) {
throw new Error(`Invalid stats_grouping: ${stats_grouping}. Supported values are: hour, day, week, month, year`);
}
// Generate all periods for the time range
const allPeriods = this.generateAllPeriods(
new Date(timeRange.start),
new Date(timeRange.end),
stats_grouping,
);
if ( global.clickhouseClient ) {
const groupByFormat = this.clickhouseGroupByFormats[stats_grouping];
const timeCondition = timeRange ?
`AND ts >= ${Math.floor(timeRange.start / 1000)} AND ts < ${Math.floor(timeRange.end / 1000)}` : '';
const [openResult, userResult] = await Promise.all([
global.clickhouseClient.query({
query: `
SELECT
${groupByFormat} as period,
COUNT(_id) as count
FROM app_opens
WHERE app_uid = '${app_uid}'
${timeCondition}
GROUP BY period
ORDER BY period
`,
format: 'JSONEachRow',
}),
global.clickhouseClient.query({
query: `
SELECT
${groupByFormat} as period,
COUNT(DISTINCT user_id) as count
FROM app_opens
WHERE app_uid = '${app_uid}'
${timeCondition}
GROUP BY period
ORDER BY period
`,
format: 'JSONEachRow',
}),
]);
const openRows = await openResult.json();
const userRows = await userResult.json();
// Ensure counts are properly parsed as integers
const processedOpenRows = openRows.map(row => ({
period: new Date(row.period),
count: parseInt(row.count),
}));
const processedUserRows = userRows.map(row => ({
period: new Date(row.period),
count: parseInt(row.count),
}));
// Calculate totals from the processed rows
const totalOpenCount = processedOpenRows.reduce((sum, row) => sum + row.count, 0);
const totalUserCount = processedUserRows.reduce((sum, row) => sum + row.count, 0);
// Generate all periods and merge with actual data
const allPeriods = this.generateAllPeriods(
new Date(timeRange.start),
new Date(timeRange.end),
stats_grouping,
);
const completeOpenStats = this.mergeWithGeneratedPeriods(processedOpenRows, allPeriods, stats_grouping);
const completeUserStats = this.mergeWithGeneratedPeriods(processedUserRows, allPeriods, stats_grouping);
return {
open_count: totalOpenCount,
user_count: totalUserCount,
grouped_stats: {
open_count: completeOpenStats,
user_count: completeUserStats,
},
referral_count: period === 'all'
? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))
: null,
};
}
else {
// MySQL queries for grouped stats
const queryParams = timeRange ?
[app_uid, timeRange.start / 1000, timeRange.end / 1000] :
[app_uid];
const [openResult, userResult] = await Promise.all([
db.read(`
SELECT ${db.case({
mysql: `DATE_FORMAT(FROM_UNIXTIME(ts/1000), '${timeFormat}') as period, `,
sqlite: `STRFTIME('%Y-%m-%d %H', datetime(ts/1000, 'unixepoch'), '${timeFormat}') as period, `,
})
}
COUNT(_id) as count
FROM app_opens
WHERE app_uid = ?
${timeRange ? 'AND ts >= ? AND ts < ?' : ''}
GROUP BY period
ORDER BY period
`, queryParams),
db.read(`
SELECT ${db.case({
mysql: `DATE_FORMAT(FROM_UNIXTIME(ts/1000), '${timeFormat}') as period, `,
sqlite: `STRFTIME('%Y-%m-%d %H', datetime(ts/1000, 'unixepoch'), '${timeFormat}') as period, `,
})
}
COUNT(DISTINCT user_id) as count
FROM app_opens
WHERE app_uid = ?
${timeRange ? 'AND ts >= ? AND ts < ?' : ''}
GROUP BY period
ORDER BY period
`, queryParams),
]);
// Calculate totals
const totalOpenCount = openResult.reduce((sum, row) => sum + parseInt(row.count), 0);
const totalUserCount = userResult.reduce((sum, row) => sum + parseInt(row.count), 0);
// Convert MySQL results to the same format as needed
const openRows = openResult.map(row => ({
period: row.period,
count: parseInt(row.count),
}));
const userRows = userResult.map(row => ({
period: row.period,
count: parseInt(row.count),
}));
// Merge with generated periods to include zero-value periods
const completeOpenStats = this.mergeWithGeneratedPeriods(openRows, allPeriods, stats_grouping);
const completeUserStats = this.mergeWithGeneratedPeriods(userRows, allPeriods, stats_grouping);
return {
open_count: totalOpenCount,
user_count: totalUserCount,
grouped_stats: {
open_count: completeOpenStats,
user_count: completeUserStats,
},
referral_count: period === 'all'
? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))
: null,
};
}
}
// Handle non-grouped stats
if ( global.clickhouseClient ) {
const openCountQuery = timeRange
? `SELECT COUNT(_id) AS open_count FROM app_opens
WHERE app_uid = '${app_uid}'
AND ts >= ${Math.floor(timeRange.start / 1000)}
AND ts < ${Math.floor(timeRange.end / 1000)}`
: `SELECT COUNT(_id) AS open_count FROM app_opens
WHERE app_uid = '${app_uid}'`;
const userCountQuery = timeRange
? `SELECT COUNT(DISTINCT user_id) AS uniqueUsers FROM app_opens
WHERE app_uid = '${app_uid}'
AND ts >= ${Math.floor(timeRange.start / 1000)}
AND ts < ${Math.floor(timeRange.end / 1000)}`
: `SELECT COUNT(DISTINCT user_id) AS uniqueUsers FROM app_opens
WHERE app_uid = '${app_uid}'`;
const [openResult, userResult] = await Promise.all([
global.clickhouseClient.query({
query: openCountQuery,
format: 'JSONEachRow',
}),
global.clickhouseClient.query({
query: userCountQuery,
format: 'JSONEachRow',
}),
]);
const openRows = await openResult.json();
const userRows = await userResult.json();
const results = {
open_count: parseInt(openRows[0].open_count),
user_count: parseInt(userRows[0].uniqueUsers),
referral_count: period === 'all'
? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))
: null,
};
// Cache the results if period is 'all'
if ( period === 'all' ) {
const key_open_count = AppRedisCacheSpace.openCountKey(app_uid);
const key_user_count = AppRedisCacheSpace.userCountKey(app_uid);
void Promise.all([
setRedisCacheValue(key_open_count, results.open_count),
setRedisCacheValue(key_user_count, results.user_count),
]);
}
return results;
} else {
// Regular MySQL queries for non-grouped stats
const baseOpenQuery = 'SELECT COUNT(_id) AS open_count FROM app_opens WHERE app_uid = ?';
const baseUserQuery = 'SELECT COUNT(DISTINCT user_id) AS user_count FROM app_opens WHERE app_uid = ?';
const generateQuery = (baseQuery, timeRange) => {
if ( ! timeRange ) return baseQuery;
return `${baseQuery} AND ts >= ? AND ts < ?`;
};
const openQuery = generateQuery(baseOpenQuery, timeRange);
const userQuery = generateQuery(baseUserQuery, timeRange);
const queryParams = timeRange ? [app_uid, timeRange.start, timeRange.end] : [app_uid];
const [openResult, userResult] = await Promise.all([
db.read(openQuery, queryParams),
db.read(userQuery, queryParams),
]);
const results = {
open_count: parseInt(openResult[0].open_count),
user_count: parseInt(userResult[0].user_count),
referral_count: period === 'all'
? parse_cached_int(await redisClient.get(AppRedisCacheSpace.referralCountKey(app_uid)))
: null,
};
// Cache the results if period is 'all'
if ( period === 'all' ) {
const key_open_count = AppRedisCacheSpace.openCountKey(app_uid);
const key_user_count = AppRedisCacheSpace.userCountKey(app_uid);
void Promise.all([
setRedisCacheValue(key_open_count, results.open_count),
setRedisCacheValue(key_user_count, results.user_count),
]);
}
return results;
}
}
/**
* Refreshes the cache of app statistics including open and user counts.
*
* @notes
* - This method logs a tick event for performance monitoring.
*
* @async
* @returns {Promise} A promise that resolves when the cache refresh operation is complete.
*/
async _refresh_app_stats () {
this.log.tick('refresh app stats');
const db = this.services.get('database').get(DB_READ, 'apps');
let openCountMap;
let userCountMap;
if ( global.clickhouseClient ) {
const [openResult, userResult] = await Promise.all([
global.clickhouseClient.query({
query: `
SELECT app_uid, COUNT(_id) AS open_count
FROM app_opens
GROUP BY app_uid
`,
format: 'JSONEachRow',
}),
global.clickhouseClient.query({
query: `
SELECT app_uid, COUNT(DISTINCT user_id) AS user_count
FROM app_opens
GROUP BY app_uid
`,
format: 'JSONEachRow',
}),
]);
const openRows = await openResult.json();
const userRows = await userResult.json();
openCountMap = new Map(openRows.map(row => [row.app_uid, parseInt(row.open_count, 10)]));
userCountMap = new Map(userRows.map(row => [row.app_uid, parseInt(row.user_count, 10)]));
} else {
const [openCounts, userCounts] = await Promise.all([
db.read(`
SELECT app_uid, COUNT(_id) AS open_count
FROM app_opens
GROUP BY app_uid
`),
db.read(`
SELECT app_uid, COUNT(DISTINCT user_id) AS user_count
FROM app_opens
GROUP BY app_uid
`),
]);
openCountMap = new Map(openCounts.map(row => [row.app_uid, row.open_count]));
userCountMap = new Map(userCounts.map(row => [row.app_uid, row.user_count]));
}
// Get all app UIDs and update the cache (apps list lives in MySQL)
const apps = await db.read('SELECT uid FROM apps');
for ( const app of apps ) {
const key_open_count = AppRedisCacheSpace.openCountKey(app.uid);
const key_user_count = AppRedisCacheSpace.userCountKey(app.uid);
// Background refresh writes should stay local to avoid broadcast churn.
void Promise.all([
setRedisCacheValue(key_open_count, openCountMap.get(app.uid) ?? 0, { emitEvent: false }),
setRedisCacheValue(key_user_count, userCountMap.get(app.uid) ?? 0, { emitEvent: false }),
]);
}
}
/**
* Refreshes the cache of app referral statistics.
*
* This method queries the database for user counts referred by each app's origin URL
* and updates the cache with the referral counts for each app.
*
* @notes
* - This method logs a tick event for performance monitoring.
*
* @async
* @returns {Promise} A promise that resolves when the cache refresh operation is complete.
*/
async _refresh_app_stat_referrals () {
this.log.tick('refresh app stat referrals');
const db = this.services.get('database').get(DB_READ, 'apps');
const apps = await db.read('SELECT uid, index_url FROM apps');
// First, build a map of valid app origins to UIDs
const validApps = [];
const svc_auth = this.services.get('auth');
for ( const app of apps ) {
const origin = origin_from_url(app.index_url);
// only count the referral if the origin hashes to the app's uid
let expected_uid;
try {
expected_uid = await svc_auth.app_uid_from_origin(origin);
} catch (e) {
// This happens if the app origin isn't valid
continue;
}
if ( expected_uid !== app.uid ) {
continue;
}
validApps.push({ uid: app.uid, origin });
}
if ( validApps.length === 0 ) {
return;
}
// Build a single query to get all referral counts
const likeConditions = validApps.map(() => 'referrer LIKE ?').join(' OR ');
const queryParams = validApps.map(app => `${app.origin}%`);
const referralResults = await db.read(`
SELECT
referrer,
COUNT(id) as referral_count
FROM user
WHERE ${likeConditions}
GROUP BY referrer
`, queryParams);
// Create a map to store referral counts by origin
const referralMap = new Map();
for ( const result of referralResults ) {
// Find which app this referrer belongs to
for ( const app of validApps ) {
if ( result.referrer.startsWith(app.origin) ) {
const currentCount = referralMap.get(app.uid) || 0;
referralMap.set(app.uid, currentCount + parseInt(result.referral_count));
break;
}
}
}
// Update cache with results
for ( const app of validApps ) {
const key_referral_count = AppRedisCacheSpace.referralCountKey(app.uid);
const count = referralMap.get(app.uid) || 0;
// Background refresh writes should stay local to avoid broadcast churn.
await setRedisCacheValue(key_referral_count, count, { emitEvent: false });
}
this.log.info('DONE refresh app stat referrals');
}
/**
* Deletes an application from the system.
*
* This method performs the following actions:
* - Retrieves the app data from cache or database if not provided.
* - Deletes the app record from the database.
* - Removes the app from all relevant caches (by name, id, and uid).
* - Removes the app from any associated tags.
*
* @param {string} app_uid - The unique identifier of the app to be deleted.
* @param {Object} [app] - The app object, if already fetched. If not provided, it will be retrieved.
* @param {Object} [options] - Optional delete behavior flags.
* @throws {Error} If the app is not found in either cache or database.
* @returns {Promise} A promise that resolves when the app has been successfully deleted.
*/
async delete_app (app_uid, app, options = {}) {
const db = this.services.get('database').get(DB_READ, 'apps');
if ( ! app ) {
app = await AppRedisCacheSpace.getCachedApp({
lookup: 'uid',
value: app_uid,
});
}
if ( ! app ) {
app = (await db.read(
'SELECT * FROM apps WHERE uid = ?',
[app_uid],
))[0];
}
if ( ! app ) {
throw new Error('app not found');
}
const associationRows = await db.read(
'SELECT type FROM app_filetype_association WHERE app_id = ?',
[app.id],
);
await db.write(
'DELETE FROM apps WHERE uid = ? LIMIT 1',
[app_uid],
);
if ( ! options.preserveCanonicalUidAlias ) {
await this.cleanupCanonicalAppUidAliases_(app_uid);
}
// remove from caches
AppRedisCacheSpace.invalidateCachedApp(app, {
includeStats: true,
});
const associationKeys = associationRows
.map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\./, ''))
.filter(Boolean)
.map(ext => AppRedisCacheSpace.associationAppsKey(ext));
if ( associationKeys.length ) {
await deleteRedisKeys(associationKeys);
}
// remove from tags
const app_tags = (app.tags ?? '').split(',')
.map(tag => tag.trim())
.filter(tag => tag.length > 0);
for ( const tag of app_tags ) {
if ( ! this.tags[tag] ) continue;
const index = this.tags[tag].indexOf(app_uid);
if ( index >= 0 ) {
this.tags[tag].splice(index, 1);
}
}
const svc_event = this.services.get('event');
await svc_event.emit('app.changed', {
app_uid: app.uid,
action: 'deleted',
app,
});
}
buildCanonicalAppUidAliasKey_ (appUid) {
return `${APP_UID_ALIAS_KEY_PREFIX}:${appUid}`;
}
buildCanonicalAppUidAliasReverseKey_ (canonicalAppUid) {
return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`;
}
normalizeCanonicalAliasUidList_ (value) {
if ( ! Array.isArray(value) ) return [];
const normalizedList = [];
const seen = new Set();
for ( const item of value ) {
if ( typeof item !== 'string' || !item ) continue;
if ( seen.has(item) ) continue;
seen.add(item);
normalizedList.push(item);
}
return normalizedList;
}
async cleanupCanonicalAppUidAliases_ (appUid) {
if ( typeof appUid !== 'string' || !appUid ) return;
const kvStore = this.services.get('puter-kvstore');
const suService = this.services.get('su');
if ( !kvStore || typeof kvStore.get !== 'function' || typeof kvStore.del !== 'function' ) return;
if ( !suService || typeof suService.sudo !== 'function' ) return;
const selfAliasKey = this.buildCanonicalAppUidAliasKey_(appUid);
const reverseKey = this.buildCanonicalAppUidAliasReverseKey_(appUid);
try {
await suService.sudo(async () => {
const reverseValue = await kvStore.get({ key: reverseKey });
const reverseAliases = this.normalizeCanonicalAliasUidList_(reverseValue);
const deleteOps = [
kvStore.del({ key: selfAliasKey }),
kvStore.del({ key: reverseKey }),
];
for ( const oldUid of reverseAliases ) {
deleteOps.push(kvStore.del({
key: this.buildCanonicalAppUidAliasKey_(oldUid),
}));
}
await Promise.all(deleteOps);
});
} catch {
// KV cleanup is best-effort.
}
}
// Helper function to generate array of all periods between start and end dates
generateAllPeriods (startDate, endDate, grouping) {
const periods = [];
let currentDate = new Date(startDate);
// ???: In local debugging, `currentDate` evaluates to `Invalid Date`.
// Does this work in prod?
while ( currentDate <= endDate ) {
let period;
switch ( grouping ) {
case 'hour':
period = `${currentDate.toISOString().slice(0, 13)}:00:00`;
currentDate.setHours(currentDate.getHours() + 1);
break;
case 'day':
period = currentDate.toISOString().slice(0, 10);
currentDate.setDate(currentDate.getDate() + 1);
break;
case 'week': {
// Get the ISO week number
const weekNum = String(this.getWeekNumber(currentDate)).padStart(2, '0');
period = `${currentDate.getFullYear()}-${weekNum}`;
currentDate.setDate(currentDate.getDate() + 7);
break;
}
case 'month':
period = currentDate.toISOString().slice(0, 7);
currentDate.setMonth(currentDate.getMonth() + 1);
break;
case 'year':
period = currentDate.getFullYear().toString();
currentDate.setFullYear(currentDate.getFullYear() + 1);
break;
}
periods.push({ period, count: 0 });
}
return periods;
}
// Helper function to get ISO week number
getWeekNumber (date) {
const target = new Date(date.valueOf());
const dayNumber = (date.getDay() + 6) % 7;
target.setDate(target.getDate() - dayNumber + 3);
const firstThursday = target.valueOf();
target.setMonth(0, 1);
if ( target.getDay() !== 4 ) {
target.setMonth(0, 1 + ((4 - target.getDay()) + 7) % 7);
}
return 1 + Math.ceil((firstThursday - target) / 604800000);
}
// Helper function to merge actual data with generated periods
mergeWithGeneratedPeriods (actualData, allPeriods, stats_grouping) {
// Create a map of period to count from actual data
// First normalize the period format from both MySQL and ClickHouse
const dataMap = new Map(actualData.map(item => {
let period = item.period;
// For ClickHouse results, convert the timestamp to match the expected format
if ( item.period instanceof Date ) {
switch ( stats_grouping ) {
case 'hour':
period = `${item.period.toISOString().slice(0, 13)}:00:00`;
break;
case 'day':
period = item.period.toISOString().slice(0, 10);
break;
case 'week': {
const weekNum = String(this.getWeekNumber(item.period)).padStart(2, '0');
period = `${item.period.getFullYear()}-${weekNum}`;
break;
}
case 'month':
period = item.period.toISOString().slice(0, 7);
break;
case 'year':
period = item.period.getFullYear().toString();
break;
}
}
return [period, parseInt(item.count)];
}));
// Map the generated periods to include actual counts where they exist
return allPeriods.map(periodObj => {
const count = dataMap.get(periodObj.period);
return {
period: periodObj.period,
count: count !== undefined ? count : 0,
};
});
}
}
module.exports = {
AppInformationService,
};
================================================
FILE: src/backend/src/modules/apps/AppPermissionService.js
================================================
const { UserActorType } = require('../../services/auth/Actor');
const { PermissionImplicator, PermissionUtil } = require('../../services/auth/permissionUtils.mjs');
const BaseService = require('../../services/BaseService');
class AppPermissionService extends BaseService {
async _init () {
const svc_permission = this.services.get('permission');
svc_permission.register_implicator(PermissionImplicator.create({
id: 'user-can-grant-read-own-apps',
matcher: permission => {
return permission.startsWith('apps-of-user:') ||
permission.startsWith('subdomains-of-user:');
},
checker: async ({ actor, permission }) => {
if ( ! (actor.type instanceof UserActorType) ) {
return undefined;
}
const parts = PermissionUtil.split(permission);
if ( parts[1] === actor.type.user.uuid ) {
return {};
}
},
}));
}
}
module.exports = {
AppPermissionService,
};
================================================
FILE: src/backend/src/modules/apps/AppRedisCacheSpace.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves when the initialization is complete.
*/
async _init () {
const svc_permission = this.services.get('permission');
svc_permission.register_rewriter(PermissionRewriter.create({
matcher: permission => {
if ( ! permission.startsWith('app:') ) return false;
const [_, specifier] = PermissionUtil.split(permission);
if ( specifier.startsWith('uid#') ) return false;
return true;
},
rewriter: async permission => {
const [_1, name, ...rest] = PermissionUtil.split(permission);
const app = await get_app({ name });
return PermissionUtil.join(_1, `uid#${app.uid}`, ...rest);
},
}));
// track: object description in comment
// Owner of procted app has implicit permission to access it
svc_permission.register_implicator(PermissionImplicator.create({
matcher: permission => {
return permission.startsWith('app:') || permission.startsWith('manage:app');
},
checker: async ({ actor, permission }) => {
if ( ! (actor.type instanceof UserActorType) ) {
return undefined;
}
const parts = PermissionUtil.split(permission);
if ( parts[0] === 'manage' ) parts.shift();
if ( parts.length < 2 ) return undefined;
const [_, uid_part] = parts;
// track: slice a prefix
const uid = uid_part.slice('uid#'.length);
const app = await get_app({ uid });
if ( app.owner_user_id !== actor.type.user.id ) {
return undefined;
}
return {};
},
}));
}
}
module.exports = {
ProtectedAppService,
};
================================================
FILE: src/backend/src/modules/apps/RecommendedAppsRedisCacheSpace.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
async _init () {
const services = this.services;
this.pager = services.get('pager');
// TODO:[self-hosted] fix this properly
this.known_errors = [];
}
/**
* AlarmService registers its commands at the consolidation phase because
* the '_init' method of CommandService may not have been called yet.
*/
'__on_boot.consolidation' () {
this._register_commands(this.services.get('commands'));
}
adapt_id_ (id) {
let shorten = true;
if ( shorten ) {
const rng = seedrandom(id);
id = this.identutil.generate_identifier('-', rng);
}
return id;
}
/**
* Method to create an alarm with the given ID, message, and fields.
* If the ID already exists, it will be updated with the new fields
* and the occurrence count will be incremented.
*
* @param {string} id - Unique identifier for the alarm.
* @param {string} message - Message associated with the alarm.
* @param {object} fields - Additional information about the alarm.
*/
create (id, message, fields) {
if ( this.config.log_upcoming_alarms ) {
this.log.error(`upcoming alarm: ${id}: ${message}`);
}
let existing = false;
/**
* Method to create an alarm with the given ID, message, and fields.
* If the ID already exists, it will be updated with the new fields.
* @param {string} id - Unique identifier for the alarm.
* @param {string} message - Message associated with the alarm.
* @param {object} fields - Additional information about the alarm.
* @returns {void}
*/
const alarm = (() => {
const short_id = this.adapt_id_(id);
if ( this.alarms[id] ) {
existing = true;
return this.alarms[id];
}
const alarm = this.alarms[id] = this.alarm_aliases[short_id] = {
id,
short_id,
started: Date.now(),
occurrences: [],
};
Object.defineProperty(alarm, 'count', {
/**
* Method to create a new alarm.
*
* This method takes an id, message, and optional fields as parameters.
* It creates a new alarm object with the provided id and message,
* and adds it to the alarms object. It also keeps track of the number of occurrences of the alarm.
* If the alarm already exists, it increments the occurrence count and calls the handle\_alarm\_repeat\_ method.
* If it's a new alarm, it calls the handle\_alarm\_on\_ method.
*
* @param {string} id - The unique identifier for the alarm.
* @param {string} message - The message associated with the alarm.
* @param {object} [fields] - Optional fields associated with the alarm.
* @returns {void}
*/
get () {
return alarm.timestamps?.length ?? 0;
},
});
Object.defineProperty(alarm, 'id_string', {
/**
* Method to handle creating a new alarm with given parameters.
* This method adds the alarm to the `alarms` object, updates the occurrences count,
* and processes any known errors that may apply to the alarm.
* @param {string} id - The unique identifier for the alarm.
* @param {string} message - The message associated with the alarm.
* @param {Object} fields - Additional fields to associate with the alarm.
*/
get () {
if ( alarm.id.length < 20 ) {
return alarm.id;
}
const truncatedLongId = `${alarm.id.slice(0, 20) }...`;
return `${alarm.short_id} (${truncatedLongId})`;
},
});
return alarm;
})();
const occurance = {
message,
fields,
timestamp: Date.now(),
};
// Keep logs from the previous occurrence if:
// - it's one of the first 3 occurrences
// - the 10th, 100th, 1000th...etc occurrence
if ( alarm.count > 3 && Math.log10(alarm.count) % 1 !== 0 ) {
delete alarm.occurrences[alarm.occurrences.length - 1].logs;
}
occurance.logs = this.log.get_log_buffer();
alarm.message = message;
alarm.fields = { ...alarm.fields, ...fields };
alarm.timestamps = (alarm.timestamps ?? []).concat(Date.now());
alarm.occurrences.push(occurance);
if ( fields?.error ) {
alarm.error = fields.error;
}
if ( alarm.source ) {
console.error(alarm.error);
}
if ( existing ) {
this.handle_alarm_repeat_(alarm);
} else {
this.handle_alarm_on_(alarm);
}
}
/**
* Method to clear an alarm with the given ID.
* @param {*} id - The ID of the alarm to clear.
* @returns {void}
*/
clear (id) {
const alarm = this.alarms[id];
if ( ! alarm ) {
return;
}
delete this.alarms[id];
this.handle_alarm_off_(alarm);
}
apply_known_errors_ (alarm) {
const rule_matches = rule => {
const match = rule.match;
if ( match.id !== alarm.id ) return false;
if ( match.message && match.message !== alarm.message ) return false;
if ( match.fields ) {
for ( const [key, value] of Object.entries(match.fields) ) {
if ( alarm.fields[key] !== value ) return false;
}
}
return true;
};
const rule_actions = {
'no-alert': () => alarm.no_alert = true,
'severity': action => alarm.severity = action.value,
};
const apply_action = action => {
rule_actions[action.type](action);
};
for ( const rule of this.known_errors ) {
if ( rule_matches(rule) ) apply_action(rule.action);
}
}
handle_alarm_repeat_ (alarm) {
this.log.warn(`REPEAT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
alarm.fields);
this.apply_known_errors_(alarm);
if ( alarm.no_alert ) return;
const severity = alarm.severity ?? 'critical';
const fields_clean = {};
for ( const [key, value] of Object.entries(alarm.fields) ) {
fields_clean[key] = util.inspect(value);
}
this.pager.alert({
id: alarm.id ?? 'something-bad',
message: alarm.message ?? alarm.id ?? 'something bad happened',
source: 'alarm-service',
severity,
custom: {
fields: fields_clean,
trace: alarm.error?.stack,
repeat_count: alarm.count,
},
});
}
handle_alarm_on_ (alarm) {
this.log.error(`ACTIVE ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
alarm.fields);
this.apply_known_errors_(alarm);
if ( this.global_config.env === 'dev' && !this.attached_dev ) {
this.attached_dev = true;
const realConsole = globalThis.original_console_object ?? console;
realConsole.error('\x1B[33;1m[alarm]\x1B[0m Active alarms detected; see logs for details.');
}
const args = this.Context.get('args') ?? {};
if ( args['quit-on-alarm'] ) {
const svc_shutdown = this.services.get('shutdown');
svc_shutdown.shutdown({
reason: '--quit-on-alarm is set',
code: 1,
});
}
if ( alarm.no_alert ) return;
const severity = alarm.severity ?? 'critical';
const fields_clean = {};
for ( const [key, value] of Object.entries(alarm.fields) ) {
fields_clean[key] = util.inspect(value);
}
this.pager.alert({
id: alarm.id ?? 'something-bad',
message: alarm.message ?? alarm.id ?? 'something bad happened',
source: 'alarm-service',
severity,
custom: {
fields: fields_clean,
trace: alarm.error?.stack,
},
});
// Write a .log file for the alert that happened
try {
const lines = [];
lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`);
lines.push(`started: ${new Date(alarm.started).toISOString()}`);
lines.push(`short id: ${alarm.short_id}`);
lines.push(`original id: ${alarm.id}`);
lines.push(`severity: ${severity}`);
lines.push(`message: ${alarm.message}`);
lines.push(`fields: ${JSON.stringify(fields_clean)}`);
const alert_info = lines.join('\n');
(async () => {
try {
fs.appendFileSync(`alert_${alarm.id}.log`, `${alert_info }\n`);
} catch (e) {
this.log.error(`failed to write alert log: ${e.message}`);
}
})();
} catch (e) {
this.log.error(`failed to write alert log: ${e.message}`);
}
}
handle_alarm_off_ (alarm) {
this.log.info(`CLEAR ${alarm.id} :: ${alarm.message} (${alarm.count})`,
alarm.fields);
}
/**
* Method to get an alarm by its ID.
*
* @param {*} id - The ID of the alarm to get.
* @returns
*/
get_alarm (id) {
return this.alarms[id] ?? this.alarm_aliases[id];
}
_register_commands (commands) {
// Function to handle a specific alarm event.
// This comment can be added above line 320.
// This function is responsible for processing specific events related to alarms.
// It can be used for tasks such as updating alarm status, sending notifications, or triggering actions.
// This function is called internally by the AlarmService class.
// /*
// * handleAlarmEvent - Handles a specific alarm event.
// *
// * @param {Object} alarm - The alarm object containing relevant information.
// * @param {Function} callback - Optional callback function to be called when the event is handled.
// */
// function handleAlarmEvent(alarm, callback) {
// // Implementation goes here.
// }
const completeAlarmID = (args) => {
// The alarm ID is the first argument, so return no results if we're on the second or later.
if ( args.length > 1 )
{
return;
}
const lastArg = args[args.length - 1];
const results = [];
for ( const alarm of Object.values(this.alarms) ) {
if ( alarm.id.startsWith(lastArg) ) {
results.push(alarm.id);
}
if ( alarm.short_id?.startsWith(lastArg) ) {
results.push(alarm.short_id);
}
}
return results;
};
commands.registerCommands('alarm', [
{
id: 'list',
description: 'list alarms',
handler: async (args, log) => {
for ( const alarm of Object.values(this.alarms) ) {
log.log(`${alarm.id_string}: ${alarm.message} (${alarm.count})`);
}
},
},
{
id: 'info',
description: 'show info about an alarm',
handler: async (args, log) => {
const [id] = args;
const alarm = this.get_alarm(id);
if ( ! alarm ) {
log.log(`no alarm with id ${id}`);
return;
}
log.log(`\x1B[33;1m${alarm.id_string}\x1B[0m :: ${alarm.message} (${alarm.count})`);
log.log(`started: ${new Date(alarm.started).toISOString()}`);
log.log(`short id: ${alarm.short_id}`);
log.log(`original id: ${alarm.id}`);
// print stack trace of alarm error
if ( alarm.error ) {
log.log(alarm.error.stack);
}
// print other fields
for ( const [key, value] of Object.entries(alarm.fields) ) {
log.log(`- ${key}: ${util.inspect(value)}`);
}
},
completer: completeAlarmID,
},
{
id: 'clear',
description: 'clear an alarm',
handler: async (args, log) => {
const [id] = args;
const alarm = this.get_alarm(id);
if ( ! alarm ) {
log.log(`no alarm with id ${id}; ` +
`but calling clear(${JSON.stringify(id)}) anyway.`);
}
this.clear(id);
},
completer: completeAlarmID,
},
{
id: 'clear-all',
description: 'clear all alarms',
handler: async (_args, _log) => {
const alarms = Object.values(this.alarms);
this.alarms = {};
for ( const alarm of alarms ) {
this.handle_alarm_off_(alarm);
}
},
},
{
id: 'sound',
description: 'sound an alarm',
handler: async (args, _log) => {
const [id, message] = args;
this.create(id ?? 'test', message, {});
},
},
{
id: 'inspect',
description: 'show logs that happened an alarm',
handler: async (args, log) => {
const [id, occurance_idx] = args;
const alarm = this.get_alarm(id);
if ( ! alarm ) {
log.log(`no alarm with id ${id}`);
return;
}
const occurance = alarm.occurrences[occurance_idx];
if ( ! occurance ) {
log.log(`no occurance with index ${occurance_idx}`);
return;
}
log.log(`┏━━ Logs before: ${alarm.id_string} ━━━━`);
for ( const lg of occurance.logs ) {
log.log(`┃ ${ this.logutil.stringify_log_entry(lg)}`);
}
log.log(`┗━━ Logs before: ${alarm.id_string} ━━━━`);
},
completer: completeAlarmID,
},
]);
}
}
module.exports = {
AlarmService,
};
================================================
FILE: src/backend/src/modules/core/ContextService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves when the initialization is complete.
*/
async init () {
const services = this.services;
this.alarm = services.get('alarm');
this.backupLogger = services.get('log-service').create('error-service');
}
/**
* Creates an ErrorContext instance with the provided logging context.
*
* @param {*} log_context The logging context to associate with the error reports.
* @returns {ErrorContext} An ErrorContext instance.
*/
create (log_context) {
return new ErrorContext(this, log_context);
}
/**
* Reports an error with the specified location and details.
* The "location" is a string up to the callers discretion to identify
* the source of the error.
*
* @param {*} location The location where the error occurred.
* @param {*} fields The error details to report.
* @param {boolean} [alarm=true] Whether to raise an alarm for the error.
* @returns {void}
*/
report (location, { source, logger, trace, extra, message }, alarm = true) {
message = message ?? source?.message;
logger = logger ?? this.backupLogger;
logger.error(`Error @ ${location}: ${message}; ${ source?.stack}`);
if ( alarm ) {
const alarm_id = `${location}:${message}`;
this.alarm.create(alarm_id, message, {
error: source,
...extra,
});
}
}
}
module.exports = { ErrorService };
================================================
FILE: src/backend/src/modules/core/ExpectationService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves when initialization is complete.
*/
async _init () {
// TODO: service to track all interval functions?
/**
* Initializes the service by setting up interval functions and registering commands.
* This method sets up a periodic interval function to purge expectations and registers
* a command to list pending expectations.
*
* @returns {void}
*/
// The comment should be placed above the method at line 68
setInterval(() => {
this.purgeExpectations_();
}, 1000);
}
/**
* Purges expectations that have been met.
*
* This method iterates through the list of expectations and removes
* those that have been satisfied. Currently, this functionality is
* disabled and needs to be re-enabled.
*
* @returns {void} This method does not return anything.
*/
purgeExpectations_ () {
return;
// TODO: Re-enable this
// for ( let i=0 ; i < this.expectations_.length ; i++ ) {
// if ( this.expectations_[i].check() ) {
// this.expectations_[i] = null;
// }
// }
// this.expectations_ = this.expectations_.filter(v => v !== null);
}
/**
* Registers an expectation to be tracked by the service.
*
* @param {Object} workUnit - The work unit to track
* @param {string} checkpoint - The checkpoint to expect
* @returns {void}
*/
expect_eventually ({ workUnit, checkpoint }) {
this.expectations_.push(new this.expect.CheckpointExpectation(workUnit, checkpoint));
}
}
module.exports = {
ExpectationService,
};
================================================
FILE: src/backend/src/modules/core/LogService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
async _init () {
this.alertHandlers_ = [];
if ( ! this.config ) {
return;
}
this.onInit();
}
/**
* Initializes PagerDuty configuration and registers alert handlers.
* If PagerDuty is enabled in the configuration, it sets up an alert handler
* to send alerts to PagerDuty.
*
* @method onInit
*/
onInit () {
if ( this.config.pagerduty && this.config.pagerduty.enabled ) {
this.alertHandlers_.push(async alert => {
const event = pdjs.event;
const fields_clean = {};
for ( const [key, value] of Object.entries(alert?.fields ?? {}) ) {
fields_clean[key] = util.inspect(value);
}
const custom_details = {
...(alert.custom || {}),
server_id: this.global_config.server_id,
};
const ctx = this.Context.get(undefined, { allow_fallback: true });
// Add request payload if any exists
const req = ctx.get('req');
if ( req ) {
if ( req.body ) {
// Remove fields which may contain sensitive information
delete req.body.password;
delete req.body.email;
// Add the request body to the custom details
custom_details.request_body = req.body;
}
}
this.log.info('it is sending to PD');
await event({
data: {
routing_key: this.config.pagerduty.routing_key,
event_action: 'trigger',
dedup_key: alert.id,
payload: {
summary: alert.message,
source: alert.source,
severity: alert.severity,
custom_details,
},
},
});
});
}
}
/**
* Sends an alert to all registered alert handlers.
*
* This method iterates through all alert handlers and attempts to send the alert.
* If any handler fails to send the alert, an error message is logged.
*
* @param {Object} alert - The alert object containing details about the alert.
*/
async alert (alert) {
for ( const handler of this.alertHandlers_ ) {
try {
await handler(alert);
} catch (e) {
this.log.error(`failed to send pager alert: ${e?.message}`);
}
}
}
_register_commands (commands) {
commands.registerCommands('pager', [
{
id: 'test-alert',
description: 'create a test alert',
handler: async (args, log) => {
const [severity] = args;
await this.alert({
id: 'test-alert',
message: 'test alert',
source: 'test',
severity,
});
},
},
]);
}
}
module.exports = {
PagerService,
};
================================================
FILE: src/backend/src/modules/core/ParameterService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } */
this.parameters_ = [];
}
/**
* Initializes the service by registering commands with the command service.
* This method is called during service startup to set up command handlers
* for parameter management.
* @private
*/
'__on_boot.consolidation' () {
this._registerCommands(this.services.get('commands'));
}
createParameters (serviceName, parameters, opt_instance) {
for ( const parameter of parameters ) {
this.log.debug(`registering parameter ${serviceName}:${parameter.id}`);
this.parameters_.push(new Parameter({
...parameter,
id: `${serviceName}:${parameter.id}`,
}));
if ( opt_instance ) {
this.bindToInstance(`${serviceName}:${parameter.id}`,
opt_instance,
parameter.id);
}
}
}
/**
* Gets the value of a parameter by its ID
* @param {string} id - The unique identifier of the parameter to retrieve
* @returns {Promise<*>} The current value of the parameter
* @throws {Error} If parameter with given ID is not found
*/
async get (id) {
const parameter = this._get_param(id);
return await parameter.get();
}
bindToInstance (id, instance, name) {
const parameter = this._get_param(id);
return parameter.bindToInstance(instance, name);
}
subscribe (id, listener) {
const parameter = this._get_param(id);
return parameter.subscribe(listener);
}
_get_param (id) {
const parameter = this.parameters_.find(p => p.spec_.id === id);
if ( ! parameter ) {
throw new Error(`unknown parameter: ${id}`);
}
return parameter;
}
/**
* Registers parameter-related commands with the command service
* @param {Object} commands - The command service instance to register with
*/
_registerCommands (commands) {
const completeParameterName = (args) => {
// The parameter name is the first argument, so return no results if we're on the second or later.
if ( args.length > 1 )
{
return;
}
const lastArg = args[args.length - 1];
return this.parameters_
.map(parameter => parameter.spec_.id)
.filter(parameterName => parameterName.startsWith(lastArg));
};
commands.registerCommands('params', [
{
id: 'get',
description: 'get a parameter',
handler: async (args, log) => {
const [name] = args;
const value = await this.get(name);
log.log(value);
},
completer: completeParameterName,
},
{
id: 'set',
description: 'set a parameter',
handler: async (args, log) => {
const [name, value] = args;
const parameter = this._get_param(name);
parameter.set(value);
log.log(value);
},
completer: completeParameterName,
},
{
id: 'list',
description: 'list parameters',
handler: async (args, log) => {
const [prefix] = args;
let parameters = this.parameters_;
if ( prefix ) {
parameters = parameters
.filter(p => p.spec_.id.startsWith(prefix));
}
log.log(`available parameters${
prefix ? ` (starting with: ${prefix})` : ''
}:`);
for ( const parameter of parameters ) {
// log.log(`- ${parameter.spec_.id}: ${parameter.spec_.description}`);
// Log parameter description and value
const value = await parameter.get();
log.log(`- ${parameter.spec_.id} = ${value}`);
log.log(` ${parameter.spec_.description}`);
}
},
},
]);
}
}
module.exports = {
ParameterService,
};
================================================
FILE: src/backend/src/modules/core/ProcessEventService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
await this.Context.allow_fallback(async () => {
errors.report('process:uncaughtException', {
source: err,
origin,
trace: true,
alarm: true,
});
});
});
process.on('unhandledRejection', async (reason, promise) => {
/**
* Handles unhandled promise rejections by reporting them to the error service
* @param {*} reason - The rejection reason/error
* @param {Promise} promise - The rejected promise
* @returns {Promise} Resolves when error is reported
*/
await this.Context.allow_fallback(async () => {
errors.report('process:unhandledRejection', {
source: reason,
promise,
trace: true,
alarm: true,
});
});
});
}
}
module.exports = {
ProcessEventService,
};
================================================
FILE: src/backend/src/modules/core/README.md
================================================
# Core2Module
A replacement for CoreModule with as few external relative requires as possible.
This will eventually be the successor to CoreModule, the main module for Puter's backend.
## Services
### AlarmService
AlarmService class is responsible for managing alarms.
It provides methods for creating, clearing, and handling alarms.
#### Listeners
##### `boot.consolidation`
AlarmService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `create`
Method to create an alarm with the given ID, message, and fields.
If the ID already exists, it will be updated with the new fields
and the occurrence count will be incremented.
###### Parameters
- **id:** Unique identifier for the alarm.
- **message:** Message associated with the alarm.
- **fields:** Additional information about the alarm.
##### `clear`
Method to clear an alarm with the given ID.
###### Parameters
- **id:** The ID of the alarm to clear.
##### `get_alarm`
Method to get an alarm by its ID.
###### Parameters
- **id:** The ID of the alarm to get.
### ErrorService
The ErrorService class is responsible for handling and reporting errors within the system.
It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms.
#### Methods
##### `init`
Initializes the ErrorService, setting up the alarm and backup logger services.
##### `create`
Creates an ErrorContext instance with the provided logging context.
###### Parameters
- **log_context:** The logging context to associate with the error reports.
##### `report`
Reports an error with the specified location and details.
The "location" is a string up to the callers discretion to identify
the source of the error.
###### Parameters
- **location:** The location where the error occurred.
- **fields:** The error details to report.
### ExpectationService
#### Listeners
##### `boot.consolidation`
ExpectationService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `expect_eventually`
Registers an expectation to be tracked by the service.
###### Parameters
- **workUnit:** The work unit to track
- **checkpoint:** The checkpoint to expect
### LogService
The `LogService` class extends `BaseService` and is responsible for managing and
orchestrating various logging functionalities within the application. It handles
log initialization, middleware registration, log directory management, and
provides methods for creating log contexts and managing log output levels.
#### Listeners
##### `boot.consolidation`
Registers logging commands with the command service.
#### Methods
##### `register_log_middleware`
Registers a custom logging middleware with the LogService.
###### Parameters
- **callback:** The callback function that modifies log parameters before delegation.
##### `create`
Create a new log context with the specified prefix
###### Parameters
- **prefix:** The prefix for the log context
- **fields:** Optional fields to include in the log context
##### `get_log_file`
Generates a sanitized file path for log files.
###### Parameters
- **name:** The name of the log file, which will be sanitized to remove any path characters.
##### `get_log_buffer`
Get the most recent log entries from the buffer maintained by the LogService.
By default, the buffer contains the last 20 log entries.
### PagerService
#### Listeners
##### `boot.consolidation`
PagerService registers its commands at the consolidation phase because
the '_init' method of CommandService may not have been called yet.
#### Methods
##### `onInit`
Initializes PagerDuty configuration and registers alert handlers.
If PagerDuty is enabled in the configuration, it sets up an alert handler
to send alerts to PagerDuty.
##### `alert`
Sends an alert to all registered alert handlers.
This method iterates through all alert handlers and attempts to send the alert.
If any handler fails to send the alert, an error message is logged.
###### Parameters
- **alert:** The alert object containing details about the alert.
### ProcessEventService
Service class that handles process-wide events and errors.
Provides centralized error handling for uncaught exceptions and unhandled promise rejections.
Sets up event listeners on the process object to capture and report critical errors
through the logging and error reporting services.
## Libraries
### core.expect
### core.util.identutil
#### Functions
##### `randomItem`
Select a random item from an array using a random number generator function.
###### Parameters
- **arr:** The array to select an item from
### core.util.logutil
#### Functions
##### `stringify_log_entry`
Stringifies a log entry into a formatted string for console output.
###### Parameters
- **logEntry:** The log entry object containing:
### stdio
#### Functions
##### `visible_length`
METADATA // {"ai-commented":{"service":"claude"}}
##### `split_lines`
Split a string into lines according to the terminal width,
preserving ANSI escape sequences, and return an array of lines.
###### Parameters
- **str:** The string to split into lines
### core.util.strutil
#### Functions
##### `quot`
METADATA // {"def":"core.util.strutil","ai-params":{"service":"claude"},"ai-commented":{"service":"claude"}}
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../services/BaseService.js`
- `../../util/context.js`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../util/context`
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
- `../../services/BaseService` (use.BaseService)
================================================
FILE: src/backend/src/modules/core/ServerHealthService/ServerHealthRedisCacheKeys.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }: An array of names of failed health checks, if any.
*/
async get_status () {
const cacheKey = ServerHealthRedisCacheKeys.status;
// Check cache first
const cached = await redisClient.get(cacheKey);
if ( cached ) {
try {
return JSON.parse(cached);
} catch (e) {
// no op cache is in an invalid state
}
}
// Compute status
const failures = this.failures_.map(v => v.name);
const status = {
ok: failures.length === 0,
...(failures.length ? { failed: failures } : {}),
};
// Cache with 5 second TTL
await setRedisCacheValue(cacheKey, JSON.stringify(status), {
ttlSeconds: 5,
eventData: status,
});
return status;
}
}
module.exports = { ServerHealthService };
================================================
FILE: src/backend/src/modules/core/lib/__lib__.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } arr - The array to select an item from
* @param {function} [random=Math.random] - Random number generator function
* @returns {T} A random item from the array
*/
const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)];
/**
* A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999).
* The result is returned as a string with components separated by the specified separator.
* It is useful when you need to create unique identifiers that are also human-friendly.
*
* @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided.
* @param {function} [rng=Math.random] - Random number generator function
* @returns {string} A unique, human-friendly identifier.
*
* @example
*
* let identifier = window.generate_identifier();
* // identifier would be something like 'clever-idea-123'
*
*/
function generate_identifier (separator = '_', rng = Math.random) {
// return a random combination of first_adj + noun + number (between 0 and 9999)
// e.g. clever-idea-123
return [
randomItem(adjectives, rng),
randomItem(nouns, rng),
Math.floor(rng() * 10000),
].join(separator);
}
// Character set used for generating human-readable, case-insensitive random codes
const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
function generate_random_code (n, {
rng = Math.random,
chars = HUMAN_READABLE_CASE_INSENSITIVE,
} = {}) {
let code = '';
for ( let i = 0 ; i < n ; i++ ) {
code += randomItem(chars, rng);
}
return code;
}
/**
* Composes a code by combining a mask string with a base-36 converted number
* @param {string} mask - Initial string template to use as base
* @param {number} value - Number to convert to base-36 and append to the right
* @returns {string} Combined uppercase code
*/
function compose_code (mask, value) {
const right_str = value.toString(36);
let out_str = mask;
console.log('right_str', right_str);
console.log('out_str', out_str);
for ( let i = 0 ; i < right_str.length ; i++ ) {
out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i];
}
out_str = out_str.toUpperCase();
return out_str;
}
module.exports = {
randomItem,
generate_identifier,
generate_random_code,
};
================================================
FILE: src/backend/src/modules/core/lib/linux.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see : to fs::
svc_permission.register_rewriter(PermissionRewriter.create({
matcher: permission => permission.startsWith('app-root-dir:'),
rewriter: async permission => {
const context = Context.get();
// Only "AppUnderUser" scope is allowed to have this permission rewritten to
// an actual filesystem permission - this is because apps will still be limited
// baesd on a user's own access.
const actor = context.get('actor');
if ( ! Context.get('is_grant_user_app_permission') ) {
return PERMISSION_FOR_NOTHING_IN_PARTICULAR;
}
const parts = PermissionUtil.split(permission);
if ( parts.length < 3 ) {
throw APIError.create('field_invalid', null, { key: 'permission', got: permission });
}
// <>::
const target_app_uid = parts[1];
const access = parts[2];
if ( ! target_app_uid ) {
throw APIError.create('field_invalid', null, { key: 'target_app_uid', got: target_app_uid });
}
if ( ! (actor.type instanceof UserActorType) ) {
throw APIError.create('forbidden');
}
const target_app = await get_app({ uid: target_app_uid });
if ( ! target_app ) {
throw APIError.create('entity_not_found', null, { identifier: `app:${target_app_uid}` });
}
if ( target_app.owner_user_id !== actor.type.user.id ) {
throw APIError.create('forbidden');
}
const root_dir_id = await svc_app.getAppRootDirId(target_app);
const svc_fs = context.get('services').get('filesystem');
const node = await svc_fs.node(new NodeInternalIDSelector('mysql', root_dir_id));
await node.fetchEntry();
if ( ! node.found ) throw APIError.create('subject_does_not_exist');
const node_uid = await node.get('uid');
return PermissionUtil.join('fs', node_uid, access);
},
}));
}
static PROTECTED_FIELDS = ['last_review'];
static READ_ONLY_FIELDS = [
'approved_for_listing',
'approved_for_opening_items',
'approved_for_incentive_program',
'godmode',
'is_private',
];
static WRITE_ALL_OWNER_PERMISSION = 'system:es:write-all-owners';
static IMPLEMENTS = {
'crud-q': {
async create ({ object, options }) {
return await this.#create({ object, options });
},
async update ({ object, id, options }) {
return await this.#update({ object, id, options });
},
async upsert ({ object, id, options }) {
// Try to find an existing entity
let existing = null;
if ( object.uid !== undefined || id !== undefined ) {
try {
existing = await this.#read({
uid: object.uid,
id,
});
} catch ( error ) {
// If entity not found, we'll create it
if ( error.fields?.code !== 'entity_not_found' ) {
throw error;
}
}
}
if ( existing ) {
// Entity exists, call update
return await this.#update({ object, id, options });
} else {
// Entity doesn't exist, call create
return await this.#create({ object, options });
}
},
async read ({ uid, id, params = {} }) {
return this.#read({ uid, id, params });
},
async select (options) {
return this.#select(options);
},
async delete ({ uid, id }) {
return await this.#delete({ uid, id });
},
},
};
// value of require('om/mappings/app.js').redundant_identifiers
static REDUNDANT_IDENTIFIERS = ['name'];
async #select ({ predicate, params, ..._rest }) {
const db = this.db;
if ( predicate === undefined ) predicate = [];
if ( params === undefined ) params = {};
if ( ! Array.isArray(predicate) ) throw new Error('predicate must be an array');
const userCanEditOnly = Array.prototype.includes.call(predicate, 'user-can-edit');
const stmt = 'SELECT apps.*, ' +
'CASE WHEN apps.icon LIKE \'data:%\' THEN 1 ELSE 0 END AS icon_is_base64, ' +
'owner_user.username AS owner_user_username, ' +
'owner_user.uuid AS owner_user_uuid, ' +
'app_owner.uid AS app_owner_uid ' +
'FROM apps ' +
'LEFT JOIN user owner_user ON apps.owner_user_id = owner_user.id ' +
'LEFT JOIN apps app_owner ON apps.app_owner = app_owner.id ' +
`${userCanEditOnly ? 'WHERE apps.owner_user_id=?' : ''} ` +
'LIMIT 5000';
const values = userCanEditOnly ? [Context.get('user').id] : [];
const rows = await db.read(stmt, values);
const shouldFetchFiletypes = rows.some(row => typeof row.filetypes !== 'string');
const filetypesByAppId = shouldFetchFiletypes
? await this.#getFiletypeAssociationsByAppIds(rows.map(row => row.id))
: new Map();
const iconSize = params.icon_size;
const shouldResolveIconPath = Boolean(iconSize)
|| rows.some(row => isStoredBase64AppIcon(row));
const svc_appIcon = shouldResolveIconPath
? this.context.get('services').get('app-icon')
: null;
const svc_error = shouldResolveIconPath
? this.context.get('services').get('error-service')
: null;
const appAndOwnerIds = [];
for ( const row of rows ) {
const app = {};
// FROM ROW
app.approved_for_incentive_program = as_bool(row.approved_for_incentive_program);
app.approved_for_listing = as_bool(row.approved_for_listing);
app.approved_for_opening_items = as_bool(row.approved_for_opening_items);
app.background = as_bool(row.background);
app.created_at = row.created_at;
app.created_from_origin = row.created_from_origin;
app.description = row.description;
app.godmode = as_bool(row.godmode);
app.icon = row.icon;
app.is_private = as_bool(row.is_private);
app.index_url = row.index_url;
app.maximize_on_start = as_bool(row.maximize_on_start);
app.metadata = row.metadata;
app.name = row.name;
app.protected = as_bool(row.protected);
app.stats = row.stats;
app.title = row.title;
app.uid = row.uid;
// REQURIES OTHER DATA
// app.app_owner;
// app.filetype_associations = row.filetype_associations;
// app.owner = row.owner;
app.app_owner = {
uid: row.app_owner_uid,
};
{
const owner_user = extract_from_prefix(row, 'owner_user_');
app.owner = user_to_client(owner_user);
}
try {
if ( typeof row.filetypes === 'string' ) {
app.filetype_associations = this.#parseFiletypeAssociationsJson(row.filetypes);
} else {
app.filetype_associations = this.#normalizeFiletypeAssociations(filetypesByAppId.get(row.id) ?? []);
}
} catch (e) {
throw new Error(`failed to get app filetype associations: ${e.message}`, { cause: e });
}
// REFINED BY OTHER DATA
// app.icon;
if ( svc_appIcon && (iconSize || isStoredBase64AppIcon(row)) ) {
try {
const iconPath = svc_appIcon.getAppIconPath({
appUid: row.uid,
size: iconSize,
});
if ( iconPath ) {
app.icon = iconPath;
}
} catch (e) {
svc_error?.report('AppES:read_transform', { source: e });
}
}
appAndOwnerIds.push({
app,
ownerUserId: row.owner_user_id,
});
}
// Check protected app access in parallel for faster large selections.
const allowed_apps = await Promise.all(appAndOwnerIds.map(async ({ app, ownerUserId }) => {
if ( await this.#check_protected_app_access(app, ownerUserId) ) {
return null;
}
return app;
}));
return allowed_apps.filter(Boolean);
}
async #read ({ uid, id, params = {}, backend_only_options = {} }) {
const db = this.db;
if ( uid === undefined && id === undefined ) {
throw new Error('read requires either uid or id');
}
// Build WHERE clause based on identifier type
let whereClause;
let whereValues;
let canonicalUidAliasPromise = null;
if ( uid !== undefined ) {
// Simple uid lookup
whereClause = 'apps.uid = ?';
whereValues = [uid];
canonicalUidAliasPromise = this.#readCanonicalAppUidAlias(uid);
} else if ( id !== null && typeof id === 'object' && !Array.isArray(id) ) {
// Complex id lookup (e.g., { name: 'editor' })
const { clause, values } = this.#build_complex_id_where(id);
whereClause = clause;
whereValues = values;
} else {
throw APIError.create('invalid_id', null, { id });
}
const stmt = 'SELECT apps.*, ' +
'CASE WHEN apps.icon LIKE \'data:%\' THEN 1 ELSE 0 END AS icon_is_base64, ' +
'owner_user.username AS owner_user_username, ' +
'owner_user.uuid AS owner_user_uuid, ' +
'app_owner.uid AS app_owner_uid ' +
'FROM apps ' +
'LEFT JOIN user owner_user ON apps.owner_user_id = owner_user.id ' +
'LEFT JOIN apps app_owner ON apps.app_owner = app_owner.id ' +
`WHERE ${whereClause} ` +
'LIMIT 1';
let rows = await db.read(stmt, whereValues);
if ( rows.length === 0 && canonicalUidAliasPromise ) {
const canonicalUid = await canonicalUidAliasPromise;
if (
typeof canonicalUid === 'string'
&& canonicalUid
&& canonicalUid !== uid
) {
rows = await db.read(stmt, [canonicalUid]);
}
}
if ( rows.length === 0 ) {
throw APIError.create('entity_not_found', null, {
identifier: uid || JSON.stringify(id),
});
}
const row = rows[0];
const app = {};
app.approved_for_incentive_program = as_bool(row.approved_for_incentive_program);
app.approved_for_listing = as_bool(row.approved_for_listing);
app.approved_for_opening_items = as_bool(row.approved_for_opening_items);
app.background = as_bool(row.background);
app.created_at = row.created_at;
app.created_from_origin = row.created_from_origin;
app.description = row.description;
app.godmode = as_bool(row.godmode);
app.icon = row.icon;
app.is_private = as_bool(row.is_private);
app.index_url = row.index_url;
app.maximize_on_start = as_bool(row.maximize_on_start);
app.metadata = row.metadata;
app.name = row.name;
app.protected = as_bool(row.protected);
app.stats = row.stats;
app.title = row.title;
app.uid = row.uid;
app.app_owner = {
uid: row.app_owner_uid,
};
{
const owner_user = extract_from_prefix(row, 'owner_user_');
if ( backend_only_options.no_filter_owner ) app.owner = owner_user;
else app.owner = user_to_client(owner_user);
}
let protectedAccessPromise;
try {
if ( typeof row.filetypes === 'string' ) {
app.filetype_associations = this.#parseFiletypeAssociationsJson(row.filetypes);
} else {
protectedAccessPromise = this.#check_protected_app_access(app, row.owner_user_id);
const filetypeAssociations = await this.#getFiletypeAssociationsByAppId(row.id);
app.filetype_associations = this.#normalizeFiletypeAssociations(filetypeAssociations);
}
} catch (e) {
throw new Error(`failed to get app filetype associations: ${e.message}`, { cause: e });
}
// Check protected app access as soon as dependent fields are resolved.
if ( ! protectedAccessPromise ) {
protectedAccessPromise = this.#check_protected_app_access(app, row.owner_user_id);
}
if ( await protectedAccessPromise ) {
// App should not be accessible
throw APIError.create('entity_not_found', null, {
identifier: uid || JSON.stringify(id),
});
}
const iconSize = params.icon_size;
if ( iconSize || isStoredBase64AppIcon(row) ) {
const svc_appIcon = this.context.get('services').get('app-icon');
if ( svc_appIcon ) {
try {
const iconPath = svc_appIcon.getAppIconPath({
appUid: row.uid,
size: iconSize,
});
if ( iconPath ) {
app.icon = iconPath;
}
} catch (e) {
const svc_error = this.context.get('services').get('error-service');
svc_error.report('AppES:read_transform', { source: e });
}
}
}
return app;
}
#parseFiletypeAssociationsJson (filetypes) {
return this.#normalizeFiletypeAssociations(JSON.parse(filetypes));
}
async #getFiletypeAssociationsByAppId (appId) {
if ( appId === undefined || appId === null ) return [];
const rows = await this.db.read(
'SELECT type FROM app_filetype_association WHERE app_id = ?',
[appId],
);
return rows
.map(row => row.type)
.filter(type => typeof type === 'string' || type === null);
}
#normalizeFiletypeAssociations (filetypesAsJSON) {
filetypesAsJSON = Array.isArray(filetypesAsJSON)
? filetypesAsJSON
: [];
filetypesAsJSON = filetypesAsJSON.filter(ft => ft !== null);
for ( let i = 0 ; i < filetypesAsJSON.length ; i++ ) {
if ( typeof filetypesAsJSON[i] !== 'string' ) {
throw new Error(`expected filetypesAsJSON[${i}] to be a string, got: ${filetypesAsJSON[i]}`);
}
if ( String.prototype.startsWith.call(filetypesAsJSON[i], '.') ) {
filetypesAsJSON[i] = filetypesAsJSON[i].slice(1);
}
}
return filetypesAsJSON;
}
async #getFiletypeAssociationsByAppIds (appIds) {
appIds = [...new Set(appIds.filter(appId => appId !== undefined && appId !== null))];
if ( appIds.length === 0 ) return new Map();
const filetypesByAppId = new Map();
for ( const appId of appIds ) {
filetypesByAppId.set(appId, []);
}
// SQLite has a low bind-parameter limit; chunk to avoid oversized IN lists.
const chunkSize = 500;
for ( let i = 0 ; i < appIds.length ; i += chunkSize ) {
const chunk = appIds.slice(i, i + chunkSize);
const placeholders = chunk.map(() => '?').join(', ');
const rows = await this.db.read(
`SELECT app_id, type FROM app_filetype_association WHERE app_id IN (${placeholders})`,
chunk,
);
for ( const row of rows ) {
if ( ! filetypesByAppId.has(row.app_id) ) {
filetypesByAppId.set(row.app_id, []);
}
filetypesByAppId.get(row.app_id).push(row.type);
}
}
return filetypesByAppId;
}
async #create ({ object, options }) {
// Only UserActorType and AppUnderUserActorType are allowed to do this
const actor = Context.get('actor');
if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) {
throw APIError.create('forbidden');
}
const user = actor.type.user;
// Remove protected/read_only fields from the input (ValidationES behavior)
{
object = { ...object };
for ( const field of this.constructor.PROTECTED_FIELDS ) {
delete object[field];
}
for ( const field of this.constructor.READ_ONLY_FIELDS ) {
delete object[field];
}
}
// Validate required fields
{
if ( object.name === undefined ) {
throw APIError.create('field_missing', null, { key: 'name' });
}
if ( object.title === undefined ) {
throw APIError.create('field_missing', null, { key: 'title' });
}
if ( object.index_url === undefined ) {
throw APIError.create('field_missing', null, { key: 'index_url' });
}
}
// Validate fields
{
validate_string(object.name, {
key: 'name',
maxlen: config.app_name_max_length,
regex: config.app_name_regex,
});
validate_string(object.title, {
key: 'title',
maxlen: config.app_title_max_length,
});
if ( object.description !== undefined && object.description !== null ) {
validate_string(object.description, {
key: 'description',
maxlen: 7000,
});
}
if ( object.icon !== undefined && object.icon !== null ) {
if ( typeof object.icon === 'string' ) {
object.icon = normalizeRawBase64ImageString(object.icon);
object.icon = migrateRelativeAppIconEndpointUrl(object.icon);
}
if ( typeof object.icon !== 'string' ) {
throw APIError.create('field_invalid', null, { key: 'icon' });
}
object.icon = object.icon.trim();
if ( ! object.icon ) {
// Empty icon is allowed to clear current icon.
} else if ( object.icon.startsWith('data:') ) {
validate_image_base64(object.icon, { key: 'icon' });
} else if ( ! isAllowedAppIconEndpointUrl(object.icon) ) {
throw APIError.create('field_invalid', null, { key: 'icon' });
}
}
validate_url(object.index_url, {
key: 'index_url',
maxlen: 3000,
});
if ( object.maximize_on_start !== undefined ) {
object.maximize_on_start = as_bool(object.maximize_on_start);
}
if ( object.background !== undefined ) {
object.background = as_bool(object.background);
}
if ( object.metadata !== undefined && object.metadata !== null ) {
validate_json(object.metadata, { key: 'metadata' });
}
if ( object.filetype_associations !== undefined ) {
validate_array_of_strings(object.filetype_associations, {
key: 'filetype_associations',
});
}
}
// Ensure puter.site subdomain is owned by user (if index_url uses it)
await this.#ensure_puter_site_subdomain_is_owned(object.index_url, user);
const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlAppOnCreate({
object,
options,
user,
});
if ( joinedApp ) {
return joinedApp;
}
await this.#ensureIndexUrlNotAlreadyInUse({
indexUrl: object.index_url,
});
// Handle app name conflicts (AppES behavior)
if ( await app_name_exists(object.name) ) {
if ( options?.dedupe_name ) {
const base = object.name;
let number = 1;
while ( await app_name_exists(`${base}-${number}`) ) {
number++;
}
object.name = `${base}-${number}`;
} else {
throw APIError.create('app_name_already_in_use', null, {
name: object.name,
});
}
}
// Generate UID for the new app (puter-uuid format: app-{uuid})
const uid = `app-${uuidv4()}`;
// Determine app_owner if actor is AppUnderUserActorType (SetOwnerES behavior)
let app_owner_id = null;
if ( actor.type instanceof AppUnderUserActorType ) {
app_owner_id = actor.type.app.id;
}
// Execute SQL INSERT
const insert_id = await this.#execute_insert(object, uid, user.id, app_owner_id);
// Handle file type associations
if ( object.filetype_associations ) {
await this.#update_filetype_associations(insert_id, object.filetype_associations);
}
// Emit icon event if icon is set
if ( object.icon ) {
const svc_event = this.services.get('event');
const event = {
app_uid: uid,
data_url: object.icon,
url: '',
};
await svc_event.emit('app.new-icon', event);
if ( typeof event.url === 'string' && event.url ) {
this.db_write.write(
'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1',
[event.url, uid],
);
}
}
// Return the created app
return await this.#read({ uid });
}
async #execute_insert (object, uid, owner_user_id, app_owner_id) {
const columns = ['uid', 'owner_user_id'];
const values = [uid, owner_user_id];
if ( app_owner_id !== null ) {
columns.push('app_owner');
values.push(app_owner_id);
}
const sql_column_map = {
name: 'name',
title: 'title',
description: 'description',
icon: 'icon',
index_url: 'index_url',
maximize_on_start: 'maximize_on_start',
background: 'background',
metadata: 'metadata',
};
for ( const [field, column] of Object.entries(sql_column_map) ) {
if ( object[field] === undefined ) continue;
let value = object[field];
// Handle JSON fields
if ( field === 'metadata' && value !== null ) {
value = JSON.stringify(value);
}
// Handle boolean fields
if ( field === 'maximize_on_start' || field === 'background' ) {
value = value ? 1 : 0;
}
columns.push(column);
values.push(value);
}
const placeholders = columns.map(() => '?').join(', ');
const stmt = `INSERT INTO apps (${columns.join(', ')}) VALUES (${placeholders})`;
const result = await this.db_write.write(stmt, values);
return result.insertId;
}
async #delete ({ uid, id }) {
// Only UserActorType and AppUnderUserActorType are allowed to do this
const actor = Context.get('actor');
if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) {
throw APIError.create('forbidden');
}
// Read the existing app
const old_app = await this.#read({
uid,
id,
backend_only_options: { no_filter_owner: true },
});
if ( ! old_app ) {
throw APIError.create('entity_not_found', null, {
identifier: uid || JSON.stringify(id),
});
}
// Check owner permission (WriteByOwnerOnlyES behavior)
await this.#check_owner_permission(old_app);
// If actor is AppUnderUserActorType, check app_owner (AppLimitedES behavior)
if ( actor.type instanceof AppUnderUserActorType ) {
await this.#check_app_owner_permission(old_app, actor);
}
// Call app-information service to perform the deletion (AppES behavior)
const svc_appInformation = this.services.get('app-information');
await svc_appInformation.delete_app(old_app.uid);
return { success: true, uid: old_app.uid };
}
async #check_app_owner_permission (old_app, actor) {
// Check if app has write permission to all user's apps
const svc_permission = this.services.get('permission');
const user = actor.type.user;
const perm = `es:app:${user.uuid}:write`;
const can_write_any = await svc_permission.check(actor, perm);
if ( can_write_any ) {
return;
}
// Otherwise verify the app owns this entity
const app = actor.type.app;
const app_owner = old_app.app_owner;
const app_owner_uid = app_owner?.uid;
if ( !app_owner_uid || app_owner_uid !== app.uid ) {
throw APIError.create('forbidden');
}
}
async #update ({ object, id, options }) {
const old_app = await this.#read({
uid: object.uid,
id,
backend_only_options: { no_filter_owner: true },
});
if ( ! old_app ) {
throw APIError.create('entity_not_found', null, {
identifier: object.uid || JSON.stringify(id),
});
}
// Only UserActorType and AppUnderUserActorType are allowed to do this
const actor = Context.get('actor');
if ( ! (actor.type instanceof UserActorType || actor.type instanceof AppUnderUserActorType) ) {
throw APIError.create('forbidden');
}
// Check owner permission (WriteByOwnerOnlyES behavior)
await this.#check_owner_permission(old_app);
// If actor is AppUnderUserActorType, check app_owner (AppLimitedES behavior)
if ( actor.type instanceof AppUnderUserActorType ) {
await this.#check_app_owner_permission(old_app, actor);
}
// Remove protected/read_only fields from the update (ValidationES behavior)
{
object = { ...object };
for ( const field of this.constructor.PROTECTED_FIELDS ) {
delete object[field];
}
for ( const field of this.constructor.READ_ONLY_FIELDS ) {
delete object[field];
}
}
// Validate fields
{
if ( object.name !== undefined ) {
validate_string(object.name, {
key: 'name',
maxlen: config.app_name_max_length,
regex: config.app_name_regex,
});
}
if ( object.title !== undefined ) {
validate_string(object.title, {
key: 'title',
maxlen: config.app_title_max_length,
});
}
if ( object.description !== undefined && object.description !== null ) {
validate_string(object.description, {
key: 'description',
maxlen: 7000,
});
}
if ( object.icon !== undefined && object.icon !== null ) {
if ( typeof object.icon === 'string' ) {
object.icon = normalizeRawBase64ImageString(object.icon);
object.icon = migrateRelativeAppIconEndpointUrl(object.icon);
}
if ( typeof object.icon !== 'string' ) {
throw APIError.create('field_invalid', null, { key: 'icon' });
}
object.icon = object.icon.trim();
if ( ! object.icon ) {
// Empty icon is allowed to clear current icon.
} else if ( object.icon.startsWith('data:') ) {
validate_image_base64(object.icon, { key: 'icon' });
} else if ( ! isAllowedAppIconEndpointUrl(object.icon) ) {
throw APIError.create('field_invalid', null, { key: 'icon' });
}
}
if ( object.index_url !== undefined ) {
validate_url(object.index_url, {
key: 'index_url',
maxlen: 3000,
});
}
// Flag type - adapt values using as_bool
if ( object.maximize_on_start !== undefined ) {
object.maximize_on_start = as_bool(object.maximize_on_start);
}
if ( object.background !== undefined ) {
object.background = as_bool(object.background);
}
if ( object.metadata !== undefined && object.metadata !== null ) {
validate_json(object.metadata, { key: 'metadata' });
}
if ( object.filetype_associations !== undefined ) {
validate_array_of_strings(object.filetype_associations, {
key: 'filetype_associations',
});
}
}
// Handle app-specific logic (AppES behavior)
const user = actor.type.user;
const oldAppId = await this.#resolveAppId(old_app);
// Ensure puter.site subdomain is owned by user (if index_url changed)
if ( object.index_url && object.index_url !== old_app.index_url ) {
await this.#ensure_puter_site_subdomain_is_owned(object.index_url, user);
const joinedApp = await this.#maybeJoinOwnedHostedIndexUrlAppOnCreate({
object,
options,
user,
excludeAppId: oldAppId,
});
if ( joinedApp ) {
return joinedApp;
}
await this.#ensureIndexUrlNotAlreadyInUse({
indexUrl: object.index_url,
excludeAppId: oldAppId,
});
}
// Handle app name conflicts
if ( object.name !== undefined ) {
await this.#handle_name_conflict(object, old_app, options);
}
// Build and execute SQL UPDATE
const { insert_id } = await this.#execute_update(object, old_app);
// Handle file type associations
if ( object.filetype_associations !== undefined ) {
await this.#update_filetype_associations(insert_id, object.filetype_associations);
}
// Emit events for icon/name or app changes
await this.#emit_change_events(object, old_app);
// Return the updated app (re-fetch for client-safe output)
// TODO: optimize this
return await this.#read({ uid: old_app.uid });
}
async #resolveAppId (app) {
const appId = Number(app?.id);
if ( Number.isInteger(appId) && appId > 0 ) return appId;
if ( typeof app?.uid !== 'string' || !app.uid ) return undefined;
const rows = await this.db.read(
'SELECT id FROM apps WHERE uid = ? LIMIT 1',
[app.uid],
);
const resolvedId = Number(rows?.[0]?.id);
if ( Number.isInteger(resolvedId) && resolvedId > 0 ) return resolvedId;
return undefined;
}
async #check_owner_permission (old_app) {
const svc_permission = this.services.get('permission');
const actor = Context.get('actor');
// Check if user has system-wide write permission
{
// We need to fix eslint rule for multi-line calls
const has_permission_to_write_all = await svc_permission.check(
actor,
this.constructor.WRITE_ALL_OWNER_PERMISSION,
);
if ( has_permission_to_write_all ) {
return;
}
}
// Check if user owns the app
{
const user = Context.get('user');
if ( ! old_app.owner ) {
throw APIError.create('forbidden');
}
if ( user.id !== old_app.owner.id ) {
throw APIError.create('forbidden');
}
}
}
/**
* Resolves an app's subdomain to its puter.site root_dir_id.
* Tries associated_app_id first, then falls back to index_url-based lookup.
* @param {Object} app - App object with id, index_url, uid
* @returns {Promise} root_dir_id
* @throws {APIError} entity_not_found if the app has no subdomain / root directory
*/
async getAppRootDirId (app) {
const db_sites = this.services.get('database').get(DB_READ, 'sites');
const rows = await db_sites.read(
'SELECT root_dir_id FROM subdomains WHERE associated_app_id = ? AND root_dir_id IS NOT NULL LIMIT 1',
[app.id],
);
if ( rows?.[0]?.root_dir_id != null ) {
return rows[0].root_dir_id;
}
let hostname;
try {
hostname = (new URL(app.index_url)).hostname.toLowerCase();
} catch {
throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` });
}
const hosting_domain = config.static_hosting_domain?.toLowerCase();
if ( !hosting_domain || !hostname.endsWith(`.${hosting_domain}`) ) {
throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` });
}
const subdomain = hostname.slice(0, hostname.length - hosting_domain.length - 1);
const site = await this.services.get('puter-site').get_subdomain(subdomain, { is_custom_domain: false });
if ( ! site?.root_dir_id ) {
throw APIError.create('entity_not_found', null, { identifier: `app ${app.uid} root directory` });
}
return site.root_dir_id;
}
async #ensure_puter_site_subdomain_is_owned (index_url, user) {
if ( ! user ) return;
const subdomain = this.#extractPuterHostedSubdomain(index_url);
if ( ! subdomain ) return;
const svc_puterSite = this.services.get('puter-site');
const site = await svc_puterSite.get_subdomain(subdomain, { is_custom_domain: false });
if ( !site || site.user_id !== user.id ) {
throw APIError.create('subdomain_not_owned', null, { subdomain });
}
}
#normalizeConfiguredHostedDomain (domainValue) {
if ( typeof domainValue !== 'string' ) return null;
const normalizedDomain = domainValue.trim().toLowerCase().replace(/^\./, '');
if ( ! normalizedDomain ) return null;
return normalizedDomain.split(':')[0] || null;
}
#getPuterHostedDomains () {
const domains = new Set();
for ( const configuredDomain of [
config.static_hosting_domain,
config.static_hosting_domain_alt,
config.private_app_hosting_domain,
config.private_app_hosting_domain_alt,
] ) {
const normalizedConfiguredDomain = this.#normalizeConfiguredHostedDomain(configuredDomain);
if ( normalizedConfiguredDomain ) {
domains.add(normalizedConfiguredDomain);
}
}
return [...domains];
}
#extractPuterHostedSubdomain (indexUrl) {
if ( typeof indexUrl !== 'string' || !indexUrl ) return null;
let hostname;
try {
hostname = (new URL(indexUrl)).hostname.toLowerCase();
} catch {
return null;
}
const hostedDomains = this.#getPuterHostedDomains();
hostedDomains.sort((domainA, domainB) => domainB.length - domainA.length);
for ( const hostedDomain of hostedDomains ) {
const suffix = `.${hostedDomain}`;
if ( hostname.endsWith(suffix) ) {
const subdomain = hostname.slice(0, hostname.length - suffix.length);
return subdomain || null;
}
}
return null;
}
#isPuterHostedIndexUrl (indexUrl) {
return !!this.#extractPuterHostedSubdomain(indexUrl);
}
#buildEquivalentIndexUrlCandidates (indexUrl) {
if ( typeof indexUrl !== 'string' || !indexUrl.trim() ) {
return [];
}
try {
const parsedIndexUrl = new URL(indexUrl);
const origin = `${parsedIndexUrl.protocol}//${parsedIndexUrl.host.toLowerCase()}`;
const pathname = parsedIndexUrl.pathname || '/';
const candidates = new Set();
if ( pathname === '/' || pathname.toLowerCase() === '/index.html' ) {
candidates.add(origin);
candidates.add(`${origin}/`);
candidates.add(`${origin}/index.html`);
} else {
const normalizedPath = pathname.endsWith('/')
? pathname.slice(0, -1)
: pathname;
candidates.add(`${origin}${normalizedPath}`);
candidates.add(`${origin}${normalizedPath}/`);
}
return [...candidates];
} catch {
return [indexUrl.trim()];
}
}
async #findIndexUrlConflictRow ({ indexUrl, excludeAppId } = {}) {
if ( ! this.#isPuterHostedIndexUrl(indexUrl) ) {
return null;
}
const indexUrlCandidates = this.#buildEquivalentIndexUrlCandidates(indexUrl);
if ( indexUrlCandidates.length === 0 ) return null;
if ( hasIndexUrlUniquenessExemption(indexUrlCandidates) ) return null;
const placeholders = indexUrlCandidates.map(() => '?').join(', ');
const parameters = [...indexUrlCandidates];
let query = `SELECT id, uid, owner_user_id, index_url FROM apps WHERE index_url IN (${placeholders})`;
if ( Number.isInteger(excludeAppId) && excludeAppId > 0 ) {
query += ' AND id != ?';
parameters.push(excludeAppId);
}
query += ' ORDER BY timestamp ASC, id ASC LIMIT 1';
const rows = await this.db.read(query, parameters);
const conflictRow = rows.find(row => {
if (
Number.isInteger(excludeAppId)
&& excludeAppId > 0
&& Number(row?.id) === excludeAppId
) {
return false;
}
if ( typeof row?.index_url === 'string' ) {
return indexUrlCandidates.includes(row.index_url);
}
return true;
});
return conflictRow || null;
}
async #ensureIndexUrlNotAlreadyInUse ({ indexUrl, excludeAppId } = {}) {
const conflictRow = await this.#findIndexUrlConflictRow({ indexUrl, excludeAppId });
if ( conflictRow ) {
throw APIError.create('app_index_url_already_in_use', null, {
index_url: indexUrl,
app_uid: conflictRow.uid,
});
}
}
async #claimAppOwnershipByIdForUser ({ appId, userId }) {
if ( !Number.isInteger(appId) || appId <= 0 ) return;
if ( !Number.isInteger(userId) || userId <= 0 ) return;
await this.db_write.write(
'UPDATE apps SET owner_user_id = ? WHERE id = ? AND owner_user_id IS NULL',
[userId, appId],
);
}
#buildCanonicalAppUidAliasKey (oldAppUid) {
return `${APP_UID_ALIAS_KEY_PREFIX}:${oldAppUid}`;
}
#buildCanonicalAppUidAliasReverseKey (canonicalAppUid) {
return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`;
}
#normalizeCanonicalAliasUidList (value) {
if ( ! Array.isArray(value) ) return [];
const normalizedList = [];
const seen = new Set();
for ( const item of value ) {
if ( typeof item !== 'string' || !item ) continue;
if ( seen.has(item) ) continue;
seen.add(item);
normalizedList.push(item);
}
return normalizedList;
}
async #readCanonicalAppUidAlias (oldAppUid) {
if ( typeof oldAppUid !== 'string' || !oldAppUid ) return null;
const kvStore = this.services.get('puter-kvstore');
const suService = this.services.get('su');
if ( !kvStore || typeof kvStore.get !== 'function' ) return null;
if ( !suService || typeof suService.sudo !== 'function' ) return null;
const key = this.#buildCanonicalAppUidAliasKey(oldAppUid);
try {
const canonicalAppUid = await suService.sudo(() => kvStore.get({ key }));
if ( typeof canonicalAppUid === 'string' && canonicalAppUid ) {
return canonicalAppUid;
}
} catch {
// Alias reads are best-effort.
}
return null;
}
async #writeCanonicalAppUidAlias ({ oldAppUid, canonicalAppUid }) {
if ( typeof oldAppUid !== 'string' || !oldAppUid ) return;
if ( typeof canonicalAppUid !== 'string' || !canonicalAppUid ) return;
if ( oldAppUid === canonicalAppUid ) return;
const kvStore = this.services.get('puter-kvstore');
const suService = this.services.get('su');
if ( !kvStore || typeof kvStore.set !== 'function' ) return;
if ( !suService || typeof suService.sudo !== 'function' ) return;
const key = this.#buildCanonicalAppUidAliasKey(oldAppUid);
const reverseKey = this.#buildCanonicalAppUidAliasReverseKey(canonicalAppUid);
const expireAt = Math.floor(Date.now() / 1000) + APP_UID_ALIAS_TTL_SECONDS;
try {
await suService.sudo(async () => {
const reverseValue = await kvStore.get({ key: reverseKey });
const reverseAliases = this.#normalizeCanonicalAliasUidList(reverseValue);
if ( ! reverseAliases.includes(oldAppUid) ) {
reverseAliases.push(oldAppUid);
}
await kvStore.set({
key,
value: canonicalAppUid,
expireAt,
});
await kvStore.set({
key: reverseKey,
value: reverseAliases,
expireAt,
});
});
} catch {
// Alias writes are best-effort.
}
}
async #maybeJoinOwnedHostedIndexUrlAppOnCreate ({
object,
options,
user,
excludeAppId,
} = {}) {
const indexUrl = object?.index_url;
const sourceAppUid = object?.uid;
if ( ! this.#isPuterHostedIndexUrl(indexUrl) ) {
return null;
}
const conflictRow = await this.#findIndexUrlConflictRow({
indexUrl,
excludeAppId,
});
if ( ! conflictRow ) {
return null;
}
const conflictOwnerUserId = Number(conflictRow.owner_user_id);
if (
Number.isInteger(conflictOwnerUserId)
&& conflictOwnerUserId > 0
&& conflictOwnerUserId !== user.id
) {
throw APIError.create('app_index_url_already_in_use', null, {
index_url: indexUrl,
app_uid: conflictRow.uid,
});
}
if ( !Number.isInteger(conflictOwnerUserId) || conflictOwnerUserId <= 0 ) {
await this.#claimAppOwnershipByIdForUser({
appId: conflictRow.id,
userId: user.id,
});
}
const appToJoin = await this.#read({
uid: conflictRow.uid,
backend_only_options: {
no_filter_owner: true,
},
});
if ( !appToJoin || appToJoin.uid !== conflictRow.uid ) {
throw APIError.create('app_index_url_already_in_use', null, {
index_url: indexUrl,
app_uid: conflictRow.uid,
});
}
const appToJoinOwnerId = Number(appToJoin.owner?.id);
if ( !Number.isInteger(appToJoinOwnerId) || appToJoinOwnerId !== user.id ) {
throw APIError.create('app_index_url_already_in_use', null, {
index_url: indexUrl,
app_uid: conflictRow.uid,
});
}
if (
Number.isInteger(conflictOwnerUserId)
&& conflictOwnerUserId === user.id
&& !this.#isOriginBootstrapApp(appToJoin)
) {
// Prevent merging arbitrary same-owner apps; only allow the
// auto-created origin bootstrap app to be absorbed.
throw APIError.create('app_index_url_already_in_use', null, {
index_url: indexUrl,
app_uid: conflictRow.uid,
});
}
const joinedObject = {
...object,
uid: appToJoin.uid,
};
const requestedJoinedName = (
typeof joinedObject.name === 'string'
? joinedObject.name.trim()
: ''
) || null;
const shouldReapplyRequestedNameAfterMerge = (
!!object?.uid
&& !!requestedJoinedName
);
if ( object?.uid && joinedObject.name !== undefined ) {
delete joinedObject.name;
}
let joinedApp = await this.#update({
object: joinedObject,
options,
});
if ( sourceAppUid && sourceAppUid !== appToJoin.uid ) {
await this.#writeCanonicalAppUidAlias({
oldAppUid: sourceAppUid,
canonicalAppUid: appToJoin.uid,
});
const svc_appInformation = this.services.get('app-information');
if ( svc_appInformation?.delete_app ) {
await svc_appInformation.delete_app(sourceAppUid, undefined, {
preserveCanonicalUidAlias: true,
});
}
}
if ( shouldReapplyRequestedNameAfterMerge ) {
joinedApp = await this.#update({
object: {
uid: appToJoin.uid,
name: requestedJoinedName,
},
options,
});
}
return joinedApp;
}
#isOriginBootstrapApp (app) {
if ( !app || typeof app !== 'object' ) return false;
if ( typeof app.uid !== 'string' || !app.uid ) return false;
if ( app.name !== app.uid ) return false;
if ( app.title !== app.uid ) return false;
if ( typeof app.description !== 'string' ) return false;
return app.description.startsWith('App created from origin ');
}
async #handle_name_conflict (object, old_app, options) {
const new_name = object.name;
const old_name = old_app.name;
// If the name hasn't changed, nothing to do
if ( new_name === old_name ) {
delete object.name;
return;
}
// Check if the name is taken
if ( await app_name_exists(new_name) ) {
if ( options?.dedupe_name ) {
// Auto-deduplicate the name
let number = 1;
while ( await app_name_exists(`${new_name}-${number}`) ) {
number++;
}
object.name = `${new_name}-${number}`;
} else {
// Check if this is an old name of the same app
const svc_oldAppName = this.services.get('old-app-name');
const name_info = await svc_oldAppName.check_app_name(new_name);
if ( !name_info || name_info.app_uid !== old_app.uid ) {
throw APIError.create('app_name_already_in_use', null, {
name: new_name,
});
}
// Remove the old name from the old-app-name service
await svc_oldAppName.remove_name(name_info.id);
}
}
}
async #execute_update (object, old_app) {
// Map object fields to SQL columns
const sql_column_map = {
name: 'name',
title: 'title',
description: 'description',
icon: 'icon',
index_url: 'index_url',
maximize_on_start: 'maximize_on_start',
background: 'background',
metadata: 'metadata',
};
const set_clauses = [];
const values = [];
for ( const [field, column] of Object.entries(sql_column_map) ) {
if ( object[field] === undefined ) continue;
let value = object[field];
// Handle JSON fields
if ( field === 'metadata' && value !== null ) {
value = JSON.stringify(value);
}
// Handle boolean fields
if ( field === 'maximize_on_start' || field === 'background' ) {
value = value ? 1 : 0;
}
set_clauses.push(`${column} = ?`);
values.push(value);
}
if ( set_clauses.length > 0 ) {
values.push(old_app.uid);
const stmt = `UPDATE apps SET ${set_clauses.join(', ')} WHERE uid = ? LIMIT 1`;
await this.db_write.write(stmt, values);
}
// Fetch the internal ID
const rows = await this.db.read(
'SELECT id FROM apps WHERE uid = ?',
[old_app.uid],
);
return { insert_id: rows[0]?.id };
}
async #update_filetype_associations (app_id, filetype_associations) {
const oldAssociations = await this.db.read(
'SELECT type FROM app_filetype_association WHERE app_id = ?',
[app_id],
);
const normalizedOld = oldAssociations
.map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\./, ''))
.filter(Boolean);
const normalizedNew = (filetype_associations ?? [])
.map(ft => String(ft).trim().toLowerCase().replace(/^\./, ''))
.filter(Boolean);
// Remove old file associations
await this.db_write.write(
'DELETE FROM app_filetype_association WHERE app_id = ?',
[app_id],
);
// Add new file associations
if ( ! normalizedNew.length ) {
const affectedExtensions = new Set(normalizedOld);
if ( affectedExtensions.size ) {
await deleteRedisKeys(Array.from(affectedExtensions)
.map(ext => AppRedisCacheSpace.associationAppsKey(ext)));
}
return;
}
const stmt =
`INSERT INTO app_filetype_association (app_id, type) VALUES ${
normalizedNew.map(() => '(?, ?)').join(', ')}`;
const values = normalizedNew.flatMap(ft => [app_id, ft]);
await this.db_write.write(stmt, values);
const affectedExtensions = new Set([...normalizedOld, ...normalizedNew]);
if ( affectedExtensions.size ) {
await deleteRedisKeys(Array.from(affectedExtensions)
.map(ext => AppRedisCacheSpace.associationAppsKey(ext)));
}
}
async #emit_change_events (object, old_app) {
const svc_event = this.services.get('event');
const app = {
...old_app,
...object,
uid: old_app.uid,
};
await svc_event.emit('app.changed', {
app_uid: old_app.uid,
action: 'updated',
app,
old_app,
});
// Emit icon change event
if ( object.icon !== undefined && object.icon !== old_app.icon ) {
const event = {
app_uid: old_app.uid,
data_url: object.icon,
};
await svc_event.emit('app.new-icon', event);
if ( typeof event.url === 'string' && event.url ) {
await this.db_write.write(
'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1',
[event.url, old_app.uid],
);
}
}
// Emit name change event
if ( object.name !== undefined && object.name !== old_app.name ) {
const event = {
app_uid: old_app.uid,
new_name: object.name,
old_name: old_app.name,
};
await svc_event.emit('app.rename', event);
}
}
#build_complex_id_where (id) {
const id_keys = Object.keys(id);
id_keys.sort();
// 1. Validate the identifier key from `id`
const redundant_identifiers = this.constructor.REDUNDANT_IDENTIFIERS;
let match_found = false;
for ( let key_set of redundant_identifiers ) {
key_set = Array.isArray(key_set) ? key_set : [key_set];
const sorted_key_set = [...key_set].sort();
// Check if id_keys matches this key_set exactly
if ( id_keys.length === sorted_key_set.length &&
id_keys.every((k, i) => k === sorted_key_set[i]) ) {
match_found = true;
break;
}
}
if ( ! match_found ) {
throw new Error(`Invalid complex id keys: ${id_keys.join(', ')}. ` +
`Allowed: ${redundant_identifiers.join(', ')}`);
}
// 2. Build the SQL string for the predicate
const conditions = [];
const values = [];
for ( const key of id_keys ) {
conditions.push(`apps.${key} = ?`);
values.push(id[key]);
}
return {
clause: conditions.join(' AND '),
values,
};
}
/**
* Checks if a protected app should be filtered out (not accessible to the current actor).
* Returns true if the app should be filtered out, false if it's accessible.
*
* @param {Object} app - The app object with protected, uid, and owner fields
* @param {number} owner_user_id - The database ID of the app owner (for accurate comparison)
* @returns {Promise} true if app should be filtered out, false if accessible
*/
async #check_protected_app_access (app, owner_user_id) {
// If it's not a protected app, no worries - allow it
if ( ! app.protected ) {
return false;
}
const actor = Context.get('actor');
const services = this.services;
// If actor is this app itself, allow it
if (
actor.type instanceof AppUnderUserActorType &&
app.uid === actor.type.app.uid
) {
return false;
}
// If actor is owner of this app, allow it
// Compare using owner_user_id from database for accuracy
if (
actor.type instanceof UserActorType &&
owner_user_id &&
owner_user_id === actor.type.user.id
) {
return false;
}
// Now we need to check for permission
const app_uid = app.uid;
const svc_permission = services.get('permission');
const permission_to_check = `app:uid#${app_uid}:access`;
// If they have permission, allow it
if ( await svc_permission.check(actor, permission_to_check) ) {
return false;
}
// No access - filter it out
return true;
}
}
================================================
FILE: src/backend/src/modules/data-access/AppService.test.js
================================================
import { beforeEach, describe, expect, it, vi } from 'vitest';
import AppService from './AppService.js';
// Mock the Context module
vi.mock('../../util/context.js', () => ({
Context: {
get: vi.fn(),
},
}));
// Mock the helpers module
vi.mock('../../helpers.js', () => ({
app_name_exists: vi.fn(),
}));
// Mock the Actor module
vi.mock('../../services/auth/Actor.js', () => ({
UserActorType: class UserActorType {
},
AppUnderUserActorType: class AppUnderUserActorType {
},
}));
// Mock the validation module
vi.mock('./lib/validation.js', () => ({
validate_string: vi.fn(),
validate_url: vi.fn(),
validate_image_base64: vi.fn(),
validate_json: vi.fn(),
validate_array_of_strings: vi.fn(),
}));
// Mock config
vi.mock('../../config.js', () => ({
default: {
app_name_max_length: 100,
app_name_regex: /^[a-z0-9-]+$/,
app_title_max_length: 200,
static_hosting_domain: 'puter.site',
static_hosting_domain_alt: 'puter.host',
private_app_hosting_domain: 'puter.app',
private_app_hosting_domain_alt: 'puter.dev',
origin: 'https://puter.localhost',
api_base_url: 'https://api.puter.localhost',
},
}));
import { app_name_exists } from '../../helpers.js';
import { AppUnderUserActorType, UserActorType } from '../../services/auth/Actor.js';
import { Context } from '../../util/context.js';
import {
validate_string,
validate_url,
} from './lib/validation.js';
describe('AppService', () => {
let appService;
let mockDb;
let mockDbWrite;
let mockServices;
let mockEventService;
let mockPermissionService;
let mockPuterSiteService;
let mockOldAppNameService;
let mockAppInformationService;
let mockKvStoreService;
let mockSuService;
// Helper to create a mock database row
const createMockAppRow = (overrides = {}) => ({
id: 1,
uid: 'app-uid-123',
name: 'test-app',
title: 'Test App',
description: 'A test application',
icon: 'icon.png',
index_url: 'https://example.com/app',
created_at: '2024-01-01T00:00:00Z',
created_from_origin: 'localhost',
metadata: '{}',
stats: '{}',
approved_for_incentive_program: 0,
approved_for_listing: 1,
approved_for_opening_items: 1,
background: 0,
godmode: 0,
is_private: 0,
maximize_on_start: 0,
protected: 0,
owner_user_id: 1,
owner_user_username: 'testuser',
owner_user_uuid: 'user-uuid-456',
app_owner_uid: 'owner-app-uid-789',
filetypes: '["txt", "doc"]',
...overrides,
});
// Helper to create a mock actor
const createMockUserActor = (userId = 1) => ({
type: Object.assign(new UserActorType(), { user: { id: userId } }),
});
const createMockAppUnderUserActor = (userId = 1, appId = 100) => ({
type: Object.assign(new AppUnderUserActorType(), {
user: { id: userId },
app: { id: appId, uid: 'creator-app-uid' },
}),
});
// Helper to setup Context.get mock for create/update tests
const setupContextForWrite = (actor, user = { id: 1 }) => {
Context.get.mockImplementation((key) => {
if ( key === 'actor' ) return actor;
if ( key === 'user' ) return user;
return null;
});
};
beforeEach(() => {
// Reset mocks
vi.clearAllMocks();
// Reset helper mocks
app_name_exists.mockResolvedValue(false);
// Mock database (read)
mockDb = {
read: vi.fn(),
case: vi.fn().mockImplementation(({ sqlite }) => sqlite),
};
// Mock database (write)
mockDbWrite = {
write: vi.fn().mockResolvedValue({ insertId: 1 }),
};
// Mock event service
mockEventService = {
emit: vi.fn().mockResolvedValue(undefined),
};
// Mock permission service
mockPermissionService = {
check: vi.fn().mockResolvedValue(false),
scan: vi.fn().mockResolvedValue([]),
};
// Mock puter-site service
mockPuterSiteService = {
get_subdomain: vi.fn().mockResolvedValue(null),
};
// Mock old-app-name service
mockOldAppNameService = {
check_app_name: vi.fn().mockResolvedValue(null),
remove_name: vi.fn().mockResolvedValue(undefined),
};
// Mock app-information service
mockAppInformationService = {
delete_app: vi.fn().mockResolvedValue(undefined),
};
mockKvStoreService = {
get: vi.fn().mockResolvedValue(null),
set: vi.fn().mockResolvedValue(true),
};
mockSuService = {
sudo: vi.fn(async (actorOrCallback, maybeCallback) => {
const callback = maybeCallback || actorOrCallback;
return await callback();
}),
};
// Mock services
mockServices = {
get: vi.fn().mockImplementation((serviceName) => {
if ( serviceName === 'database' ) {
return {
get: vi.fn().mockImplementation((mode) => {
if ( mode === 'write' ) return mockDbWrite;
return mockDb;
}),
};
}
if ( serviceName === 'event' ) return mockEventService;
if ( serviceName === 'permission' ) return mockPermissionService;
if ( serviceName === 'puter-site' ) return mockPuterSiteService;
if ( serviceName === 'old-app-name' ) return mockOldAppNameService;
if ( serviceName === 'app-information' ) return mockAppInformationService;
if ( serviceName === 'puter-kvstore' ) return mockKvStoreService;
if ( serviceName === 'su' ) return mockSuService;
return null;
}),
};
// Create AppService instance
appService = new AppService({
services: mockServices,
config: {},
name: 'app-service',
args: {},
context: {
get: vi.fn().mockReturnValue(mockServices),
},
});
// Manually call _init to set up the service
appService.repository = {};
appService.db = mockDb;
appService.db_write = mockDbWrite;
});
describe('#read', () => {
it('should read an app by uid', async () => {
const mockRow = createMockAppRow();
mockDb.read.mockResolvedValueOnce([mockRow]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(mockDb.read).toHaveBeenCalledTimes(1);
expect(mockDb.read).toHaveBeenNthCalledWith(
1,
expect.stringContaining('WHERE apps.uid = ?'),
['app-uid-123'],
);
expect(result).toBeDefined();
expect(result.uid).toBe('app-uid-123');
expect(result.name).toBe('test-app');
expect(result.title).toBe('Test App');
});
it('should read an app by complex id (name)', async () => {
const mockRow = createMockAppRow();
mockDb.read.mockResolvedValueOnce([mockRow]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { id: { name: 'test-app' } });
expect(mockDb.read).toHaveBeenCalledTimes(1);
expect(mockDb.read).toHaveBeenNthCalledWith(
1,
expect.stringContaining('WHERE apps.name = ?'),
['test-app'],
);
expect(result).toBeDefined();
expect(result.name).toBe('test-app');
});
it('should throw entity_not_found when no app is found', async () => {
mockDb.read.mockResolvedValue([]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.read.call(appService, { uid: 'nonexistent-uid' })).rejects.toMatchObject({
fields: { code: 'entity_not_found' },
});
});
it('should resolve app by canonical uid alias when old uid is missing', async () => {
const canonicalRow = createMockAppRow({
uid: 'app-canonical-uid-123',
});
mockDb.read
.mockResolvedValueOnce([])
.mockResolvedValueOnce([canonicalRow]);
mockKvStoreService.get.mockResolvedValue('app-canonical-uid-123');
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-old-uid-123' });
expect(result.uid).toBe('app-canonical-uid-123');
expect(mockSuService.sudo).toHaveBeenCalled();
expect(mockKvStoreService.get).toHaveBeenCalledWith({
key: 'app:canonicalUidAlias:app-old-uid-123',
});
expect(mockDb.read).toHaveBeenNthCalledWith(
2,
expect.stringContaining('WHERE apps.uid = ?'),
['app-canonical-uid-123'],
);
});
it('should throw an error when neither uid nor id is provided', async () => {
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.read.call(appService, {})).rejects.toThrow(
'read requires either uid or id',
);
});
it('should throw an error for invalid complex id keys', async () => {
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.read.call(appService, { id: { invalidKey: 'value' } })).rejects.toThrow('Invalid complex id keys');
});
it('should correctly coerce boolean fields from database', async () => {
const mockRow = createMockAppRow({
approved_for_incentive_program: 1,
approved_for_listing: '1',
approved_for_opening_items: 0,
background: '0',
godmode: 1,
is_private: '1',
maximize_on_start: '1',
protected: 0,
});
mockDb.read.mockResolvedValue([mockRow]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(result.approved_for_incentive_program).toBe(true);
expect(result.approved_for_listing).toBe(true);
expect(result.approved_for_opening_items).toBe(false);
expect(result.background).toBe(false);
expect(result.godmode).toBe(true);
expect(result.is_private).toBe(true);
expect(result.maximize_on_start).toBe(true);
expect(result.protected).toBe(false);
});
it('should parse filetypes JSON and strip leading dots', async () => {
const mockRow = createMockAppRow({
filetypes: '[".txt", ".doc", "pdf"]',
});
mockDb.read.mockResolvedValueOnce([mockRow]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(result.filetype_associations).toEqual(['txt', 'doc', 'pdf']);
expect(mockDb.read).toHaveBeenCalledTimes(1);
});
it('should filter out null values in filetypes array', async () => {
const mockRow = createMockAppRow({
filetypes: '[".txt", null, "pdf"]',
});
mockDb.read.mockResolvedValueOnce([mockRow]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(result.filetype_associations).toEqual(['txt', 'pdf']);
expect(mockDb.read).toHaveBeenCalledTimes(1);
});
it('should query filetype associations table when filetypes JSON is missing', async () => {
const mockRow = createMockAppRow({ filetypes: null });
mockDb.read
.mockResolvedValueOnce([mockRow])
.mockResolvedValueOnce([
{ type: '.txt' },
{ type: null },
{ type: 'pdf' },
]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(result.filetype_associations).toEqual(['txt', 'pdf']);
expect(mockDb.read).toHaveBeenCalledTimes(2);
expect(mockDb.read).toHaveBeenNthCalledWith(
2,
'SELECT type FROM app_filetype_association WHERE app_id = ?',
[mockRow.id],
);
});
it('should have owner parameter', async () => {
const mockRow = createMockAppRow();
mockDb.read.mockResolvedValue([mockRow]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(result.owner).toEqual({
username: 'testuser',
uuid: 'user-uuid-456',
});
});
it('should include app_owner in the result', async () => {
const mockRow = createMockAppRow();
mockDb.read.mockResolvedValue([mockRow]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(result.app_owner).toEqual({
uid: 'owner-app-uid-789',
});
});
it('should fetch icon with size when icon_size param is provided', async () => {
const mockRow = createMockAppRow();
mockDb.read.mockResolvedValue([mockRow]);
const mockIconService = {
getAppIconPath: vi.fn().mockReturnValue('/app-icon/app-uid-123/64'),
};
appService.context = {
get: vi.fn().mockImplementation((key) => {
if ( key === 'services' ) {
return {
get: vi.fn().mockImplementation((name) => {
if ( name === 'app-icon' ) return mockIconService;
return null;
}),
};
}
return null;
}),
};
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, {
uid: 'app-uid-123',
params: { icon_size: 64 },
});
expect(mockIconService.getAppIconPath).toHaveBeenCalledWith({
appUid: 'app-uid-123',
size: 64,
});
expect(result.icon).toBe('/app-icon/app-uid-123/64');
});
it('should route base64 icons through app-icon endpoint even without icon_size', async () => {
const mockRow = createMockAppRow({
icon: 'data:image/png;base64,abc123',
icon_is_base64: 1,
});
mockDb.read.mockResolvedValue([mockRow]);
const mockIconService = {
getAppIconPath: vi.fn().mockReturnValue('/app-icon/app-uid-123/128'),
};
appService.context = {
get: vi.fn().mockImplementation((key) => {
if ( key === 'services' ) {
return {
get: vi.fn().mockImplementation((name) => {
if ( name === 'app-icon' ) return mockIconService;
return null;
}),
};
}
return null;
}),
};
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, { uid: 'app-uid-123' });
expect(mockIconService.getAppIconPath).toHaveBeenCalledWith({
appUid: 'app-uid-123',
size: undefined,
});
expect(result.icon).toBe('/app-icon/app-uid-123/128');
});
it('should keep original icon when icon service throws', async () => {
const mockRow = createMockAppRow();
mockDb.read.mockResolvedValue([mockRow]);
const mockErrorService = {
report: vi.fn(),
};
const mockIconService = {
getAppIconPath: vi.fn().mockImplementation(() => {
throw new Error('Icon fetch failed');
}),
};
appService.context = {
get: vi.fn().mockImplementation((key) => {
if ( key === 'services' ) {
return {
get: vi.fn().mockImplementation((name) => {
if ( name === 'app-icon' ) return mockIconService;
if ( name === 'error-service' ) return mockErrorService;
return null;
}),
};
}
return null;
}),
};
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.read.call(appService, {
uid: 'app-uid-123',
params: { icon_size: 64 },
});
expect(mockErrorService.report).toHaveBeenCalledWith(
'AppES:read_transform',
expect.objectContaining({ source: expect.any(Error) }),
);
expect(result.icon).toBe('icon.png');
});
});
describe('#select', () => {
it('should select all apps with default parameters', async () => {
const mockRows = [
createMockAppRow({ id: 1, uid: 'app-1', name: 'app-one' }),
createMockAppRow({ id: 2, uid: 'app-2', name: 'app-two' }),
];
mockDb.read.mockResolvedValue(mockRows);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {});
expect(mockDb.read).toHaveBeenCalledTimes(1);
expect(mockDb.read).toHaveBeenCalledWith(
expect.not.stringContaining('WHERE'),
[],
);
expect(result).toHaveLength(2);
expect(result[0].uid).toBe('app-1');
expect(result[1].uid).toBe('app-2');
});
it('should filter by user-can-edit predicate', async () => {
const mockUser = { id: 42 };
Context.get.mockReturnValue(mockUser);
const mockRows = [createMockAppRow()];
mockDb.read.mockResolvedValue(mockRows);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {
predicate: ['user-can-edit'],
});
expect(mockDb.read).toHaveBeenCalledWith(
expect.stringContaining('WHERE apps.owner_user_id=?'),
[42],
);
expect(result).toHaveLength(1);
});
it('should throw error when predicate is not an array', async () => {
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.select.call(appService, { predicate: 'invalid' })).rejects.toThrow('predicate must be an array');
});
it('should correctly coerce boolean fields for all selected apps', async () => {
const mockRows = [
createMockAppRow({
id: 1,
approved_for_listing: 1,
godmode: 0,
}),
createMockAppRow({
id: 2,
approved_for_listing: '0',
godmode: '1',
}),
];
mockDb.read.mockResolvedValue(mockRows);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {});
expect(result[0].approved_for_listing).toBe(true);
expect(result[0].godmode).toBe(false);
expect(result[1].approved_for_listing).toBe(false);
expect(result[1].godmode).toBe(true);
});
it('should parse filetypes for all selected apps', async () => {
const mockRows = [
createMockAppRow({ id: 1, filetypes: '[".txt"]' }),
createMockAppRow({ id: 2, filetypes: '[".pdf", ".doc"]' }),
];
mockDb.read.mockResolvedValue(mockRows);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {});
expect(result[0].filetype_associations).toEqual(['txt']);
expect(result[1].filetype_associations).toEqual(['pdf', 'doc']);
});
it('should fetch icons with size for all apps when icon_size is provided', async () => {
const mockRows = [
createMockAppRow({ id: 1, uid: 'app-1', icon: 'icon1.png' }),
createMockAppRow({ id: 2, uid: 'app-2', icon: 'icon2.png' }),
];
mockDb.read.mockResolvedValue(mockRows);
const mockIconService = {
getAppIconPath: vi.fn().mockImplementation(({ appUid, size }) => `/app-icon/${appUid}/${size}`),
};
appService.context = {
get: vi.fn().mockImplementation((key) => {
if ( key === 'services' ) {
return {
get: vi.fn().mockImplementation((name) => {
if ( name === 'app-icon' ) return mockIconService;
return null;
}),
};
}
return null;
}),
};
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {
params: { icon_size: 32 },
});
expect(mockIconService.getAppIconPath).toHaveBeenCalledTimes(2);
expect(result[0].icon).toBe('/app-icon/app-1/32');
expect(result[1].icon).toBe('/app-icon/app-2/32');
});
it('should only route base64 icons through app-icon endpoint when icon_size is not provided', async () => {
const mockRows = [
createMockAppRow({
id: 1,
uid: 'app-1',
icon: 'data:image/png;base64,abc123',
icon_is_base64: 1,
}),
createMockAppRow({
id: 2,
uid: 'app-2',
icon: 'https://puter-app-icons.puter.site/app-2-128.png',
icon_is_base64: 0,
}),
];
mockDb.read.mockResolvedValue(mockRows);
const mockIconService = {
getAppIconPath: vi.fn().mockImplementation(({ appUid }) => `/app-icon/${appUid}/128`),
};
appService.context = {
get: vi.fn().mockImplementation((key) => {
if ( key === 'services' ) {
return {
get: vi.fn().mockImplementation((name) => {
if ( name === 'app-icon' ) return mockIconService;
return null;
}),
};
}
return null;
}),
};
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {});
expect(mockIconService.getAppIconPath).toHaveBeenCalledTimes(1);
expect(result[0].icon).toBe('/app-icon/app-1/128');
expect(result[1].icon).toBe('https://puter-app-icons.puter.site/app-2-128.png');
});
it('should return empty array when no apps exist', async () => {
mockDb.read.mockResolvedValue([]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {});
expect(result).toEqual([]);
});
it('should have owner parameter for all selected apps', async () => {
const mockRows = [
createMockAppRow({
id: 1,
owner_user_username: 'user1',
owner_user_uuid: 'uuid-1',
}),
createMockAppRow({
id: 2,
owner_user_username: 'user2',
owner_user_uuid: 'uuid-2',
}),
];
mockDb.read.mockResolvedValue(mockRows);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.select.call(appService, {});
expect(result[0].owner).toEqual({
username: 'user1',
uuid: 'uuid-1',
});
expect(result[1].owner).toEqual({
username: 'user2',
uuid: 'uuid-2',
});
});
it('should handle filetypes that are not strings', async () => {
const mockRows = [
createMockAppRow({ id: 1, filetypes: '[".txt", 123]' }),
];
mockDb.read.mockResolvedValue(mockRows);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.select.call(appService, {})).rejects.toThrow(
'expected filetypesAsJSON[1] to be a string',
);
});
it('should handle malformed filetypes JSON', async () => {
const mockRows = [
createMockAppRow({ id: 1, filetypes: 'not valid json' }),
];
mockDb.read.mockResolvedValue(mockRows);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.select.call(appService, {})).rejects.toThrow(
'failed to get app filetype associations',
);
});
it('should not require dialect-specific JSON aggregation for app selection', async () => {
mockDb.read.mockResolvedValue([]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.select.call(appService, {});
expect(mockDb.case).not.toHaveBeenCalled();
});
});
describe('#build_complex_id_where (via #read)', () => {
it('should accept "name" as a valid redundant identifier', async () => {
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.read.call(appService, { id: { name: 'test' } });
expect(mockDb.read).toHaveBeenCalledWith(
expect.stringContaining('apps.name = ?'),
['test'],
);
});
it('should reject identifiers not in REDUNDANT_IDENTIFIERS', async () => {
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.read.call(appService, { id: { title: 'test' } })).rejects.toThrow('Invalid complex id keys: title');
});
});
describe('#create', () => {
it('should create an app with valid input', async () => {
setupContextForWrite(createMockUserActor(1));
// Mock the read after insert
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [];
}
return [createMockAppRow({
uid: expect.stringContaining('app-'),
name: 'new-app',
title: 'New App',
index_url: 'https://example.com/new',
})];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'new-app',
title: 'New App',
index_url: 'https://example.com/new',
},
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('INSERT INTO apps'),
expect.arrayContaining(['new-app', 'New App', 'https://example.com/new']),
);
});
it('should throw forbidden for non-user actors', async () => {
// Mock an invalid actor type
Context.get.mockImplementation((key) => {
if ( key === 'actor' ) return { type: {} };
return null;
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
},
})).rejects.toThrow();
});
it('should throw field_missing when name is not provided', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
title: 'Test',
index_url: 'https://example.com',
},
})).rejects.toThrow();
});
it('should throw field_missing when title is not provided', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'test-app',
index_url: 'https://example.com',
},
})).rejects.toThrow();
});
it('should throw field_missing when index_url is not provided', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
},
})).rejects.toThrow();
});
it('should remove protected fields from input', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
last_review: '2024-01-01', // protected field
},
});
// The INSERT should not include last_review
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('INSERT INTO apps'),
expect.not.arrayContaining(['2024-01-01']),
);
});
it('should remove read_only fields from input', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
approved_for_listing: true, // read_only field
godmode: true, // read_only field
is_private: true, // read_only field
},
});
// These fields should not appear in the INSERT
const writeCall = mockDbWrite.write.mock.calls[0];
expect(writeCall[0]).not.toContain('approved_for_listing');
expect(writeCall[0]).not.toContain('godmode');
expect(writeCall[0]).not.toContain('is_private');
});
it('should handle name conflict with dedupe_name option', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
// First check returns true (name exists), second returns false
app_name_exists
.mockResolvedValueOnce(true) // 'new-app' exists
.mockResolvedValueOnce(false); // 'new-app-1' doesn't exist
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'new-app',
title: 'New App',
index_url: 'https://example.com',
},
options: { dedupe_name: true },
});
// Should have inserted with deduped name
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('INSERT INTO apps'),
expect.arrayContaining(['new-app-1']),
);
});
it('should throw error when name conflict without dedupe_name', async () => {
setupContextForWrite(createMockUserActor(1));
app_name_exists.mockResolvedValue(true);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'existing-app',
title: 'Test',
index_url: 'https://example.com',
},
})).rejects.toThrow();
});
it('should allow equivalent index_url already in use on create for non-hosted origins', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 999,
uid: 'app-existing-uid',
owner_user_id: 1,
index_url: 'https://example.com/',
}];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'new-app',
title: 'New App',
index_url: 'https://example.com/index.html',
},
})).resolves.toBeDefined();
});
it('should allow duplicate dev-center placeholder index_url on create', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 999,
uid: 'app-existing-placeholder',
owner_user_id: 1,
index_url: 'https://dev-center.puter.com/coming-soon.html',
}];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'new-app',
title: 'New App',
index_url: 'https://dev-center.puter.com/coming-soon.html',
},
})).resolves.toBeDefined();
});
it('should join existing hosted app when index_url is owned and already used', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 999,
uid: 'app-existing-hosted',
owner_user_id: null,
index_url: 'https://mysite.puter.site',
}];
}
return [createMockAppRow({
id: 999,
uid: 'app-existing-hosted',
name: 'existing-hosted-app',
title: 'Existing Hosted App',
index_url: 'https://mysite.puter.site',
owner_user_id: 1,
})];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
const joined = await crudQ.create.call(appService, {
object: {
name: 'joined-hosted-app',
title: 'Joined Hosted App',
index_url: 'https://mysite.puter.site',
},
});
expect(joined.uid).toBe('app-existing-hosted');
expect(mockDbWrite.write).not.toHaveBeenCalledWith(
expect.stringContaining('INSERT INTO apps'),
expect.any(Array),
);
expect(mockKvStoreService.set).not.toHaveBeenCalled();
});
it('should throw when hosted index_url is already in use by another owner on create', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 999,
uid: 'app-existing-hosted',
owner_user_id: 2,
index_url: 'https://mysite.puter.site',
}];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'new-app',
title: 'New App',
index_url: 'https://mysite.puter.site',
},
})).rejects.toMatchObject({
fields: {
code: 'app_index_url_already_in_use',
},
});
});
it('should set app_owner when actor is AppUnderUserActorType', async () => {
setupContextForWrite(createMockAppUnderUserActor(1, 100));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
},
});
// Should include app_owner in the INSERT
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('app_owner'),
expect.arrayContaining([100]),
);
});
it('should emit app.new-icon event when icon is provided', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: 'data:image/png;base64,abc123',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
data_url: 'data:image/png;base64,abc123',
}),
);
});
it('should accept raw base64 icon and normalize to data URL on create', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const rawBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ';
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: rawBase64,
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
data_url: `data:image/png;base64,${rawBase64}`,
}),
);
});
it('should migrate relative app-icon endpoint path to absolute URL on create', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
validate_url.mockImplementation((_value, { key }) => {
if ( key === 'icon' ) {
throw new Error('icon should not be validated as a URL');
}
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: '/app-icon/app-uid-123/64',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
data_url: 'https://api.puter.localhost/app-icon/app-uid-123',
}),
);
expect(validate_url).toHaveBeenCalledWith('https://example.com', expect.objectContaining({ key: 'index_url' }));
});
it('should reject object icon payloads on create', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: { url: '/app-icon/app-uid-123/64' },
},
})).rejects.toMatchObject({
fields: { code: 'field_invalid', key: 'icon' },
});
});
it('should allow empty icon string on create', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: '',
},
});
expect(mockEventService.emit).not.toHaveBeenCalledWith('app.new-icon', expect.anything());
});
it('should migrate legacy app-icons host URL to app-icon endpoint URL on create', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: 'https://puter-app-icons.puter.site/app-uid-123-64.png',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
data_url: 'https://api.puter.localhost/app-icon/app-uid-123',
}),
);
});
it('should allow absolute app-icon endpoint URL on API origin', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: 'https://api.puter.localhost/app-icon/app-uid-123/64',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
data_url: 'https://api.puter.localhost/app-icon/app-uid-123',
}),
);
});
it('should reject foreign absolute app-icon endpoint URL on create', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: 'https://evil.example/app-icon/app-uid-123/64',
},
})).rejects.toMatchObject({
fields: { code: 'field_invalid', key: 'icon' },
});
});
it('should reject non app-icon URL icon on create', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
icon: 'https://example.com/webhook',
},
})).rejects.toMatchObject({
fields: { code: 'field_invalid', key: 'icon' },
});
});
it('should handle filetype_associations', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
filetype_associations: ['txt', 'pdf'],
},
});
// Should have three write calls: INSERT app, DELETE old associations, INSERT new associations
// (DELETE is called even for create since #update_filetype_associations always clears first)
expect(mockDbWrite.write).toHaveBeenCalledTimes(3);
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('DELETE FROM app_filetype_association'),
[1],
);
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('INSERT INTO app_filetype_association'),
expect.arrayContaining([1, 'txt', 1, 'pdf']),
);
});
it('should call validate_string for name and title', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test Title',
index_url: 'https://example.com',
},
});
expect(validate_string).toHaveBeenCalledWith('test-app', expect.objectContaining({ key: 'name' }));
expect(validate_string).toHaveBeenCalledWith('Test Title', expect.objectContaining({ key: 'title' }));
});
it('should call validate_url for index_url', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com/app',
},
});
expect(validate_url).toHaveBeenCalledWith('https://example.com/app', expect.objectContaining({ key: 'index_url' }));
});
it('should generate a UID with app- prefix', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.create.call(appService, {
object: {
name: 'test-app',
title: 'Test',
index_url: 'https://example.com',
},
});
const writeCall = mockDbWrite.write.mock.calls[0];
const values = writeCall[1];
const uidValue = values[0]; // uid is first value
expect(uidValue).toMatch(/^app-[0-9a-f-]{36}$/);
});
});
describe('#update', () => {
beforeEach(() => {
// Default: return an existing app for updates
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
})]);
});
it('should update an app with valid input', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', title: 'Updated Title' },
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['Updated Title', 'app-uid-123']),
);
});
it('should throw entity_not_found when app does not exist', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: { uid: 'nonexistent-uid', title: 'Test' },
})).rejects.toThrow();
});
it('should throw forbidden when user does not own the app', async () => {
// User 2 trying to update app owned by user 1
setupContextForWrite(createMockUserActor(2), { id: 2 });
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
})]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: { uid: 'app-uid-123', title: 'Hacked Title' },
})).rejects.toThrow();
});
it('should allow update when user has write-all-owners permission', async () => {
setupContextForWrite(createMockUserActor(2), { id: 2 });
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
})]);
mockPermissionService.check.mockResolvedValue(true);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', title: 'Admin Update' },
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['Admin Update']),
);
});
it('should remove protected fields from update', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
title: 'Updated',
last_review: '2024-12-01', // protected field
},
});
const writeCall = mockDbWrite.write.mock.calls[0];
expect(writeCall[0]).not.toContain('last_review');
});
it('should remove read_only fields from update', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
title: 'Updated',
approved_for_listing: true,
godmode: true,
is_private: true,
},
});
const writeCall = mockDbWrite.write.mock.calls[0];
expect(writeCall[0]).not.toContain('approved_for_listing');
expect(writeCall[0]).not.toContain('godmode');
expect(writeCall[0]).not.toContain('is_private');
});
it('should handle name change with conflict', async () => {
setupContextForWrite(createMockUserActor(1));
app_name_exists.mockResolvedValue(true);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: { uid: 'app-uid-123', name: 'taken-name' },
})).rejects.toThrow();
});
it('should allow name change with dedupe_name option', async () => {
setupContextForWrite(createMockUserActor(1));
app_name_exists
.mockResolvedValueOnce(true) // 'new-name' exists
.mockResolvedValueOnce(false); // 'new-name-1' doesn't exist
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', name: 'new-name' },
options: { dedupe_name: true },
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['new-name-1']),
);
});
it('should allow reclaiming old app name', async () => {
setupContextForWrite(createMockUserActor(1));
app_name_exists.mockResolvedValue(true);
mockOldAppNameService.check_app_name.mockResolvedValue({
id: 99,
app_uid: 'app-uid-123', // Same app
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', name: 'old-name' },
});
expect(mockOldAppNameService.remove_name).toHaveBeenCalledWith(99);
expect(mockDbWrite.write).toHaveBeenCalled();
});
it('should not update name if unchanged', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', name: 'test-app' }, // Same as existing
});
// Should only have the read for ID, no name in update
const writeCall = mockDbWrite.write.mock.calls.find(call => call[0].includes('UPDATE'));
if ( writeCall ) {
expect(writeCall[1]).not.toContain('test-app');
}
});
it('should emit app.new-icon event when icon changes', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: 'data:image/png;base64,newicon',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
app_uid: 'app-uid-123',
data_url: 'data:image/png;base64,newicon',
}),
);
});
it('should accept raw base64 icon and normalize to data URL on update', async () => {
setupContextForWrite(createMockUserActor(1));
const rawBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ';
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: rawBase64,
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
app_uid: 'app-uid-123',
data_url: `data:image/png;base64,${rawBase64}`,
}),
);
});
it('should migrate relative app-icon endpoint path to absolute URL on update', async () => {
setupContextForWrite(createMockUserActor(1));
validate_url.mockImplementation((_value, { key }) => {
if ( key === 'icon' ) {
throw new Error('icon should not be validated as a URL');
}
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: '/app-icon/app-uid-123/64',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
app_uid: 'app-uid-123',
data_url: 'https://api.puter.localhost/app-icon/app-uid-123',
}),
);
});
it('should reject object icon payloads on update', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: { url: '/app-icon/app-uid-123/64' },
},
})).rejects.toMatchObject({
fields: { code: 'field_invalid', key: 'icon' },
});
});
it('should allow empty icon string on update', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: '',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
app_uid: 'app-uid-123',
data_url: '',
}),
);
});
it('should migrate legacy app-icons host URL to app-icon endpoint URL on update', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: 'https://puter-app-icons.puter.site/app-uid-123-64.png',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
app_uid: 'app-uid-123',
data_url: 'https://api.puter.localhost/app-icon/app-uid-123',
}),
);
});
it('should allow absolute app-icon endpoint URL on API origin when updating icon', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: 'https://api.puter.localhost/app-icon/app-uid-123/64',
},
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.new-icon',
expect.objectContaining({
app_uid: 'app-uid-123',
data_url: 'https://api.puter.localhost/app-icon/app-uid-123',
}),
);
});
it('should reject foreign absolute app-icon endpoint URL on update', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: 'https://evil.example/app-icon/app-uid-123/64',
},
})).rejects.toMatchObject({
fields: { code: 'field_invalid', key: 'icon' },
});
});
it('should reject non app-icon URL icon on update', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
icon: 'https://example.com/webhook',
},
})).rejects.toMatchObject({
fields: { code: 'field_invalid', key: 'icon' },
});
});
it('should emit app.rename event when name changes', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', name: 'renamed-app' },
});
expect(mockEventService.emit).toHaveBeenCalledWith(
'app.rename',
expect.objectContaining({
app_uid: 'app-uid-123',
new_name: 'renamed-app',
old_name: 'test-app',
}),
);
});
it('should update filetype_associations', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
filetype_associations: ['doc', 'xls'],
},
});
// Should delete old associations
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('DELETE FROM app_filetype_association'),
[1],
);
// Should insert new associations
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('INSERT INTO app_filetype_association'),
expect.arrayContaining([1, 'doc', 1, 'xls']),
);
});
it('should validate fields when provided', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
name: 'updated-name',
title: 'Updated Title',
description: 'Updated description',
index_url: 'https://updated.com',
},
});
expect(validate_string).toHaveBeenCalledWith('updated-name', expect.objectContaining({ key: 'name' }));
expect(validate_string).toHaveBeenCalledWith('Updated Title', expect.objectContaining({ key: 'title' }));
expect(validate_string).toHaveBeenCalledWith('Updated description', expect.objectContaining({ key: 'description' }));
expect(validate_url).toHaveBeenCalledWith('https://updated.com', expect.objectContaining({ key: 'index_url' }));
});
it('should check subdomain ownership when index_url changes to puter.site', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue(null);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
index_url: 'https://mysite.puter.site',
},
})).rejects.toThrow();
});
it('should allow index_url change when subdomain is owned', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
index_url: 'https://mysite.puter.site',
},
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['https://mysite.puter.site']),
);
});
it('should allow index_url change when private hosted subdomain is owned', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
index_url: 'https://mysite.puter.dev',
},
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['https://mysite.puter.dev']),
);
});
it('should allow equivalent index_url already in use on update for non-hosted origins', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 777,
uid: 'app-conflict-uid',
owner_user_id: 2,
index_url: 'https://updated.com/',
}];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
index_url: 'https://updated.com/index.html',
},
})).resolves.toBeDefined();
});
it('should allow duplicate dev-center placeholder index_url on update', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 777,
uid: 'app-existing-placeholder',
owner_user_id: 1,
index_url: 'https://dev-center.puter.com/coming-soon.html',
}];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
index_url: 'https://dev-center.puter.com/coming-soon.html',
},
})).resolves.toBeDefined();
});
it('should join existing unowned hosted app when index_url is already in use on update', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });
let readCallCount = 0;
mockDb.read.mockImplementation(async (query, params) => {
readCallCount++;
if ( readCallCount > 100 ) {
throw new Error(`excessive mockDb.read calls in join test: ${String(query)} :: ${JSON.stringify(params)}`);
}
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
if ( Array.isArray(params) && params[params.length - 1] === 777 ) {
// Mirrors SQL `AND id != ?` behavior during join follow-up updates.
return [];
}
return [{
id: 777,
uid: 'app-conflict-uid',
owner_user_id: null,
index_url: 'https://mysite.puter.site/',
}];
}
if ( Array.isArray(params) && params[0] === 'app-conflict-uid' ) {
return [createMockAppRow({
id: 777,
uid: 'app-conflict-uid',
name: 'existing-hosted-app',
title: 'Existing Hosted App',
index_url: 'https://mysite.puter.site/',
owner_user_id: 1,
})];
}
return [createMockAppRow({
id: 1,
uid: 'app-uid-123',
name: 'updating-app',
title: 'Updating App',
index_url: 'https://other.puter.site',
owner_user_id: 1,
})];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
title: 'Joined Update Title',
index_url: 'https://mysite.puter.site/index.html',
},
});
expect(result.uid).toBe('app-conflict-uid');
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['Joined Update Title', 'app-conflict-uid']),
);
expect(mockAppInformationService.delete_app).toHaveBeenCalledWith(
'app-uid-123',
undefined,
{ preserveCanonicalUidAlias: true },
);
expect(mockKvStoreService.set).toHaveBeenCalledWith(expect.objectContaining({
key: 'app:canonicalUidAlias:app-uid-123',
value: 'app-conflict-uid',
}));
});
it('should throw when owned hosted index_url is already in use on update', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 777,
uid: 'app-conflict-uid',
owner_user_id: 1,
index_url: 'https://mysite.puter.site/',
}];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
index_url: 'https://mysite.puter.site/index.html',
},
})).rejects.toMatchObject({
fields: {
code: 'app_index_url_already_in_use',
},
});
});
it('should throw when equivalent hosted index_url is already in use on update', async () => {
setupContextForWrite(createMockUserActor(1));
mockPuterSiteService.get_subdomain.mockResolvedValue({ user_id: 1 });
mockDb.read.mockImplementation(async (query) => {
if ( typeof query === 'string' && query.includes('FROM apps WHERE index_url IN') ) {
return [{
id: 777,
uid: 'app-conflict-uid',
owner_user_id: 2,
index_url: 'https://mysite.puter.site/',
}];
}
return [createMockAppRow()];
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: {
uid: 'app-uid-123',
index_url: 'https://mysite.puter.site/index.html',
},
})).rejects.toMatchObject({
fields: {
code: 'app_index_url_already_in_use',
},
});
});
it('should throw forbidden when app actor does not own the entity (AppLimitedES behavior)', async () => {
// App actor trying to update an app it didn't create
setupContextForWrite(createMockAppUnderUserActor(1, 999));
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
app_owner_uid: 'different-app-uid',
})]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.update.call(appService, {
object: { uid: 'app-uid-123', title: 'Hacked Title' },
})).rejects.toThrow();
});
it('should allow app actor to update entity it owns (AppLimitedES behavior)', async () => {
// App actor updating an app it created
const actor = createMockAppUnderUserActor(1, 100);
actor.type.app.uid = 'creator-app-uid';
setupContextForWrite(actor);
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
app_owner_uid: 'creator-app-uid',
})]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', title: 'Updated by App' },
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['Updated by App']),
);
});
it('should allow app actor with write permission to update any entity (AppLimitedES behavior)', async () => {
setupContextForWrite(createMockAppUnderUserActor(1, 999));
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
app_owner_uid: 'different-app-uid',
})]);
// Grant write permission
mockPermissionService.check.mockResolvedValue(true);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.update.call(appService, {
object: { uid: 'app-uid-123', title: 'Admin Update' },
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.arrayContaining(['Admin Update']),
);
});
});
describe('#upsert', () => {
it('should call create when entity does not exist', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.upsert.call(appService, {
object: {
name: 'new-app',
title: 'New App',
index_url: 'https://example.com',
},
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('INSERT INTO apps'),
expect.any(Array),
);
});
it('should call update when entity exists', async () => {
setupContextForWrite(createMockUserActor(1));
// Read returns existing entity
mockDb.read.mockResolvedValue([createMockAppRow()]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.upsert.call(appService, {
object: { uid: 'app-uid-123', title: 'Updated Title' },
});
expect(mockDbWrite.write).toHaveBeenCalledWith(
expect.stringContaining('UPDATE apps SET'),
expect.any(Array),
);
});
});
describe('#delete', () => {
beforeEach(() => {
// Mock app-information service
mockAppInformationService = {
delete_app: vi.fn().mockResolvedValue(undefined),
};
// Update mockServices to include app-information
mockServices.get.mockImplementation((serviceName) => {
if ( serviceName === 'database' ) {
return {
get: vi.fn().mockImplementation((mode) => {
if ( mode === 'write' ) return mockDbWrite;
return mockDb;
}),
};
}
if ( serviceName === 'event' ) return mockEventService;
if ( serviceName === 'permission' ) return mockPermissionService;
if ( serviceName === 'puter-site' ) return mockPuterSiteService;
if ( serviceName === 'old-app-name' ) return mockOldAppNameService;
if ( serviceName === 'app-information' ) return mockAppInformationService;
if ( serviceName === 'puter-kvstore' ) return mockKvStoreService;
if ( serviceName === 'su' ) return mockSuService;
return null;
});
// Default: return an existing app for deletes
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
})]);
});
it('should delete an app by uid', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });
expect(mockAppInformationService.delete_app).toHaveBeenCalledWith('app-uid-123');
expect(result.success).toBe(true);
expect(result.uid).toBe('app-uid-123');
});
it('should delete an app by complex id (name)', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.delete.call(appService, { id: { name: 'test-app' } });
expect(mockAppInformationService.delete_app).toHaveBeenCalledWith('app-uid-123');
expect(result.success).toBe(true);
});
it('should throw entity_not_found when app does not exist', async () => {
setupContextForWrite(createMockUserActor(1));
mockDb.read.mockResolvedValue([]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.delete.call(appService, { uid: 'nonexistent-uid' }))
.rejects.toThrow();
});
it('should throw forbidden for non-user actors', async () => {
Context.get.mockImplementation((key) => {
if ( key === 'actor' ) return { type: {} };
return null;
});
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' }))
.rejects.toThrow();
});
it('should throw forbidden when user does not own the app', async () => {
// User 2 trying to delete app owned by user 1
setupContextForWrite(createMockUserActor(2), { id: 2 });
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
})]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' }))
.rejects.toThrow();
});
it('should allow delete when user has write-all-owners permission', async () => {
setupContextForWrite(createMockUserActor(2), { id: 2 });
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
})]);
mockPermissionService.check.mockResolvedValue(true);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });
expect(mockAppInformationService.delete_app).toHaveBeenCalled();
expect(result.success).toBe(true);
});
it('should invalidate app cache after delete', async () => {
setupContextForWrite(createMockUserActor(1));
const crudQ = AppService.IMPLEMENTS['crud-q'];
await crudQ.delete.call(appService, { uid: 'app-uid-123' });
});
it('should throw forbidden when app actor does not own the entity', async () => {
// App actor trying to delete an app it didn't create
setupContextForWrite(createMockAppUnderUserActor(1, 999));
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
app_owner_uid: 'different-app-uid',
})]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
await expect(crudQ.delete.call(appService, { uid: 'app-uid-123' }))
.rejects.toThrow();
});
it('should allow app actor to delete entity it owns', async () => {
// App actor deleting an app it created
const actor = createMockAppUnderUserActor(1, 100);
actor.type.app.uid = 'creator-app-uid';
setupContextForWrite(actor);
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
app_owner_uid: 'creator-app-uid',
})]);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });
expect(mockAppInformationService.delete_app).toHaveBeenCalled();
expect(result.success).toBe(true);
});
it('should allow app actor with write permission to delete any entity', async () => {
setupContextForWrite(createMockAppUnderUserActor(1, 999));
mockDb.read.mockResolvedValue([createMockAppRow({
owner_user_id: 1,
app_owner_uid: 'different-app-uid',
})]);
// Grant write permission
mockPermissionService.check.mockResolvedValue(true);
const crudQ = AppService.IMPLEMENTS['crud-q'];
const result = await crudQ.delete.call(appService, { uid: 'app-uid-123' });
expect(mockAppInformationService.delete_app).toHaveBeenCalled();
expect(result.success).toBe(true);
});
});
});
================================================
FILE: src/backend/src/modules/data-access/DEV.md
================================================
## Development for `data-access` module
This document will contain notes, documentation, and snippets written
while developing the `data-access` module replacements for what was
formerly handled by EntityStoreService and OM (Object Mapping).
### App List Test Code
This code is used to test listing apps with one of the available
CRUD-implementing drivers.
```javascript
await (async () => {
const resp = await fetch('http://api.puter.localhost:4100/drivers/call', {
method: 'POST',
headers: {
Authorization: `Bearer ${puter.authToken}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
args: { predicate: ['user-can-edit'] },
driver: 'es:app',
interface: 'puter-apps',
method: 'select',
}),
})
return (await resp.json()).result;
})();
```
### AI-Generated Compare Function
I asked an LLM to find me a javascript object compare function
that I can paste in developer tools and it started generating
one from scratch. To my surprise it worked just fine, so I'm pasting
this here for the time being for convenience:
```javascript
(() => {
// Deep compare + diff reporter for DevTools (no deps)
// Usage:
// const r = deepCompare(a, b);
// console.log(r.pass, r.message);
// r.print(); // pretty console output
// Options:
// deepCompare(a,b,{ showSame:false, maxDiffs:200, sortKeys:true })
function deepCompare(a, b, opts = {}) {
const options = {
showSame: false, // include "same" entries in the diff list
maxDiffs: 200, // cap diffs so you don't nuke your console
sortKeys: true, // stable key ordering when iterating plain objects
...opts,
};
const diffs = [];
const seenPairs = new WeakMap(); // a -> WeakMap(b -> true)
const isObjectLike = (v) => v !== null && (typeof v === "object" || typeof v === "function");
const tagOf = (v) => Object.prototype.toString.call(v); // "[object X]"
const isPlainObject = (v) => {
if (tagOf(v) !== "[object Object]") return false;
const proto = Object.getPrototypeOf(v);
return proto === Object.prototype || proto === null;
};
const typeLabel = (v) => {
if (v === null) return "null";
const t = typeof v;
if (t !== "object") return t;
return tagOf(v).slice(8, -1);
};
const formatVal = (v) => {
// Safe-ish inline formatter for messages (keeps things short)
try {
if (typeof v === "string") return JSON.stringify(v.length > 120 ? v.slice(0, 117) + "…" : v);
if (typeof v === "number" && Object.is(v, -0)) return "-0";
if (typeof v === "bigint") return `${v}n`;
if (typeof v === "symbol") return v.toString();
if (typeof v === "function") return `[Function ${v.name || "anonymous"}]`;
if (v instanceof Date) return isNaN(v.getTime()) ? "Invalid Date" : `Date(${v.toISOString()})`;
if (v instanceof RegExp) return v.toString();
if (v instanceof Map) return `Map(${v.size})`;
if (v instanceof Set) return `Set(${v.size})`;
if (ArrayBuffer.isView(v) && !(v instanceof DataView)) return `${v.constructor.name}(${v.length})`;
if (v instanceof ArrayBuffer) return `ArrayBuffer(${v.byteLength})`;
if (v && v.constructor && v.constructor !== Object) return `${v.constructor.name}{…}`;
if (Array.isArray(v)) return `Array(${v.length})`;
if (isPlainObject(v)) return "Object{…}";
return `${typeLabel(v)}{…}`;
} catch {
return "[Unformattable]";
}
};
const pathToString = (path) => {
if (!path.length) return "(root)";
let s = "";
for (const p of path) {
if (typeof p === "number") s += `[${p}]`;
else if (typeof p === "string") {
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(p)) s += (s ? "." : "") + p;
else s += `[${JSON.stringify(p)}]`;
} else if (typeof p === "symbol") s += `[${p.toString()}]`;
else s += `[${String(p)}]`;
}
return s;
};
const pushDiff = (kind, path, left, right, extra) => {
if (diffs.length >= options.maxDiffs) return;
diffs.push({
kind, // "type" | "value" | "missing-left" | "missing-right" | "prototype" | "keys" | ...
path: [...path],
left,
right,
extra,
});
};
const markSeen = (x, y) => {
if (!isObjectLike(x) || !isObjectLike(y)) return false;
let inner = seenPairs.get(x);
if (!inner) {
inner = new WeakMap();
seenPairs.set(x, inner);
}
if (inner.get(y)) return true;
inner.set(y, true);
return false;
};
const sameValueZero = (x, y) => Object.is(x, y); // handles NaN, -0
const compareArrays = (x, y, path) => {
if (x.length !== y.length) pushDiff("value", [...path, "length"], x.length, y.length, "array length mismatch");
const n = Math.max(x.length, y.length);
for (let i = 0; i < n; i++) {
if (i >= x.length) pushDiff("missing-left", [...path, i], undefined, y[i], "missing index in left");
else if (i >= y.length) pushDiff("missing-right", [...path, i], x[i], undefined, "missing index in right");
else walk(x[i], y[i], [...path, i]);
if (diffs.length >= options.maxDiffs) return;
}
};
const compareTypedArrays = (x, y, path) => {
if (x.constructor !== y.constructor) {
pushDiff("type", path, x.constructor?.name, y.constructor?.name, "typed array class mismatch");
return;
}
if (x.length !== y.length) pushDiff("value", [...path, "length"], x.length, y.length, "typed array length mismatch");
const n = Math.min(x.length, y.length);
for (let i = 0; i < n; i++) {
if (!sameValueZero(x[i], y[i])) pushDiff("value", [...path, i], x[i], y[i], "typed array element mismatch");
if (diffs.length >= options.maxDiffs) return;
}
};
const compareArrayBuffer = (x, y, path) => {
if (x.byteLength !== y.byteLength) {
pushDiff("value", [...path, "byteLength"], x.byteLength, y.byteLength, "ArrayBuffer byteLength mismatch");
return;
}
const a8 = new Uint8Array(x);
const b8 = new Uint8Array(y);
for (let i = 0; i < a8.length; i++) {
if (a8[i] !== b8[i]) {
pushDiff("value", [...path, i], a8[i], b8[i], "ArrayBuffer byte mismatch");
if (diffs.length >= options.maxDiffs) return;
}
}
};
const compareDates = (x, y, path) => {
const tx = x.getTime();
const ty = y.getTime();
if (!sameValueZero(tx, ty)) pushDiff("value", path, x, y, "Date mismatch");
};
const compareRegex = (x, y, path) => {
if (x.source !== y.source || x.flags !== y.flags) pushDiff("value", path, x, y, "RegExp mismatch");
};
const compareMaps = (x, y, path) => {
if (x.size !== y.size) pushDiff("value", [...path, "size"], x.size, y.size, "Map size mismatch");
// Map key equality is identity-based; here we:
// 1) try direct key lookup for primitive keys
// 2) for object keys, we require the *same object reference* exists as key in the other map
// (test frameworks do similar unless they do expensive key deep-matching)
for (const [k, xv] of x.entries()) {
if (!y.has(k)) {
pushDiff("missing-right", [...path, `MapKey(${formatVal(k)})`], xv, undefined, "Map missing key on right");
continue;
}
walk(xv, y.get(k), [...path, `MapKey(${formatVal(k)})`]);
if (diffs.length >= options.maxDiffs) return;
}
for (const [k, yv] of y.entries()) {
if (!x.has(k)) {
pushDiff("missing-left", [...path, `MapKey(${formatVal(k)})`], undefined, yv, "Map missing key on left");
if (diffs.length >= options.maxDiffs) return;
}
}
};
const compareSets = (x, y, path) => {
if (x.size !== y.size) pushDiff("value", [...path, "size"], x.size, y.size, "Set size mismatch");
// Same logic: membership is identity for object values.
for (const v of x.values()) {
if (!y.has(v)) pushDiff("missing-right", [...path, `SetVal(${formatVal(v)})`], v, undefined, "Set missing value on right");
if (diffs.length >= options.maxDiffs) return;
}
for (const v of y.values()) {
if (!x.has(v)) pushDiff("missing-left", [...path, `SetVal(${formatVal(v)})`], undefined, v, "Set missing value on left");
if (diffs.length >= options.maxDiffs) return;
}
};
const comparePlainObjects = (x, y, path) => {
// Compare prototypes (handy when something is class instance vs plain object)
const px = Object.getPrototypeOf(x);
const py = Object.getPrototypeOf(y);
if (px !== py) pushDiff("prototype", path, px?.constructor?.name || px, py?.constructor?.name || py, "Prototype mismatch");
const keysX = Reflect.ownKeys(x);
const keysY = Reflect.ownKeys(y);
const norm = (ks) => {
// Sort only string keys for stability; keep symbols in original order
if (!options.sortKeys) return ks;
const str = ks.filter(k => typeof k === "string").sort();
const sym = ks.filter(k => typeof k === "symbol");
const numLike = []; // keep numeric-looking strings in numeric order if you want; leaving out to stay simple
// We'll just do lexical sort for strings; okay for devtools output.
return [...str, ...sym];
};
const kx = norm(keysX);
const ky = norm(keysY);
const setY = new Set(keysY);
const setX = new Set(keysX);
for (const k of kx) {
if (!setY.has(k)) {
pushDiff("missing-right", [...path, k], x[k], undefined, "Missing property on right");
} else {
walk(x[k], y[k], [...path, k]);
}
if (diffs.length >= options.maxDiffs) return;
}
for (const k of ky) {
if (!setX.has(k)) {
pushDiff("missing-left", [...path, k], undefined, y[k], "Missing property on left");
if (diffs.length >= options.maxDiffs) return;
}
}
};
function walk(x, y, path) {
if (diffs.length >= options.maxDiffs) return;
if (sameValueZero(x, y)) {
if (options.showSame) pushDiff("same", path, x, y);
return;
}
const tx = typeLabel(x);
const ty = typeLabel(y);
if (tx !== ty) {
pushDiff("type", path, tx, ty, "Type mismatch");
return;
}
// Circular / repeated references
if (markSeen(x, y)) return;
// Per-type comparisons
if (Array.isArray(x)) return compareArrays(x, y, path);
if (ArrayBuffer.isView(x) && !(x instanceof DataView)) return compareTypedArrays(x, y, path);
if (x instanceof ArrayBuffer) return compareArrayBuffer(x, y, path);
if (x instanceof Date) return compareDates(x, y, path);
if (x instanceof RegExp) return compareRegex(x, y, path);
if (x instanceof Map) return compareMaps(x, y, path);
if (x instanceof Set) return compareSets(x, y, path);
// Functions: compare by reference already failed; treat as value mismatch
if (typeof x === "function") {
pushDiff("value", path, x, y, "Function reference mismatch");
return;
}
// Objects (including class instances): compare own keys + nested values.
if (isObjectLike(x)) return comparePlainObjects(x, y, path);
// Primitives (should have been caught by Object.is earlier)
pushDiff("value", path, x, y, "Value mismatch");
}
walk(a, b, []);
const pass = diffs.length === 0;
const message = pass
? "✅ Values are deeply equal."
: buildMessage(diffs, options);
function buildMessage(diffs, options) {
const lines = [];
lines.push(`❌ Values differ (${diffs.length}${diffs.length >= options.maxDiffs ? "+" : ""} diff${diffs.length === 1 ? "" : "s"}):`);
for (let i = 0; i < diffs.length; i++) {
const d = diffs[i];
const p = pathToString(d.path);
const left = formatVal(d.left);
const right = formatVal(d.right);
const label = d.kind.padEnd(14, " ");
const extra = d.extra ? ` — ${d.extra}` : "";
lines.push(`${String(i + 1).padStart(3, " ")}. ${label} ${p}${extra}`);
lines.push(` left : ${left}`);
lines.push(` right: ${right}`);
}
if (diffs.length >= options.maxDiffs) {
lines.push(`… (diffs capped at maxDiffs=${options.maxDiffs})`);
}
return lines.join("\n");
}
function print() {
if (pass) {
console.log("%c✅ deepCompare: PASS", "font-weight:bold");
return;
}
console.groupCollapsed(`%c❌ deepCompare: FAIL (${diffs.length}${diffs.length >= options.maxDiffs ? "+" : ""})`, "font-weight:bold");
console.log(message);
// Also log a structured table for quick scanning
const table = diffs.map((d) => ({
kind: d.kind,
path: pathToString(d.path),
left: formatVal(d.left),
right: formatVal(d.right),
note: d.extra || "",
}));
try { console.table(table); } catch {}
console.groupEnd();
}
return { pass, diffs, message, print };
}
// Expose globally for DevTools convenience
window.deepCompare = deepCompare;
console.log("deepCompare installed. Usage: deepCompare(a,b).print()");
})();
```
================================================
FILE: src/backend/src/modules/data-access/DataAccessModule.js
================================================
import { AdvancedBase } from '@heyputer/putility';
import AppService from './AppService.js';
export class DataAccessModule extends AdvancedBase {
async install (context) {
const services = context.get('services');
services.registerService('app', AppService);
}
}
================================================
FILE: src/backend/src/modules/data-access/lib/coercion.js
================================================
// These utility functions describe how values stored in the database
// are to be understood as their higher-level counterparts.
import { CoercionTypeError } from './error.js';
/**
* MySQL lets us store `1` (an integer) or `0` (also an integer) as
* the closest parallel to a boolean "true or false" value.
* Sqlite lets us store `"1"` (a string) or `0` (also a string) as
* the closest parallel to a boolean "true of false" value.
*
* So we define a function here called `as_bool` that will make
* `"0"` or `0` become `false`, and `"1"` or `1` become `true`.
*
* @param {any} value - The value to coerce to a boolean.
* @returns {boolean} The coerced boolean value.
*/
export const as_bool = value => {
if ( value === undefined ) return false;
if ( value === 0 ) value = false;
if ( value === 1 ) value = true;
if ( value === '0' ) value = false;
if ( value === '1' ) value = true;
if ( typeof value !== 'boolean' ) {
throw new CoercionTypeError({ expected: 'boolean', got: typeof value });
}
return value;
};
================================================
FILE: src/backend/src/modules/data-access/lib/error.js
================================================
/**
* Replaces `OMTypeError` from ES/OM implementation.
* This might be removed or replaced in the future.
*/
export class CoercionTypeError extends Error {
constructor ({ expected, got }) {
const message = `expected ${expected}, got ${got}`;
super(message);
this.name = 'CoercionTypeError';
}
}
================================================
FILE: src/backend/src/modules/data-access/lib/filter.js
================================================
// These utility functions describe how to produce an object safe
// for transfer that came from a "raw" object.
export const user_to_client = raw_user => {
return {
username: raw_user.username,
// This `uuid` is not an internal-only ID.
uuid: raw_user.uuid,
};
};
================================================
FILE: src/backend/src/modules/data-access/lib/sqlutil.js
================================================
/**
* When columns are selected from a joined table and prefixed:
*
* SELECT joined_table.* AS joined_table_
*
* This function is able to extract the object from the result:
*
* extract_from_prefix(row, 'joined_table_') // columns of joined_table
*
* @param {*} row
* @param {*} prefix
*/
export const extract_from_prefix = (row, prefix) => {
const result = {};
for ( const [key, value] of Object.entries(row) ) {
if ( key.startsWith(prefix) ) {
result[key.replace(prefix, '')] = value;
}
}
return result;
};
================================================
FILE: src/backend/src/modules/data-access/lib/validation.js
================================================
import validator from 'validator';
import APIError from '../../../api/APIError.js';
/**
* Validates a string value with optional maxlen and regex constraints.
* @param {string} value - The value to validate
* @param {object} meta - Metadata for the validation
* @param {string} meta.key - The field name (for error messages)
* @param {number} [meta.maxlen] - Maximum length allowed
* @param {RegExp} [meta.regex] - Regex pattern the string must match
*/
export const validate_string = (value, { key, maxlen, regex }) => {
if ( typeof value !== 'string' ) {
throw APIError.create('field_invalid', null, { key });
}
if ( maxlen !== undefined && value.length > maxlen ) {
throw APIError.create('field_too_long', null, { key, max_length: maxlen });
}
if ( regex !== undefined && !regex.test(value) ) {
throw APIError.create('field_invalid', null, { key });
}
};
/**
* Validates an image-base64 value (data URL for images).
* Checks for proper prefix and XSS characters.
* @param {string} value - The value to validate
* @param {object} meta - Metadata for the validation
* @param {string} meta.key - The field name (for error messages)
*/
export const validate_image_base64 = (value, { key }) => {
if ( typeof value !== 'string' ) {
throw APIError.create('field_invalid', null, { key });
}
if ( ! value.startsWith('data:image/') ) {
throw APIError.create('field_invalid', null, { key });
}
// XSS character check from image-base64 prop type
const xss_chars = ['<', '>', '&', '"', "'", '`'];
if ( xss_chars.some(char => value.includes(char)) ) {
throw APIError.create('field_invalid', null, { key });
}
};
/**
* Validates a URL value with optional maxlen constraint.
* Uses the validator library, allowing localhost.
* @param {string} value - The value to validate
* @param {object} meta - Metadata for the validation
* @param {string} meta.key - The field name (for error messages)
* @param {number} [meta.maxlen] - Maximum length allowed
*/
export const validate_url = (value, { key, maxlen }) => {
if ( typeof value !== 'string' ) {
throw APIError.create('field_invalid', null, { key });
}
if ( maxlen !== undefined && value.length > maxlen ) {
throw APIError.create('field_too_long', null, { key, max_length: maxlen });
}
// URL validation using validator library (same as url prop type)
let valid = validator.isURL(value);
if ( ! valid ) {
valid = validator.isURL(value, { host_whitelist: ['localhost'] });
}
if ( ! valid ) {
throw APIError.create('field_invalid', null, { key });
}
};
/**
* Validates a JSON value (must be an object or array).
* @param {*} value - The value to validate
* @param {object} meta - Metadata for the validation
* @param {string} meta.key - The field name (for error messages)
*/
export const validate_json = (value, { key }) => {
if ( typeof value !== 'object' ) {
throw APIError.create('field_invalid', null, { key });
}
};
/**
* Validates an array where each element is a string.
* @param {*} value - The value to validate
* @param {object} meta - Metadata for the validation
* @param {string} meta.key - The field name (for error messages)
*/
export const validate_array_of_strings = (value, { key }) => {
if ( ! Array.isArray(value) ) {
throw APIError.create('field_invalid', null, { key });
}
for ( const item of value ) {
if ( typeof item !== 'string' ) {
throw APIError.create('field_invalid', null, { key });
}
}
};
================================================
FILE: src/backend/src/modules/development/DevelopmentModule.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
async _init () {
const dns2 = require('dns2');
// this.dns = new dns2(this.config.client);
this.dns = new dns2({
nameServers: ['127.0.0.1'],
port: 5300,
});
if ( this.config.test_server ) {
this.test_server_();
}
}
/**
* Returns the DNS client instance
* @returns {Object} The DNS client
*/
get_client () {
return this.dns;
}
/**
* Creates and starts a test DNS server that responds to A and TXT record queries
* The server listens on port 5300 and returns mock responses for testing purposes
*/
test_server_ () {
const dns2 = require('dns2');
const { Packet } = dns2;
const server = dns2.createServer({
udp: true,
handle: (request, send, rinfo) => {
const { questions } = request;
const response = Packet.createResponseFromRequest(request);
for ( const question of questions ) {
if ( question.type === Packet.TYPE.A || question.type === Packet.TYPE.ANY ) {
response.answers.push({
name: question.name,
type: Packet.TYPE.A,
class: Packet.CLASS.IN,
ttl: 300,
address: '127.0.0.11',
});
}
if ( question.type === Packet.TYPE.TXT || question.type === Packet.TYPE.ANY ) {
response.answers.push({
name: question.name,
type: Packet.TYPE.TXT,
class: Packet.CLASS.IN,
ttl: 300,
data: [
JSON.stringify({ username: 'ed3' }),
],
});
}
}
send(response);
},
});
server.on('listening', () => {
this.log.debug('Fake DNS server listening', server.addresses());
if ( this.config.test_server_selftest ) {
(async () => {
await sleep(5000);
{
console.log('Trying first test');
const result = await this.dns.resolveA('test.local');
console.log('Test 1', result);
}
{
console.log('Trying second test');
const result = await this.dns.resolve('_puter-verify.test.local', 'TXT');
console.log('Test 2', result);
}
})();
}
});
server.on('close', () => {
console.log('Fake DNS server closed');
});
server.on('request', (request, response, rinfo) => {
console.log(request.header.id, request.questions[0]);
});
server.on('requestError', (error) => {
console.log('Client sent an invalid request', error);
});
server.listen({
udp: {
port: 5300,
address: '127.0.0.1',
},
});
}
}
module.exports = { DNSService };
================================================
FILE: src/backend/src/modules/domain/DomainModule.js
================================================
const { AdvancedBase } = require('@heyputer/putility');
class DomainModule extends AdvancedBase {
async install (context) {
const services = context.get('services');
const { DomainVerificationService } = require('./DomainVerificationService');
services.registerService('domain-verification', DomainVerificationService);
// TODO: enable flag
const { TXTVerifyService } = require('./TXTVerifyService');
services.registerService('__txt-verify', TXTVerifyService);
}
}
module.exports = { DomainModule };
================================================
FILE: src/backend/src/modules/domain/DomainVerificationService.js
================================================
const { get_user } = require('../../helpers');
const BaseService = require('../../services/BaseService');
class DomainVerificationService extends BaseService {
_init () {
this._register_commands();
}
async get_controlling_user ({ domain }) {
const svc_event = this.services.get('event');
// 1 :: Allow event listeners to verify domains
const event = {
domain,
user: undefined,
};
await svc_event.emit('domain.get-controlling-user', event);
if ( event.user ) {
return event.user;
}
// 2 :: If there is no controlling user, 'admin' is the
// controlling user.
return await get_user({ username: 'admin' });
}
_register_commands (commands) {
const svc_commands = this.services.get('commands');
svc_commands.registerCommands('domain', [
{
id: 'user',
description: '',
handler: async (args, log) => {
const res = await this.get_controlling_user({ domain: args[0] });
log.log(res);
},
},
]);
}
}
module.exports = {
DomainVerificationService,
};
================================================
FILE: src/backend/src/modules/domain/TXTVerifyService.js
================================================
const { get_user } = require('../../helpers');
const BaseService = require('../../services/BaseService');
const { atimeout } = require('../../util/asyncutil');
class TXTVerifyService extends BaseService {
'__on_boot.consolidation' () {
const svc_dns = this.services.get('dns');
const dns = svc_dns.get_client();
const svc_event = this.services.get('event');
svc_event.on('domain.get-controlling-user', async (_, event) => {
const record_name = `_puter-verify.${event.domain}`;
try {
const result = await atimeout(5000,
dns.resolve(record_name, 'TXT'));
const answer = result.answers.filter(a => a.name === record_name &&
a.type === 16)[0];
const data_raw = answer.data;
const data = JSON.parse(data_raw);
event.user = await get_user({ username: data.username });
} catch (e) {
console.error('ERROR', e);
}
});
}
}
module.exports = {
TXTVerifyService,
};
================================================
FILE: src/backend/src/modules/entitystore/EntityStoreInterfaceService.js
================================================
/*
* Copyright (C) 2025-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } get - Retrieve the value(s) for the given key(s).
* @property {function(KVStoreSetParams): Promise} set - Set a value for a key, with optional expiration.
* @property {function(KVStoreDelParams): Promise} del - Delete a value by key.
* @property {function(KVStoreListParams): Promise} list - List key-value pairs, optionally with pagination.
* @property {function(): Promise} flush - Delete all key-value pairs in the store.
* @property {(params: KVStoreUpdateParams) => Promise} update - Update nested values by key.
* @property {(params: KVStoreAddParams) => Promise} add - Append values into list paths by key.
* @property {(params: KVStoreRemoveParams) => Promise} remove - Remove nested values by key.
* @property {(params: {key:string, pathAndAmountMap: Record}) => Promise} incr - Increment a numeric value by key.
* @property {(params: {key:string, pathAndAmountMap: Record}) => Promise} decr - Decrement a numeric value by key.
* @property {function(KVStoreExpireAtParams): Promise} expireAt - Set a key to expire at a specific UNIX timestamp (seconds).
* @property {function(KVStoreExpireParams): Promise} expire - Set a key to expire after a given TTL (seconds).
*
* @typedef {Object} KVStoreGetParams
* @property {string|string[]} key - The key or array of keys to retrieve.
*
* @typedef {Object} KVStoreSetParams
* @property {string} key - The key to set.
* @property {*} value - The value to store.
* @property {number} [expireAt] - Optional UNIX timestamp (seconds) when the key should expire.
*
* @typedef {Object} KVStoreDelParams
* @property {string} key - The key to delete.
*
* @typedef {Object} KVStoreListParams
* @property {string} [as] - Optional type to list as ("keys", "values", or "entries").
* @property {string} [pattern] - Optional key prefix to match.
* @property {number} [limit] - Optional max number of items to return.
* @property {string} [cursor] - Optional cursor to continue listing from.
*
* @typedef {Object} KVStoreListResult
* @property {Array} items - Items in the current page.
* @property {string} [cursor] - Cursor for the next page, if available.
*
* @typedef {Object} KVStoreUpdateParams
* @property {string} key - The key to update.
* @property {Object.} pathAndValueMap - Map of period-joined paths to values.
* @property {number} [ttl] - Optional TTL in seconds for the whole object.
*
* @typedef {Object} KVStoreAddParams
* @property {string} key - The key to update.
* @property {Object.} pathAndValueMap - Map of period-joined paths to values to append.
*
* @typedef {Object} KVStoreRemoveParams
* @property {string} key - The key to update.
* @property {string[]} paths - List of period-joined paths to remove.
*
* @typedef {Object} KVStoreExpireAtParams
* @property {string} key - The key to set expiration for.
* @property {number} timestamp - UNIX timestamp (seconds) when the key should expire.
*
* @typedef {Object} KVStoreExpireParams
* @property {string} key - The key to set expiration for.
* @property {number} ttl - Time-to-live in seconds.
*/
/**
* Service for registering the puter-kvstore interface, exposing a simple key-value store API
* with support for get, set, delete, list, flush, increment, decrement, and key expiration.
* @extends BaseService
*/
class KVStoreInterfaceService extends BaseService {
/**
* Service class for managing KVStore interface registrations.
* Extends the base service to provide key-value store interface management.
*/
async '__on_driver.register.interfaces' () {
const svc_registry = this.services.get('registry');
const col_interfaces = svc_registry.get('interfaces');
// Register the puter-kvstore interface
col_interfaces.set('puter-kvstore', {
description: 'A simple key-value store.',
methods: {
get: {
description: 'Get a value by key.',
parameters: {
key: { type: 'json', required: true },
},
result: { type: 'json' },
},
set: {
description: 'Set a value by key.',
parameters: {
key: { type: 'string', required: true },
value: { type: 'json' },
expireAt: { type: 'number' },
},
result: { type: 'void' },
},
del: {
description: 'Delete a value by key.',
parameters: {
key: { type: 'string' },
},
result: { type: 'void' },
},
list: {
description: 'List key-value pairs with optional pagination.',
parameters: {
as: {
type: 'string',
},
pattern: {
type: 'string',
},
limit: {
type: 'number',
},
cursor: {
type: 'string',
},
},
result: { type: 'json' },
},
flush: {
description: 'Delete all key-value pairs.',
parameters: {},
result: { type: 'void' },
},
update: {
description: 'Update nested values by key.',
parameters: {
key: { type: 'string', required: true },
pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value' },
ttl: { type: 'number', description: 'optional TTL in seconds for the whole object' },
},
result: { type: 'json', description: 'The updated value' },
},
add: {
description: 'Append values into list paths by key.',
parameters: {
key: { type: 'string', required: true },
pathAndValueMap: { type: 'json', required: true, description: 'map of period-joined path to value to append' },
},
result: { type: 'json', description: 'The updated value' },
},
remove: {
description: 'Remove nested values by key.',
parameters: {
key: { type: 'string', required: true },
paths: { type: 'json', required: true, description: 'list of period-joined paths to remove' },
},
result: { type: 'json', description: 'The updated value' },
},
incr: {
description: 'Increment a value by key.',
parameters: {
key: { type: 'string', required: true },
pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' },
},
result: { type: 'json', description: 'The updated value' },
},
decr: {
description: 'Decrement a value by key.',
parameters: {
key: { type: 'string', required: true },
pathAndAmountMap: { type: 'json', required: true, description: 'map of period-joined path to amount to increment by' },
},
result: { type: 'json', description: 'The updated value' },
},
expireAt: {
description: 'Set a key to expire at a given timestamp in sec.',
parameters: {
key: { type: 'string', required: true },
timestamp: { type: 'number', required: true },
},
result: { type: 'number' },
},
expire: {
description: 'Set a key to expire in ttl many seconds.',
parameters: {
key: { type: 'string', required: true },
ttl: { type: 'number', required: true },
},
result: { type: 'number' },
},
},
});
}
}
module.exports = {
KVStoreInterfaceService,
};
================================================
FILE: src/backend/src/modules/kvstore/KVStoreModule.js
================================================
/*
* Copyright (C) 2025-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
_integrity_check () {
if ( config.env !== 'dev' ) {
// only check in debug mode since it's expensive
return;
}
// check the 2 maps are consistent
if ( this.entriesByPath.size !== this.entriesByUUID.size ) {
throw new Error('Path map and UUID map have different sizes');
}
for ( const [inner_path, uuid] of this.entriesByPath ) {
const entry = this.entriesByUUID.get(uuid);
// entry should exist
if ( ! entry ) {
throw new Error(`Entry ${uuid} does not exist`);
}
// path should match
if ( this._inner_path(entry.path) !== inner_path ) {
throw new Error(`Path ${inner_path} does not match entry ${uuid}`);
}
// uuid should match
if ( entry.uuid !== uuid ) {
throw new Error(`UUID ${uuid} does not match entry ${entry.uuid}`);
}
// parent should exist
if ( entry.parent_uid ) {
const parent_entry = this.entriesByUUID.get(entry.parent_uid);
if ( ! parent_entry ) {
throw new Error(`Parent ${entry.parent_uid} does not exist`);
}
}
// parent's path should be a prefix of the entry's path
if ( entry.parent_uid ) {
const parent_entry = this.entriesByUUID.get(entry.parent_uid);
if ( ! entry.path.startsWith(parent_entry.path) ) {
throw new Error(`Parent ${entry.parent_uid} path ${parent_entry.path} is not a prefix of entry ${entry.path}`);
}
}
// parent should be a directory
if ( entry.parent_uid ) {
const parent_entry = this.entriesByUUID.get(entry.parent_uid);
if ( ! parent_entry.is_dir ) {
throw new Error(`Parent ${entry.parent_uid} is not a directory`);
}
}
}
}
/**
* Check if a given node exists.
*
* @param {Object} param
* @param {NodePathSelector | NodeUIDSelector | NodeChildSelector | RootNodeSelector | NodeRawEntrySelector} param.selector - The selector used for checking.
* @returns {Promise} - True if the node exists, false otherwise.
*/
async quick_check ({ selector }) {
if ( selector instanceof NodePathSelector ) {
const inner_path = this._inner_path(selector.value);
return this.entriesByPath.has(inner_path);
}
if ( selector instanceof NodeUIDSelector ) {
return this.entriesByUUID.has(selector.value);
}
// fallback to stat
const entry = await this.stat({ selector });
return !!entry;
}
/**
* Performs a stat operation using the given selector.
*
* NB: Some returned fields currently contain placeholder values. And the
* `path` of the absolute path from the root.
*
* @param {Object} param
* @param {NodePathSelector | NodeUIDSelector | NodeChildSelector | RootNodeSelector | NodeRawEntrySelector} param.selector - The selector to stat.
* @returns {Promise} - The result of the stat operation, or `null` if the node doesn't exist.
*/
async stat ({ selector }) {
try_infer_attributes(selector);
let entry_uuid = null;
if ( selector instanceof NodePathSelector ) {
// stat by path
const inner_path = this._inner_path(selector.value);
entry_uuid = this.entriesByPath.get(inner_path);
} else if ( selector instanceof NodeUIDSelector ) {
// stat by uid
entry_uuid = selector.value;
} else if ( selector instanceof NodeChildSelector ) {
if ( selector.path ) {
// Shouldn't care about about parent when the "path" is present
// since it might have different provider.
return await this.stat({
selector: new NodePathSelector(selector.path),
});
} else {
// recursively stat the parent and then stat the child
const parent_entry = await this.stat({
selector: selector.parent,
});
if ( parent_entry ) {
const full_path = _path.join(parent_entry.path, selector.name);
return await this.stat({
selector: new NodePathSelector(full_path),
});
}
}
} else {
// other selectors shouldn't reach here, i.e., it's an internal logic error
throw APIError.create('invalid_node');
}
const entry = this.entriesByUUID.get(entry_uuid);
if ( ! entry ) {
return null;
}
// Return a copied entry with `full_path`, since external code only cares
// about full path.
const copied_entry = { ...entry };
copied_entry.path = _path.join(this.mountpoint, entry.path);
return copied_entry;
}
/**
* Read directory contents.
*
* @param {Object} param
* @param {Context} param.context - The context of the operation.
* @param {FSNodeContext} param.node - The directory node to read.
* @returns {Promise} - Array of child UUIDs.
*/
async readdir ({ context, node }) {
// prerequistes: get required path via stat
const entry = await this.stat({ selector: node.selector });
if ( ! entry ) {
throw APIError.create('invalid_node');
}
const inner_path = this._inner_path(entry.path);
const child_uuids = [];
// Find all entries that are direct children of this directory
for ( const [path, uuid] of this.entriesByPath ) {
if ( path === inner_path ) {
continue; // Skip the directory itself
}
const dirname = _path.dirname(path);
if ( dirname === inner_path ) {
child_uuids.push(uuid);
}
}
return child_uuids;
}
/**
* Create a new directory.
*
* @param {Object} param
* @param {Context} param.context - The context of the operation.
* @param {FSNodeContext} param.parent - The parent node to create the directory in. Must exist and be a directory.
* @param {string} param.name - The name of the new directory.
* @returns {Promise} - The new directory node.
*/
async mkdir ({ context, parent, name }) {
// prerequistes: get required path via stat
const parent_entry = await this.stat({ selector: parent.selector });
if ( ! parent_entry ) {
throw APIError.create('invalid_node');
}
const full_path = _path.join(parent_entry.path, name);
const inner_path = this._inner_path(full_path);
let entry = null;
if ( this.entriesByPath.has(inner_path) ) {
throw APIError.create('item_with_same_name_exists', null, {
entry_name: full_path,
});
} else {
entry = new MemoryFile({
path: inner_path,
is_dir: true,
content: null,
parent_uid: parent_entry.uuid,
});
this.entriesByPath.set(inner_path, entry.uuid);
this.entriesByUUID.set(entry.uuid, entry);
}
// create the node
const fs = context.get('services').get('filesystem');
const node = await fs.node(new NodeUIDSelector(entry.uuid));
await node.fetchEntry();
this._integrity_check();
return node;
}
/**
* Remove a directory.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNodeContext} param.node: The directory to remove.
* @param {Object} param.options: The options for the operation.
* @returns {Promise}
*/
async rmdir ({ context, node, options = {} }) {
this._integrity_check();
// prerequistes: get required path via stat
const entry = await this.stat({ selector: node.selector });
if ( ! entry ) {
throw APIError.create('invalid_node');
}
const inner_path = this._inner_path(entry.path);
// for mode: non-recursive
if ( ! options.recursive ) {
const children = await this.readdir({ context, node });
if ( children.length > 0 ) {
throw APIError.create('not_empty');
}
}
// remove all descendants
for ( const [other_inner_path, other_entry_uuid] of this.entriesByPath ) {
if ( other_entry_uuid === entry.uuid ) {
// skip the directory itself
continue;
}
if ( other_inner_path.startsWith(inner_path) ) {
this.entriesByPath.delete(other_inner_path);
this.entriesByUUID.delete(other_entry_uuid);
}
}
// for mode: non-descendants-only
if ( ! options.descendants_only ) {
// remove the directory itself
this.entriesByPath.delete(inner_path);
this.entriesByUUID.delete(entry.uuid);
}
this._integrity_check();
}
/**
* Remove a file.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNodeContext} param.node: The file to remove.
* @returns {Promise}
*/
async unlink ({ context, node }) {
// prerequistes: get required path via stat
const entry = await this.stat({ selector: node.selector });
if ( ! entry ) {
throw APIError.create('invalid_node');
}
const inner_path = this._inner_path(entry.path);
this.entriesByPath.delete(inner_path);
this.entriesByUUID.delete(entry.uuid);
}
/**
* Move a file.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNodeContext} param.node: The file to move.
* @param {FSNodeContext} param.new_parent: The new parent directory of the file.
* @param {string} param.new_name: The new name of the file.
* @param {Object} param.metadata: The metadata of the file.
* @returns {Promise}
*/
async move ({ context, node, new_parent, new_name, metadata }) {
// prerequistes: get required path via stat
const new_parent_entry = await this.stat({ selector: new_parent.selector });
if ( ! new_parent_entry ) {
throw APIError.create('invalid_node');
}
// create the new entry
const new_full_path = _path.join(new_parent_entry.path, new_name);
const new_inner_path = this._inner_path(new_full_path);
const entry = new MemoryFile({
path: new_inner_path,
is_dir: node.entry.is_dir,
content: node.entry.content,
parent_uid: new_parent_entry.uuid,
});
entry.uuid = node.entry.uuid;
this.entriesByPath.set(new_inner_path, entry.uuid);
this.entriesByUUID.set(entry.uuid, entry);
// remove the old entry
const inner_path = this._inner_path(node.path);
this.entriesByPath.delete(inner_path);
// NB: should not delete the entry by uuid because uuid does not change
// after the move.
this._integrity_check();
return entry;
}
/**
* Copy a tree of files and directories.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNodeContext} param.source - The source node to copy.
* @param {FSNodeContext} param.parent - The parent directory for the copy.
* @param {string} param.target_name - The name for the copied item.
* @returns {Promise} - The copied node.
*/
async copy_tree ({ context, source, parent, target_name }) {
const fs = context.get('services').get('filesystem');
if ( source.entry.is_dir ) {
// Create the directory
const new_dir = await this.mkdir({ context, parent, name: target_name });
// Copy all children
const children = await this.readdir({ context, node: source });
for ( const child_uuid of children ) {
const child_node = await fs.node(new NodeUIDSelector(child_uuid));
await child_node.fetchEntry();
const child_name = child_node.entry.name;
await this.copy_tree({
context,
source: child_node,
parent: new_dir,
target_name: child_name,
});
}
return new_dir;
} else {
// Copy the file
const new_file = await this.write_new({
context,
parent,
name: target_name,
file: { stream: { read: () => source.entry.content } },
});
return new_file;
}
}
/**
* Write a new file to the filesystem. Throws an error if the destination
* already exists.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNodeContext} param.parent: The parent directory of the destination directory.
* @param {string} param.name: The name of the destination directory.
* @param {Object} param.file: The file to write.
* @returns {Promise}
*/
async write_new ({ context, parent, name, file }) {
// prerequistes: get required path via stat
const parent_entry = await this.stat({ selector: parent.selector });
if ( ! parent_entry ) {
throw APIError.create('invalid_node');
}
const full_path = _path.join(parent_entry.path, name);
const inner_path = this._inner_path(full_path);
let entry = null;
if ( this.entriesByPath.has(inner_path) ) {
throw APIError.create('item_with_same_name_exists', null, {
entry_name: full_path,
});
} else {
entry = new MemoryFile({
path: inner_path,
is_dir: false,
content: file.stream.read(),
parent_uid: parent_entry.uuid,
});
this.entriesByPath.set(inner_path, entry.uuid);
this.entriesByUUID.set(entry.uuid, entry);
}
const fs = context.get('services').get('filesystem');
const node = await fs.node(new NodeUIDSelector(entry.uuid));
await node.fetchEntry();
this._integrity_check();
return node;
}
/**
* Overwrite an existing file. Throws an error if the destination does not
* exist.
*
* @param {Object} param
* @param {Context} param.context
* @param {FSNodeContext} param.node: The node to write to.
* @param {Object} param.file: The file to write.
* @returns {Promise}
*/
async write_overwrite ({ context, node, file }) {
const entry = await this.stat({ selector: node.selector });
if ( ! entry ) {
throw APIError.create('invalid_node');
}
const inner_path = this._inner_path(entry.path);
this.entriesByPath.set(inner_path, entry.uuid);
let original_entry = this.entriesByUUID.get(entry.uuid);
if ( ! original_entry ) {
throw new Error(`File ${entry.path} does not exist`);
} else {
if ( original_entry.is_dir ) {
throw new Error('Cannot overwrite a directory');
}
original_entry.content = file.stream.read();
original_entry.modified = Math.floor(Date.now() / 1000);
original_entry.size = original_entry.content ? original_entry.content.length : 0;
this.entriesByUUID.set(entry.uuid, original_entry);
}
const fs = context.get('services').get('filesystem');
node = await fs.node(new NodeUIDSelector(original_entry.uuid));
await node.fetchEntry();
this._integrity_check();
return node;
}
async read ({
context,
node,
}) {
// TODO: once MemoryFS aggregates its own storage, don't get it
// via mountpoint service.
const svc_mountpoint = context.get('services').get('mountpoint');
const storage = svc_mountpoint.get_storage(this.constructor.name);
const stream = (await storage.create_read_stream(await node.get('uid'), {
memory_file: node.entry,
}));
return stream;
}
}
module.exports = {
MemoryFSProvider,
};
================================================
FILE: src/backend/src/modules/puterfs/customfs/MemoryFSService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } codes - A map of error codes to error specifications
*/
register (codes) {
for ( const code in codes ) {
this.codes[code] = codes[code];
}
}
create (code, fields) {
const error_spec = this.codes[code];
if ( ! error_spec ) {
return new APIError(500, 'Missing error message.', null, {
code,
});
}
return new APIError(error_spec.status, error_spec.message, null, {
...fields,
code,
});
}
}
module.exports = APIErrorService;
================================================
FILE: src/backend/src/modules/web/README.md
================================================
# WebModule
This module initializes a pre-configured web server and socket.io server.
The main service, WebServerService, emits 'install.routes' and provides
the server instance to the callback.
## Services
### SocketioService
SocketioService provides a service for sending messages to clients.
socket.io is used behind the scenes. This service provides a simpler
interface for sending messages to rooms or socket ids.
#### Listeners
##### `install.socketio`
Initializes socket.io
###### Parameters
- **server:** The server to attach socket.io to.
### WebServerService
This class, WebServerService, is responsible for starting and managing the Puter web server.
It initializes the Express app, sets up middlewares, routes, and handles authentication and web sockets.
It also validates the host header and IP addresses to prevent security vulnerabilities.
#### Listeners
##### `boot.consolidation`
This method initializes the backend web server for Puter. It sets up the Express app, configures middleware, and starts the HTTP server.
##### `boot.activation`
Starts the web server and listens for incoming connections.
This method sets up the Express app, sets up middleware, and starts the server on the specified port.
It also sets up the Socket.io server for real-time communication.
##### `start.webserver`
This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use.
If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299.
Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events.
If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser.
## Notes
### Outside Imports
This module has external relative imports. When these are
removed it may become possible to move this module to an
extension.
**Imports:**
- `../../services/BaseService` (use.BaseService)
- `../../util/context.js`
- `../../services/BaseService.js`
- `../../config.js`
- `../../middleware/auth.js`
- `../../util/strutil.js`
- `../../helpers.js`
================================================
FILE: src/backend/src/modules/web/SocketioService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
async send (socket_specifiers, key, data) {
if ( ! Array.isArray(socket_specifiers) ) {
socket_specifiers = [socket_specifiers];
}
for ( const socket_specifier of socket_specifiers ) {
if ( socket_specifier.room ) {
this.io.to(socket_specifier.room).emit(key, data);
} else if ( socket_specifier.socket ) {
this.io.to(socket_specifier.socket).emit(key, data);
}
}
}
/**
* Checks if the specified socket or room exists
*
* @param {Object} socket_specifier - The socket specifier object
* @returns {boolean} True if the socket exists, false otherwise
*/
has (socket_specifier) {
if ( socket_specifier.room ) {
const room = this.io?.sockets.adapter.rooms.get(socket_specifier.room);
return (!!room) && room.size > 0;
}
if ( socket_specifier.socket ) {
return this.io?.sockets.sockets.has(socket_specifier.socket);
}
}
}
module.exports = SocketioService;
================================================
FILE: src/backend/src/modules/web/WebModule.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves once the server is started.
*/
async '__on_boot.activation' () {
const services = this.services;
await services.emit('start.webserver');
await services.emit('ready.webserver');
console.log('in case you care, ready.webserver hooks are done');
}
/**
* This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use.
* If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299.
* Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events.
* If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser.
*
* @return {Promise} A promise that resolves when the server is up and running.
*/
async '__on_start.webserver' () {
// error handling middleware goes last, as per the
// expressjs documentation:
// https://expressjs.com/en/guide/error-handling.html
this.app.use(require('./lib/api_error_handler.js'));
const { jwt_auth } = require('../../helpers.js');
config.http_port = process.env.PORT ?? config.http_port;
globalThis.deployment_type =
config.http_port === 5101 ? 'green' :
config.http_port === 5102 ? 'blue' :
'not production';
let server;
const auto_port = config.http_port === 'auto';
let ports_to_try = auto_port ? (() => {
const ports = [];
for ( let i = 0 ; i < 20 ; i++ ) {
ports.push(4100 + i);
}
return ports;
})() : [Number.parseInt(config.http_port)];
for ( let i = 0 ; i < ports_to_try.length ; i++ ) {
const port = ports_to_try[i];
const is_last_port = i === ports_to_try.length - 1;
if ( auto_port ) this.log.debug(`trying port: ${ port}`);
try {
server = http.createServer(this.app).listen(port);
server.timeout = 1000 * 60 * 60 * 2; // 2 hours
let should_continue = false;
await new Promise((rslv, rjct) => {
server.on('error', e => {
if ( e.code === 'EADDRINUSE' ) {
if ( !is_last_port && e.code === 'EADDRINUSE' ) {
this.log.info(`port in use: ${ port}`);
should_continue = true;
}
rslv();
} else {
rjct(e);
}
});
/**
* Starts the web server.
*
* This method is responsible for creating the HTTP server, setting up middleware, and starting the server on the specified port. If the specified port is "auto", it will attempt to find an available port within a range.
*
* @returns {Promise}
*/
// Add this comment above line 110
// (line 110 of the provided code)
server.on('listening', () => {
rslv();
});
});
if ( should_continue ) continue;
} catch (e) {
if ( !is_last_port && e.code === 'EADDRINUSE' ) {
this.log.info(`port in use:${ port}`);
continue;
}
throw e;
}
config.http_port = port;
break;
}
ports_to_try = null; // GC
const url = config.origin;
const args = yargs(hideBin(process.argv)).argv;
if ( args['server'] ) {
(async () => {
(await import('./../../../../../tools/auth_gui.js')).default(args['puter-backend']);
})();
config.no_browser_launch = true;
}
// Open the browser to the URL of Puter
// (if we are in development mode only)
if ( config.env === 'dev' && !config.no_browser_launch ) {
try {
const openModule = await import('open');
openModule.default(url);
} catch (e) {
console.log('Error opening browser', e);
}
}
const link = `\x1B[34;1m${url}\x1B[0m`;
const lines = [
`Puter is now live at: ${link}`,
`listening on port: ${config.http_port}`,
];
const realConsole = globalThis.original_console_object ?? console;
lines.forEach(line => realConsole.log(line));
realConsole.log('\n************************************************************');
realConsole.log(`* Puter is now live at: ${url}`);
realConsole.log('************************************************************');
server.timeout = 1000 * 60 * 60 * 2; // 2 hours
server.requestTimeout = 1000 * 60 * 60 * 2; // 2 hours
server.headersTimeout = 1000 * 60 * 60 * 2; // 2 hours
// server.keepAliveTimeout = 1000 * 60 * 60 * 2; // 2 hours
// Socket.io server instance
// const socketio = require('../../socketio.js').init(server);
// TODO: ^ Replace above line with the following code:
await this.services.emit('install.socketio', { server });
const socketio = this.services.get('socketio').io;
const authService = this.services.get('auth');
// Socket.io middleware for authentication
socketio.use(async (socket, next) => {
const authToken = socket.handshake?.auth?.auth_token;
if ( ! authToken ) {
next(new Error('socket auth token missing'));
return;
}
try {
const authRes = await jwt_auth(socket, authService);
// successful auth
socket.actor = authRes.actor;
socket.user = authRes.user;
socket.token = authRes.token;
// join user room
socket.join(socket.user.id);
// setTimeout 0 is needed because we need to send
// the notifications after this handler is done
// setTimeout(() => {
// }, 1000);
next();
} catch ( error ) {
console.warn('socket auth err', error);
const authError = error instanceof Error
? error
: new Error('socket auth failed');
next(authError);
}
});
const context = Context.get();
socketio.on('connection', (socket) => {
socket.on('disconnect', () => {
});
socket.on('trash.is_empty', (msg) => {
socket.broadcast.to(socket.user.id).emit('trash.is_empty', msg);
});
const svc_event = this.services.get('event');
svc_event.emit('web.socket.connected', {
socket,
user: socket.user,
});
socket.on('puter_is_actually_open', async (_msg) => {
await context.sub({
actor: socket.actor,
}).arun(async () => {
await svc_event.emit('web.socket.user-connected', {
socket,
user: socket.user,
});
});
});
});
this.server_ = server;
await this.services.emit('install.websockets');
}
/**
* Starts the Puter web server and sets up routes, middleware, and error handling.
*
* @param {object} services - An object containing all services available to the web server.
* @returns {Promise} A promise that resolves when the web server is fully started.
*/
get_server () {
return this.server_;
}
/**
* Handles starting and managing the Puter web server.
*
* @param {Object} services - An object containing all services.
*/
async _init () {
const app = express();
this.app = app;
app.set('services', this.services);
this.middlewares = { auth };
const require = this.require;
const config = this.global_config;
new ContextExpressMiddleware({
parent: globalThis.root_context.sub({
puter_environment: Context.create({
env: config.env,
version: relative_require('../../../package.json').version,
}),
}, 'mw'),
}).install(app);
app.use(async (req, res, next) => {
req.services = this.services;
next();
});
// When the user visits the main origin (not api/dav subdomain) with ?auth_token=
// (e.g. QR login), set the HTTP-only session cookie so user-protected endpoints work.
app.use(async (req, res, next) => {
const has_subdomain = req.hostname.slice(0, -1 * (config.domain.length + 1)) !== '';
if ( has_subdomain ) return next();
const token = req.query?.auth_token;
if ( !token || typeof token !== 'string' ) return next();
try {
const svc_auth = req.services.get('auth');
const cleanToken = token.replace('Bearer ', '').trim();
const actor = await svc_auth.authenticate_from_token(cleanToken);
const session_token = svc_auth.create_session_token_for_session(
actor.type.user,
actor.type.session,
);
res.cookie(config.cookie_name, session_token, {
sameSite: 'none',
secure: true,
httpOnly: true,
});
} catch ( e ) {
console.log('query auth token (QR Code login probably) failed');
console.error(e);
}
next();
});
// Measure data transfer amounts
app.use(measure());
// Instrument logging to use our log service
{
// Switch log function at config time; info log is configurable
const logfn = (config.logging ?? []).includes('http')
? (log, { message, fields }) => {
log.info(message);
log.debug(message, fields);
}
: (log, { message, fields }) => {
log.debug(message, fields);
};
const morgan = require('morgan');
const stream = {
write: (message) => {
const [method, url, status, responseTime] = message.split(' ');
const fields = {
method,
url,
status: parseInt(status, 10),
responseTime: parseFloat(responseTime),
};
if ( url.includes('android-icon') ) return;
// remove `puter.auth.*` query params
const safe_url = (u => {
// We need to prepend an arbitrary domain to the URL
const url = new URL(`https://example.com${ u}`);
const search = url.searchParams;
for ( const key of search.keys() ) {
if ( key.startsWith('puter.auth.') ) search.delete(key);
}
return `${url.pathname }?${ search.toString()}`;
})(fields.url);
fields.url = safe_url;
// re-write message
message = [
fields.method, fields.url,
fields.status, fields.responseTime,
].join(' ');
const log = this.services.get('log-service').create('morgan');
try {
this.context.arun(() => {
logfn(log, { message, fields });
});
} catch (e) {
console.log('failed to log this message properly:', message, fields);
console.error(e);
}
},
};
app.use(morgan(':method :url :status :response-time', { stream }));
}
/**
* Initialize the web server, start it, and handle any related logic.
*
* This method is responsible for creating the server and listening on the
* appropriate port. It also sets up middleware, routes, and other necessary
* configurations.
*
* @returns {Promise} A promise that resolves once the server is up and running.
*/
app.use((() => {
// const router = express.Router();
// router.get('/wut', express.json(), (req, res, next) => {
// return res.status(500).send('Internal Error');
// });
// return router;
return eggspress('/wut', {
allowedMethods: ['GET'],
}, async (req, res, _next) => {
// throw new Error('throwy error');
return res.status(200).send('test endpoint');
});
})());
(() => {
const onFinished = require('on-finished');
app.use((req, res, next) => {
onFinished(res, () => {
if ( res.statusCode !== 500 ) return;
if ( req.__error_handled ) return;
const alarm = this.services.get('alarm');
alarm.create('responded-500', 'server sent a 500 response', {
error: req.__error_source,
url: req.url,
method: req.method,
body: req.body,
headers: req.headers,
});
});
next();
});
})();
app.use(async function (req, res, next) {
// Express does not document that this can be undefined.
// The browser likely doesn't follow the HTTP/1.1 spec
// (bot client?) and express is handling this badly by
// not setting the header at all. (that's my theory)
if ( req.hostname === undefined ) {
res.status(400).send(
'Please verify your browser is up-to-date.',
);
return;
}
return next();
});
// Validate host header against allowed domains to prevent host header injection
// https://www.owasp.org/index.php/Host_Header_Injection
app.use((req, res, next) => {
const allowedDomains = new Set();
const pushAllowedDomain = (domain) => {
const normalizedDomain = normalizeHostDomain(domain);
if ( normalizedDomain ) {
allowedDomains.add(normalizedDomain);
}
};
const staticHostingDomain = normalizeHostDomain(config.static_hosting_domain);
pushAllowedDomain(config.domain);
pushAllowedDomain(staticHostingDomain);
pushAllowedDomain(config.static_hosting_domain_alt);
pushAllowedDomain(config.private_app_hosting_domain);
pushAllowedDomain(config.private_app_hosting_domain_alt);
if ( staticHostingDomain ) {
pushAllowedDomain(`at.${staticHostingDomain}`);
}
if ( config.allow_nipio_domains ) {
pushAllowedDomain('nip.io');
}
// Retrieve the Host header and ensure it's in a valid format
const hostHeader = req.headers.host;
if ( !config.allow_no_host_header && !hostHeader ) {
return res.status(400).send('Missing Host header.');
}
if ( config.allow_all_host_values ) {
next();
return;
}
// Parse the Host header to isolate the hostname (strip out port if present)
const hostName = hostHeader.split(':')[0].trim().toLowerCase();
// Check if the hostname matches any of the allowed domains or is a subdomain of an allowed domain
// Exception: allow /healthcheck endpoint on the root domain
if (
req.path === '/healthcheck'
) {
next();
return;
}
if ( [...allowedDomains].some(allowedDomain => hostMatchesDomain(hostName, allowedDomain)) ) {
next(); // Proceed if the host is valid
return;
} else {
if ( ! config.custom_domains_enabled ) {
res.status(400).send('Invalid Host header.');
return;
}
req.is_custom_domain = true;
next();
return;
}
});
// Validate IP with any IP checkers
app.use(async (req, res, next) => {
const svc_event = this.services.get('event');
const event = {
allow: true,
ip: req.headers?.['x-forwarded-for'] ||
req.connection?.remoteAddress,
};
if ( ! this.config.disable_ip_validate_event ) {
await svc_event.emit('ip.validate', event);
}
// rules that don't apply to notification endpoints
const undefined_origin_allowed = config.undefined_origin_allowed || this.allowedRoutesWithUndefinedOrigins.some(rule => {
if ( typeof rule === 'string' ) return rule === req.path;
return rule.test(req.path);
});
if ( ! undefined_origin_allowed ) {
// check if no origin
if ( req.method === 'POST' && req.headers.origin === undefined ) {
event.allow = false;
}
}
if ( ! event.allow ) {
return res.status(403).send('Forbidden');
}
next();
});
// Web hooks need a router that occurs before JSON parse middleware
// so that signatures of the raw JSON can be verified
this.router_webhooks = express.Router();
app.use(this.router_webhooks);
app.use((req, res, next) => {
if ( req.get('x-amz-sns-message-type') ) {
req.headers['content-type'] = 'application/json';
}
next();
});
const rawBodyBuffer = (req, res, buf, encoding) => {
req.rawBody = buf.toString(encoding || 'utf8');
};
app.use(express.json({ limit: '50mb', verify: rawBodyBuffer }));
app.use((req, res, next) => {
if ( req.headers['content-type']?.startsWith('application/json')
&& req.body
&& Buffer.isBuffer(req.body)
) {
try {
req.rawBody = req.body;
req.body = JSON.parse(req.body.toString('utf8'));
} catch {
return res.status(400).send({
error: {
message: 'Invalid JSON body',
},
});
}
}
next();
});
const cookieParser = require('cookie-parser');
app.use(cookieParser({ limit: '50mb' }));
// gzip compression for all requests
const compression = require('compression');
app.use(compression());
// Helmet and other security
const helmet = require('helmet');
app.use(helmet.noSniff());
app.use(helmet.hsts());
app.use(helmet.ieNoOpen());
app.use(helmet.permittedCrossDomainPolicies());
app.use(helmet.xssFilter());
// app.use(helmet.referrerPolicy());
app.disable('x-powered-by');
// remove object and array query parameters
app.use(function (req, res, next) {
for ( let k in req.query ) {
if ( req.query[k] === undefined || req.query[k] === null ) {
continue;
}
const allowed_types = ['string', 'number', 'boolean'];
if ( ! allowed_types.includes(typeof req.query[k]) ) {
req.query[k] = undefined;
}
}
next();
});
const uaParser = require('ua-parser-js');
app.use(function (req, res, next) {
const ua_header = req.headers['user-agent'];
const ua = uaParser(ua_header);
req.ua = ua;
next();
});
app.use(function (req, res, next) {
req.co_isolation_enabled =
['Chrome', 'Edge'].includes(req.ua.browser.name)
&& (Number(req.ua.browser.major) >= 110);
next();
});
app.use(function (req, res, next) {
const origin = req.headers.origin;
const subdomain = req.subdomains[req.subdomains.length - 1];
const isApiOrDavRequest =
config.experimental_no_subdomain ||
subdomain === 'api' ||
subdomain === 'dav';
const isCrossOriginAuthRoute =
req.path === '/signup' ||
req.path === '/login' ||
req.path.startsWith('/extensions/') ||
req.path.startsWith('/auth/oidc');
const is_site =
hostMatchesDomain(req.hostname, config.static_hosting_domain) ||
hostMatchesDomain(req.hostname, config.static_hosting_domain_alt) ||
hostMatchesDomain(req.hostname, config.private_app_hosting_domain) ||
hostMatchesDomain(req.hostname, config.private_app_hosting_domain_alt);
req.hostname === 'docs.puter.com'
;
const is_popup = !!req.query.embedded_in_popup;
const is_parent_co = !!req.query.cross_origin_isolated;
const is_app = !!req.query['puter.app_instance_id'];
const co_isolation_okay =
(!is_popup || is_parent_co) &&
(is_app || !is_site) &&
req.co_isolation_enabled
;
if ( isCrossOriginAuthRoute || isApiOrDavRequest ) {
res.setHeader('Access-Control-Allow-Origin', origin ?? '*');
if ( origin ) {
res.vary('Origin');
}
}
// Allow browser credentials on API/DAV cross-origin requests.
if ( isApiOrDavRequest && origin ) {
res.setHeader('Access-Control-Allow-Credentials', 'true');
}
// Request methods to allow
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS, PUT, PATCH, DELETE, PROPFIND, PROPPATCH, MKCOL, COPY, MOVE, LOCK, UNLOCK');
const allowed_headers = [
'Origin', 'X-Requested-With', 'Content-Type', 'Accept', 'Authorization', 'sentry-trace', 'baggage',
'Depth', 'Destination', 'Overwrite', 'If', 'Lock-Token', 'DAV', 'stripe-signature',
];
// Request headers to allow
res.header('Access-Control-Allow-Headers', allowed_headers.join(', '));
// Needed for SharedArrayBuffer
// NOTE: This is put behind a configuration flag because we
// need some experimentation to ensure the interface
// between apps and Puter doesn't break.
if ( config.cross_origin_isolation && co_isolation_okay ) {
res.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
res.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
}
res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');
// Pass to next layer of middleware
// disable iframes on the main domain
if ( req.hostname === config.domain ) {
// disable iframes
res.setHeader('X-Frame-Options', 'SAMEORIGIN');
}
next();
});
}
}
module.exports = WebServerService;
================================================
FILE: src/backend/src/modules/web/lib/__lib__.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } Query predicate object
*/
async create_predicate (id, ...args) {
if ( id === 'user-can-edit' ) {
return new Eq({
key: 'owner',
value: Context.get('user').id,
});
}
if ( id === 'name-like' ) {
return new Like({
key: 'name',
value: args[0],
});
}
},
async delete (uid, _extra) {
const svc_appInformation = this.context.get('services').get('app-information');
await svc_appInformation.delete_app(uid);
},
async read (uid) {
if ( typeof uid !== 'string' || !uid ) {
return await this.upstream.read(uid);
}
const canonicalUidAliasPromise = this.read_canonical_app_uid_alias_(uid);
const entity = await this.upstream.read(uid);
if ( entity ) {
return entity;
}
const canonicalUid = await canonicalUidAliasPromise;
if ( !canonicalUid || canonicalUid === uid ) {
return null;
}
return await this.upstream.read(canonicalUid);
},
/**
* Filters app selection based on user permissions and visibility settings
* @param {Object} options - Selection options including predicates
* @returns {Promise} Filtered selection results
*/
async select (options) {
const actor = Context.get('actor');
const user = actor.type.user;
const additional = [];
// An app is also allowed to read itself
if ( actor.type instanceof AppUnderUserActorType ) {
additional.push(new Eq({
key: 'uid',
value: actor.type.app.uid,
}));
}
options.predicate = options.predicate.and(new Or({
children: [
new Eq({
key: 'approved_for_listing',
value: 1,
}),
new Eq({
key: 'owner',
value: user.id,
}),
...additional,
],
}));
return await this.upstream.select(options);
},
/**
* Creates or updates an application with proper name handling and associations
* @param {Object} entity - Application entity to upsert
* @param {Object} extra - Additional upsert parameters
* @returns {Promise} Upsert operation results
*/
async upsert (entity, extra) {
extra = extra || {};
const actor = Context.get('actor');
const user = actor?.type?.user;
const preJoinFullEntity = extra.old_entity
? await (await extra.old_entity.clone()).apply(entity)
: entity
;
await this.ensurePuterSiteSubdomainIsOwned(preJoinFullEntity, extra, user);
await this.maybe_join_owned_hosted_index_url_app_on_create_(entity, extra, user);
const full_entity = extra.old_entity
? await (await extra.old_entity.clone()).apply(entity)
: entity
;
await this.ensureIndexUrlUnique(full_entity, extra);
if ( await app_name_exists(await entity.get('name')) ) {
const { old_entity } = extra;
const is_name_change = ( !old_entity ) ||
( await old_entity.get('name') !== await entity.get('name') );
if ( is_name_change && extra?.options?.dedupe_name ) {
const base = await entity.get('name');
let number = 1;
while ( await app_name_exists(`${base}-${number}`) ) {
number++;
}
await entity.set('name', `${base}-${number}`);
}
else if ( is_name_change ) {
// The name might be taken because it's the old name
// of this same app. If it is, the app takes it back.
const svc_oldAppName = this.context.get('services').get('old-app-name');
const name_info = await svc_oldAppName.check_app_name(await entity.get('name'));
if ( !name_info || name_info.app_uid !== await entity.get('uid') ) {
// Throw error because the name really is taken
throw APIError.create('app_name_already_in_use', null, {
name: await entity.get('name'),
});
}
// Remove the old name from the old-app-name service
await svc_oldAppName.remove_name(name_info.id);
} else {
entity.del('name');
}
}
const subdomain_id = await this.maybe_insert_subdomain_(entity);
const result = await this.upstream.upsert(entity, extra);
const { insert_id } = result;
const oldAssociations = await this.db.read(
'SELECT type FROM app_filetype_association WHERE app_id = ?',
[insert_id],
);
const normalizedOldAssociations = oldAssociations
.map(row => String(row.type ?? '').trim().toLowerCase().replace(/^\./, ''))
.filter(Boolean);
// Remove old file associations (if applicable)
if ( extra.old_entity ) {
await this.db.write(
'DELETE FROM app_filetype_association WHERE app_id = ?',
[insert_id],
);
}
// Add file associations (if applicable)
const filetype_associations = await entity.get('filetype_associations');
const normalizedNewAssociations = (filetype_associations ?? [])
.map(association => String(association).trim().toLowerCase().replace(/^\./, ''))
.filter(Boolean);
if ( (a => a && a.length > 0)(filetype_associations) ) {
const stmt =
'INSERT INTO app_filetype_association ' +
`(app_id, type) VALUES ${
normalizedNewAssociations.map(() => '(?, ?)').join(', ')}`;
const rows = normalizedNewAssociations.map(a => [insert_id, a]);
await this.db.write(stmt, rows.flat());
}
const affectedAssociationExtensions = new Set([
...normalizedOldAssociations,
...normalizedNewAssociations,
]);
if ( affectedAssociationExtensions.size ) {
await deleteRedisKeys(Array.from(affectedAssociationExtensions)
.map(ext => AppRedisCacheSpace.associationAppsKey(ext)));
}
const has_new_icon =
( !extra.old_entity ) || (
await entity.get('icon') !== await extra.old_entity.get('icon')
);
if ( has_new_icon ) {
const svc_event = this.context.get('services').get('event');
const event = {
app_uid: await entity.get('uid'),
data_url: await entity.get('icon'),
url: '',
};
await svc_event.emit('app.new-icon', event);
if ( typeof event.url === 'string' && event.url ) {
this.db.write(
'UPDATE apps SET icon = ? WHERE id = ? LIMIT 1',
[event.url, insert_id],
);
await entity.set('icon', event.url);
}
}
const has_new_name =
extra.old_entity && (
await entity.get('name') !== await extra.old_entity.get('name')
);
if ( has_new_name ) {
const svc_event = this.context.get('services').get('event');
const event = {
app_uid: await entity.get('uid'),
new_name: await entity.get('name'),
old_name: await extra.old_entity.get('name'),
};
await svc_event.emit('app.rename', event);
}
// Associate app with subdomain (if applicable)
if ( subdomain_id ) {
await this.db.write(
'UPDATE subdomains SET associated_app_id = ? WHERE id = ?',
[insert_id, subdomain_id],
);
}
if ( extra.old_entity ) {
const svc_event = this.context.get('services').get('event');
const [app] = await this.db.read(
'SELECT * FROM apps WHERE uid = ? LIMIT 1',
[await full_entity.get('uid')],
);
const old_app = {
uid: await extra.old_entity.get('uid'),
index_url: await extra.old_entity.get('index_url'),
};
await svc_event.emit('app.changed', {
app_uid: await full_entity.get('uid'),
action: 'updated',
app,
old_app,
});
}
if ( extra.joined_source_app_uid ) {
await this.write_canonical_app_uid_alias_({
oldAppUid: extra.joined_source_app_uid,
canonicalAppUid: await full_entity.get('uid'),
});
const svc_appInformation = this.context.get('services').get('app-information');
if ( svc_appInformation?.delete_app ) {
await svc_appInformation.delete_app(extra.joined_source_app_uid, undefined, {
preserveCanonicalUidAlias: true,
});
}
}
if ( typeof extra.joined_requested_name === 'string' && extra.joined_requested_name.trim() ) {
const renameResult = await this.apply_joined_requested_name_({
canonicalUid: await full_entity.get('uid'),
requestedName: extra.joined_requested_name,
});
if ( renameResult ) {
const svc_event = this.context.get('services').get('event');
await svc_event.emit('app.rename', {
app_uid: await full_entity.get('uid'),
old_name: renameResult.oldName,
new_name: renameResult.newName,
});
await full_entity.set('name', renameResult.newName);
}
}
return result;
},
async retry_predicate_rewrite ({ predicate }) {
const recurse = async (predicate) => {
if ( predicate instanceof Or ) {
return new Or({
children: await Promise.all(predicate.children.map(recurse)),
});
}
if ( predicate instanceof And ) {
return new And({
children: await Promise.all(predicate.children.map(recurse)),
});
}
if ( predicate instanceof Eq ) {
if ( predicate.key === 'name' ) {
const svc_oldAppName = this.context.get('services').get('old-app-name');
const name_info = await svc_oldAppName.check_app_name(predicate.value);
return new Eq({
key: 'uid',
value: name_info?.app_uid,
});
}
}
};
return await recurse(predicate);
},
async queueIconMigration (entity) {
if ( ! this.pending_icon_migrations_ ) {
this.pending_icon_migrations_ = new Set();
}
const migration_key = entity.private_meta?.mysql_id ?? Symbol('app-icon-migration');
if ( this.pending_icon_migrations_.has(migration_key) ) {
return;
}
this.pending_icon_migrations_.add(migration_key);
Promise.resolve().then(async () => {
const icon = await entity.get('icon');
if ( typeof icon !== 'string' || !icon.startsWith('data:') ) {
return;
}
const app_uid = await entity.get('uid');
if ( ! app_uid ) {
return;
}
const svc_event = this.context.get('services').get('event');
const event = {
app_uid,
data_url: icon,
};
await svc_event.emit('app.new-icon', event);
if ( typeof event.url !== 'string' || !event.url ) return;
await this.db.write(
'UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1',
[event.url, app_uid],
);
}).catch(e => {
const svc_error = this.context.get('services').get('error-service');
svc_error.report('AppES:queue_icon_migration', { source: e });
}).finally(() => {
this.pending_icon_migrations_.delete(migration_key);
});
},
/**
* Transforms app data before reading by adding associations and handling permissions
* @param {Object} entity - App entity to transform
*/
async read_transform (entity) {
const {
getActorUserUid,
resolvePrivateLaunchAccess,
} = await getPrivateLaunchAccessModule();
const services = this.context.get('services');
const actor = Context.get('actor');
const esParams = Context.get('es_params') ?? {};
const appUid = await entity.get('uid');
const appName = await entity.get('name');
const appIndexUrl = await entity.get('index_url');
const appCreatedAt = await entity.get('created_at');
const appIsPrivate = await entity.get('is_private');
const appInformationService = services.get('app-information');
const authService = services.get('auth');
const statsPromise = appInformationService
? appInformationService.get_stats(appUid, {
period: esParams.stats_period,
grouping: esParams.stats_grouping,
created_at: appCreatedAt,
})
: Promise.resolve(undefined);
const fileAssociationsPromise = this.db.read(
'SELECT type FROM app_filetype_association WHERE app_id = ?',
[entity.private_meta.mysql_id],
);
const createdFromOriginPromise = (async () => {
if ( ! authService ) return null;
try {
const origin = origin_from_url(appIndexUrl);
const expectedUid = await authService.app_uid_from_origin(origin);
return expectedUid === appUid ? origin : null;
} catch {
// This happens when index_url is not a valid URL.
return null;
}
})();
const privateAccessPromise = resolvePrivateLaunchAccess({
app: {
uid: appUid,
name: appName,
is_private: appIsPrivate,
},
services,
userUid: getActorUserUid(actor),
source: 'driverRead',
args: esParams,
});
const [
fileAssociationRows,
stats,
createdFromOrigin,
privateAccess,
] = await Promise.all([
fileAssociationsPromise,
statsPromise,
createdFromOriginPromise,
privateAccessPromise,
]);
await entity.set(
'filetype_associations',
fileAssociationRows.map(row => row.type),
);
await entity.set('stats', stats);
await entity.set('created_from_origin', createdFromOrigin);
await entity.set('privateAccess', privateAccess);
// Migrate b64 icons to the filesystem-backed icon flow without blocking reads.
this.queueIconMigration(entity);
// Check if the user is the owner
const is_owner = await (async () => {
let owner = await entity.get('owner');
// TODO: why does this happen?
if ( typeof owner === 'number' ) {
owner = { id: owner };
}
if ( ! owner ) return false;
const actor = Context.get('actor');
return actor.type.user.id === owner.id;
})();
// Remove fields that are not allowed for non-owners
if ( ! is_owner ) {
entity.del('approved_for_listing');
entity.del('approved_for_opening_items');
entity.del('approved_for_incentive_program');
}
// Replace icon if an icon size is specified
const iconSize = Context.get('es_params')?.icon_size;
if ( iconSize ) {
const svc_appIcon = this.context.get('services').get('app-icon');
try {
const iconPath = svc_appIcon.getAppIconPath({
appUid: await entity.get('uid'),
size: iconSize,
});
if ( iconPath ) {
await entity.set('icon', iconPath);
}
} catch (e) {
const svc_error = this.context.get('services').get('error-service');
svc_error.report('AppES:read_transform', { source: e });
}
}
},
/**
* Creates a subdomain entry for the app if required
* @param {Object} entity - App entity
* @returns {Promise} Subdomain ID if created
* @private
*/
async maybe_insert_subdomain_ (entity) {
// Create and update is a situation where we might create a subdomain
let subdomain_id;
if ( await entity.get('source_directory') ) {
await (await entity.get('source_directory')
).fetchEntry();
const subdomain = await entity.get('subdomain');
const user = Context.get('user');
let subdomain_res = await this.db.write(
`INSERT ${this.db.case({
mysql: 'IGNORE',
sqlite: 'OR IGNORE',
})} INTO subdomains
(subdomain, user_id, root_dir_id, uuid) VALUES
( ?, ?, ?, ?)`,
[
//subdomain
subdomain,
//user_id
user.id,
//root_dir_id
(await entity.get('source_directory')).mysql_id,
//uuid, `sd` stands for subdomain
`sd-${ uuidv4()}`,
],
);
subdomain_id = subdomain_res.insertId;
}
return subdomain_id;
},
/**
* Ensures that when an app uses a puter.site subdomain as its index_url,
* the subdomain belongs to the user creating/updating the app.
*/
async ensurePuterSiteSubdomainIsOwned (entity, extra, user) {
if ( ! user ) return;
// Only enforce when the index_url is being set or changed
const new_index_url = await entity.get('index_url');
if ( ! new_index_url ) return;
if ( extra.old_entity ) {
const old_index_url = await extra.old_entity.get('index_url');
if ( old_index_url === new_index_url ) {
return;
}
}
const subdomain = extractPuterHostedSubdomainFromIndexUrl(new_index_url);
if ( ! subdomain ) return;
const svc_puterSite = this.context.get('services').get('puter-site');
const site = await svc_puterSite.get_subdomain(subdomain, { is_custom_domain: false });
if ( !site || site.user_id !== user.id ) {
throw APIError.create('subdomain_not_owned', null, { subdomain });
}
},
is_puter_hosted_index_url_ (index_url) {
return !!extractPuterHostedSubdomainFromIndexUrl(index_url);
},
build_equivalent_index_url_candidates_ (index_url) {
if ( typeof index_url !== 'string' || !index_url.trim() ) {
return [];
}
try {
const parsedUrl = new URL(index_url);
const origin = `${parsedUrl.protocol}//${parsedUrl.host.toLowerCase()}`;
const pathname = parsedUrl.pathname || '/';
const values = new Set();
if ( pathname === '/' || pathname.toLowerCase() === '/index.html' ) {
values.add(origin);
values.add(`${origin}/`);
values.add(`${origin}/index.html`);
} else {
const normalizedPath = pathname.endsWith('/')
? pathname.slice(0, -1)
: pathname;
values.add(`${origin}${normalizedPath}`);
values.add(`${origin}${normalizedPath}/`);
}
return [...values];
} catch {
return [index_url.trim()];
}
},
async find_index_url_conflict_ ({ indexUrl, excludeMysqlId }) {
if ( ! this.is_puter_hosted_index_url_(indexUrl) ) {
return null;
}
const candidates = this.build_equivalent_index_url_candidates_(indexUrl);
if ( candidates.length === 0 ) return null;
if ( hasIndexUrlUniquenessExemption(candidates) ) return null;
const placeholders = candidates.map(() => '?').join(', ');
const parameters = [...candidates];
let query = `SELECT id, uid, owner_user_id, index_url FROM apps WHERE index_url IN (${placeholders})`;
if ( Number.isInteger(excludeMysqlId) && excludeMysqlId > 0 ) {
query += ' AND id != ?';
parameters.push(excludeMysqlId);
}
query += ' ORDER BY timestamp ASC, id ASC LIMIT 1';
const rows = await this.db.read(query, parameters);
const normalizedExcludeMysqlId = Number(excludeMysqlId);
const conflictRow = rows.find(row => {
if (
Number.isInteger(normalizedExcludeMysqlId)
&& normalizedExcludeMysqlId > 0
&& Number(row?.id) === normalizedExcludeMysqlId
) {
return false;
}
if ( typeof row?.index_url === 'string' ) {
return candidates.includes(row.index_url);
}
return true;
});
return conflictRow || null;
},
async resolve_entity_mysql_id_ (entity) {
const directMysqlId = Number(entity?.private_meta?.mysql_id);
if ( Number.isInteger(directMysqlId) && directMysqlId > 0 ) {
return directMysqlId;
}
if ( !entity || typeof entity.get !== 'function' ) {
return undefined;
}
const uid = await entity.get('uid');
if ( typeof uid !== 'string' || !uid ) {
return undefined;
}
const rows = await this.db.read(
'SELECT id FROM apps WHERE uid = ? LIMIT 1',
[uid],
);
const mysqlId = Number(rows?.[0]?.id);
if ( Number.isInteger(mysqlId) && mysqlId > 0 ) {
return mysqlId;
}
return undefined;
},
async claim_app_ownership_by_id_for_user_ ({ appId, userId }) {
if ( !Number.isInteger(appId) || appId <= 0 ) return;
if ( !Number.isInteger(userId) || userId <= 0 ) return;
await this.db.write(
'UPDATE apps SET owner_user_id = ? WHERE id = ? AND owner_user_id IS NULL',
[userId, appId],
);
},
build_canonical_app_uid_alias_key_ (oldAppUid) {
return `${APP_UID_ALIAS_KEY_PREFIX}:${oldAppUid}`;
},
build_canonical_app_uid_alias_reverse_key_ (canonicalAppUid) {
return `${APP_UID_ALIAS_REVERSE_KEY_PREFIX}:${canonicalAppUid}`;
},
normalize_canonical_alias_uid_list_ (value) {
if ( ! Array.isArray(value) ) return [];
const normalizedList = [];
const seen = new Set();
for ( const item of value ) {
if ( typeof item !== 'string' || !item ) continue;
if ( seen.has(item) ) continue;
seen.add(item);
normalizedList.push(item);
}
return normalizedList;
},
async read_canonical_app_uid_alias_ (oldAppUid) {
if ( typeof oldAppUid !== 'string' || !oldAppUid ) return null;
const services = this.context.get('services');
const kvStore = services.get('puter-kvstore');
const suService = services.get('su');
if ( !kvStore || typeof kvStore.get !== 'function' ) return null;
if ( !suService || typeof suService.sudo !== 'function' ) return null;
const key = this.build_canonical_app_uid_alias_key_(oldAppUid);
try {
const canonicalAppUid = await suService.sudo(() => kvStore.get({ key }));
if ( typeof canonicalAppUid === 'string' && canonicalAppUid ) {
return canonicalAppUid;
}
} catch {
// Alias reads are best-effort.
}
return null;
},
async write_canonical_app_uid_alias_ ({ oldAppUid, canonicalAppUid }) {
if ( typeof oldAppUid !== 'string' || !oldAppUid ) return;
if ( typeof canonicalAppUid !== 'string' || !canonicalAppUid ) return;
if ( oldAppUid === canonicalAppUid ) return;
const services = this.context.get('services');
const kvStore = services.get('puter-kvstore');
const suService = services.get('su');
if ( !kvStore || typeof kvStore.set !== 'function' ) return;
if ( !suService || typeof suService.sudo !== 'function' ) return;
const key = this.build_canonical_app_uid_alias_key_(oldAppUid);
const reverseKey = this.build_canonical_app_uid_alias_reverse_key_(canonicalAppUid);
const expireAt = Math.floor(Date.now() / 1000) + APP_UID_ALIAS_TTL_SECONDS;
try {
await suService.sudo(async () => {
const reverseValue = await kvStore.get({ key: reverseKey });
const reverseAliases = this.normalize_canonical_alias_uid_list_(reverseValue);
if ( ! reverseAliases.includes(oldAppUid) ) {
reverseAliases.push(oldAppUid);
}
await kvStore.set({
key,
value: canonicalAppUid,
expireAt,
});
await kvStore.set({
key: reverseKey,
value: reverseAliases,
expireAt,
});
});
} catch {
// Alias writes are best-effort.
}
},
async maybe_join_owned_hosted_index_url_app_on_create_ (entity, extra, user) {
if ( ! user ) return;
const new_index_url = await entity.get('index_url');
const source_entity = extra.old_entity;
const currentMysqlId = await this.resolve_entity_mysql_id_(extra.old_entity);
const conflictRow = await this.find_index_url_conflict_({
indexUrl: new_index_url,
excludeMysqlId: currentMysqlId,
});
if ( ! conflictRow ) return;
const conflictOwnerUserId = Number(conflictRow.owner_user_id);
if (
Number.isInteger(conflictOwnerUserId)
&& conflictOwnerUserId > 0
&& conflictOwnerUserId !== user.id
) {
throw APIError.create('app_index_url_already_in_use', null, {
index_url: new_index_url,
app_uid: conflictRow.uid,
});
}
if ( !Number.isInteger(conflictOwnerUserId) || conflictOwnerUserId <= 0 ) {
await this.claim_app_ownership_by_id_for_user_({
appId: conflictRow.id,
userId: user.id,
});
}
const old_entity = await this.upstream.read(conflictRow.uid);
const owner = await old_entity?.get('owner');
let ownerUserId = owner?.id ?? owner;
if ( owner instanceof Entity ) {
ownerUserId = owner.private_meta.mysql_id;
}
ownerUserId = Number(ownerUserId);
if ( !old_entity || !Number.isInteger(ownerUserId) || ownerUserId !== user.id ) {
throw APIError.create('app_index_url_already_in_use', null, {
index_url: new_index_url,
app_uid: conflictRow.uid,
});
}
if (
Number.isInteger(conflictOwnerUserId)
&& conflictOwnerUserId === user.id
&& !await this.is_origin_bootstrap_app_entity_(old_entity)
) {
// Prevent merging arbitrary same-owner apps; only allow the
// auto-created origin bootstrap app to be absorbed.
throw APIError.create('app_index_url_already_in_use', null, {
index_url: new_index_url,
app_uid: conflictRow.uid,
});
}
if ( source_entity ) {
const sourceUid = await source_entity.get('uid');
const targetUid = await old_entity.get('uid');
const requestedName = await entity.get('name');
if (
sourceUid
&& targetUid
&& sourceUid !== targetUid
&& requestedName !== undefined
) {
entity.del('name');
if ( typeof requestedName === 'string' && requestedName.trim() ) {
extra.joined_requested_name = requestedName.trim();
}
}
if ( sourceUid && targetUid && sourceUid !== targetUid ) {
extra.joined_source_app_uid = sourceUid;
}
}
await entity.set('uid', await old_entity.get('uid'));
extra.old_entity = old_entity;
},
async apply_joined_requested_name_ ({ canonicalUid, requestedName }) {
if ( typeof canonicalUid !== 'string' || !canonicalUid ) return null;
if ( typeof requestedName !== 'string' || !requestedName.trim() ) return null;
const normalizedName = requestedName.trim();
const currentRows = await this.db.read(
'SELECT name FROM apps WHERE uid = ? LIMIT 1',
[canonicalUid],
);
const currentName = currentRows?.[0]?.name;
if ( typeof currentName !== 'string' ) return null;
if ( currentName === normalizedName ) return null;
const conflictRows = await this.db.read(
'SELECT uid FROM apps WHERE name = ? AND uid != ? LIMIT 1',
[normalizedName, canonicalUid],
);
if ( conflictRows.length > 0 ) {
throw APIError.create('app_name_already_in_use', null, {
name: normalizedName,
});
}
await this.db.write(
'UPDATE apps SET name = ? WHERE uid = ? LIMIT 1',
[normalizedName, canonicalUid],
);
return {
oldName: currentName,
newName: normalizedName,
};
},
async is_origin_bootstrap_app_entity_ (entity) {
if ( ! entity ) return false;
const uid = await entity.get('uid');
if ( typeof uid !== 'string' || !uid ) return false;
if ( await entity.get('name') !== uid ) return false;
if ( await entity.get('title') !== uid ) return false;
const description = await entity.get('description');
if ( typeof description !== 'string' ) return false;
return description.startsWith('App created from origin ');
},
async ensureIndexUrlUnique (entity, extra) {
const new_index_url = await entity.get('index_url');
if ( ! new_index_url ) return;
if ( ! this.is_puter_hosted_index_url_(new_index_url) ) return;
if ( extra.old_entity ) {
const old_index_url = await extra.old_entity.get('index_url');
if ( old_index_url === new_index_url ) {
return;
}
}
const currentMysqlId = await this.resolve_entity_mysql_id_(extra.old_entity);
const conflictRow = await this.find_index_url_conflict_({
indexUrl: new_index_url,
excludeMysqlId: currentMysqlId,
});
if ( conflictRow ) {
throw APIError.create('app_index_url_already_in_use', null, {
index_url: new_index_url,
app_uid: conflictRow.uid,
});
}
},
};
}
module.exports = AppES;
================================================
FILE: src/backend/src/om/entitystorage/AppLimitedES.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see Log in ';
// docs
h += '';
h += `${config.protocol}://docs.${config.domain}/ `;
h += ' ';
// apps
// TODO: use service for app discovery
let apps = await db.read('SELECT * FROM apps WHERE approved_for_listing = 1');
if ( apps.length > 0 ) {
for ( let i = 0; i < apps.length; i++ ) {
const app = apps[i];
h += '';
h += `${config.protocol}://${config.domain}/app/${app.name} `;
h += ' ';
}
}
h += ' ';
res.set('Content-Type', 'application/xml');
return res.send(h);
}
else if ( path === '/unsubscribe' ) {
let h = '';
if ( req.query.user_uuid === undefined )
{
h += 'user_uuid is required
';
}
else {
// modules
const { get_user } = require('../helpers');
// get user
const user = await get_user({ uuid: req.query.user_uuid });
// more validation
if ( ! user )
{
h += 'User not found.
';
}
else if ( user.unsubscribed === 1 )
{
h += 'You are already unsubscribed.
';
}
// mark user as confirmed
else {
await db.write(
'UPDATE `user` SET `unsubscribed` = 1 WHERE id = ?',
[user.id],
);
invalidate_cached_user(user);
// return results
h += 'Your have successfully unsubscribed from all emails.
';
}
}
h += '';
res.send(h);
}
else if ( path === '/confirm-email-by-token' ) {
let h = '';
if ( req.query.user_uuid === undefined )
{
h += 'user_uuid is required
';
}
else if ( req.query.token === undefined )
{
h += 'token is required
';
}
else {
// modules
const { get_user } = require('../helpers');
// get user
const user = await get_user({ uuid: req.query.user_uuid, force: true });
// more validation
if ( user === undefined || user === null || user === false )
{
h += 'user not found.
';
}
else if ( user.email_confirmed === 1 )
{
h += 'Email already confirmed.
';
}
else if ( user.email_confirm_token !== req.query.token )
{
h += 'invalid token.
';
}
// mark user as confirmed
else {
// This IIFE is here to return early on conditions, and
// avoid further nested branching. This is a temporary
// solution; next time this code should be refactored.
await (async () => {
const svc_cleanEmail = req.services.get('clean-email');
const clean_email = svc_cleanEmail.clean(user.email);
// If other users have the same CONFIRMED email, display an error
const maybe_rows = await db.read(
`SELECT EXISTS(
SELECT 1 FROM user WHERE (email=? OR clean_email=?)
AND email_confirmed=1
AND password IS NOT NULL
) AS email_exists`,
[user.email, clean_email],
);
if ( maybe_rows[0]?.email_exists ) {
// TODO: maybe display the username of that account
h += '' +
'This email was confirmed on a different account.
';
return;
}
// If other users have the same unconfirmed email, revoke it
await db.write(
'UPDATE `user` SET `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `unconfirmed_change_email` = ?',
[user.email],
);
// update user
await db.write(
'UPDATE `user` SET `email_confirmed` = 1, `requires_email_confirmation` = 0 WHERE id = ?',
[user.id],
);
invalidate_cached_user(user);
// send realtime success msg to client
const svc_socketio = req.services.get('socketio');
svc_socketio.send({ room: user.id }, 'user.email_confirmed', {});
// return results
h += 'Your email has been successfully confirmed.
';
const svc_event = req.services.get('event');
svc_event.emit('user.email-confirmed', {
user_uid: user.uuid,
email: user.email,
});
})();
}
}
h += '';
res.send(h);
}
// ------------------------
// /assets/
// ------------------------
else if ( path.startsWith('/assets/') ) {
path = PathBuilder.resolve(path);
return res.sendFile(path, { root: `${__dirname }../../public` }, function (err) {
if ( err && err.statusCode ) {
return res.status(err.statusCode).send('Error /public/');
}
});
}
// ------------------------
// GUI
// ------------------------
else {
let app;
let canonical_url = config.origin + path;
let app_name, app_title, app_description, app_icon, app_social_media_image;
let launch_options = {
on_initialized: [],
};
// default title
app_title = config.title;
// /action/
if ( path.startsWith('/action/') || path.startsWith('/@') ) {
path = '/';
}
// /settings
else if ( path.startsWith('/settings') ) {
path = '/';
}
// /dashboard
else if ( path === '/dashboard' || path === '/dashboard/' ) {
path = '/';
}
// /app/
else if ( path.startsWith('/app/') ) {
app_name = path.replace('/app/', '');
app = await get_app({
follow_old_names: true,
name: app_name,
});
if ( app ) {
// parse app metadata if available
app.metadata = parseMetadata(app.metadata);
// set app attributes to be passed to the homepage service
app_title = app.title;
app_description = app.description;
app_icon = app.icon;
app_social_media_image = app.metadata?.social_image;
}
// 404 - Not found!
else if ( app_name ) {
app_title = app_name.charAt(0).toUpperCase() + app_name.slice(1);
res.status(404);
}
path = '/';
}
else if ( path.startsWith('/show/') ) {
const filepath = path.slice('/show'.length);
launch_options.on_initialized.push({
$: 'window-call',
fn_name: 'launch_app',
args: [{
name: 'explorer',
path: filepath,
}],
});
path = '/';
}
// index.js
if ( path === '/' ) {
const svc_puterHomepage = Context.get('services').get('puter-homepage');
return svc_puterHomepage.send({ req, res }, {
title: app_title,
description: app_description || config.short_description,
short_description: app_description || config.short_description,
social_media_image: app_social_media_image || config.social_media_image,
company: 'Puter Technologies Inc.',
canonical_url: canonical_url,
icon: app_icon,
app: app,
}, launch_options);
}
// /dist/...
else if ( path.startsWith('/dist/') || path.startsWith('/src/') ) {
path = PathBuilder.resolve(path);
return res.sendFile(path, { root: config.assets.gui }, function (err) {
if ( err && err.statusCode ) {
return res.status(err.statusCode).send('Error /gui/dist/');
}
});
}
// All other paths
else {
path = PathBuilder.resolve(path);
return res.sendFile(path, { root: _path.join(config.assets.gui, 'src') }, function (err) {
if ( err && err.statusCode ) {
return res.status(err.statusCode).send('Error /gui/');
}
});
}
}
}
// --------------------------------------
// Native Apps
// --------------------------------------
else if ( subdomain === 'viewer' || subdomain === 'editor' || subdomain === 'about' || subdomain === 'docs' ||
subdomain === 'player' || subdomain === 'pdf' || subdomain === 'code' || subdomain === 'markus' ||
subdomain === 'draw' || subdomain === 'camera' || subdomain === 'recorder' ||
subdomain === 'dev-center' || subdomain === 'developer' ) {
let root = PathBuilder
.add(__dirname)
.add(config.defaultjs_asset_path, { allow_traversal: true })
.add('apps').add(subdomain)
.build();
const has_dist = ['docs', 'developer'];
if ( has_dist.includes(subdomain) ) {
root += '/dist';
}
root = _path.normalize(root);
path = _path.normalize(path);
const real_path = _path.normalize(_path.join(root, path));
// Determine if the path is a directory
// (necessary because otherwise res.sendFile() will HANG!)
try {
const is_dir = (await _fs.promises.stat(real_path)).isDirectory();
if ( is_dir && !path.endsWith('/') ) {
// Redirect to directory (use 307 to avoid browser caching)
path += '/';
let redirect_url = `${req.protocol }://${ req.get('host') }${path}`;
// We need to add the query string to the redirect URL
if ( req.query ) {
const old_url = `${req.protocol }://${ req.get('host') }${req.originalUrl}`;
redirect_url += new URL(old_url).search;
}
return res.redirect(307, redirect_url);
}
} catch (e) {
console.error(e);
return res.status(404).send('Not found');
}
try {
return res.sendFile(path, { root }, function (err) {
if ( err && err.statusCode ) {
return res.status(err.statusCode).send('Error /apps/');
}
});
} catch (e) {
console.error('error from sendFile', e);
return res.status(e.statusCode).send('Error /apps/');
}
}
// --------------------------------------
// WWW, redirect to root domain
// --------------------------------------
else if ( subdomain === 'www' ) {
return res.redirect(config.origin);
}
//------------------------------------------
// User-defined subdomains: *.puter.com
// redirect to static hosting domain *.puter.site
//------------------------------------------
else {
if ( req.get('host').toLowerCase().endsWith(config.domain) ) {
return res.redirect(302, `${req.protocol }://${ req.get('host').replace(config.domain, config.static_hosting_domain) }${req.originalUrl}`);
// replace hostname with static hosting domain and redirect to the same path
}
}
});
module.exports.catchAllRouter = router;
================================================
FILE: src/backend/src/routers/apps.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see Re-validated Re-validated. Closing…
`);
});
export default router;
================================================
FILE: src/backend/src/routers/auth/request-app-root-dir.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see Your email has been successfully confirmed.
';
return res.send(h);
});
module.exports = app => {
app.use(CHANGE_EMAIL_CONFIRM);
};
================================================
FILE: src/backend/src/routers/change_username.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see Puter Driver API
`);
});
================================================
FILE: src/backend/src/routers/file.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see Sign In Required | ${safeAppTitle}
Sign in required
${safeAppName} requires Puter authentication before private files can load.
Click “Sign In with Puter” to continue.
Sign In with Puter
Retry
`);
res.status(200);
res.set('Cache-Control', 'no-store');
res.set('X-Robots-Tag', 'noindex, nofollow');
setReferrerPolicyHeader(res);
appendLinkHeader(
res,
marketplaceAppUrl ? `<${marketplaceAppUrl}>; rel="canonical"` : null,
);
res.set('Content-Type', 'text/html; charset=UTF-8');
return res.send(loginHtml);
}
async function evaluatePrivateAppAccess ({ req, res, services, app, requestPath }) {
const identity = await resolvePrivateIdentity({
req,
services,
appUid: app.uid,
});
if ( ! identity.userUid ) {
logPrivateAccessEvent('private_access.auth_required', {
appUid: app.uid,
userUid: null,
requestHost: req.hostname,
requestPath,
source: identity.source,
hasPrivateCookie: identity.hasPrivateCookie,
hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie,
});
respondPrivateLoginBootstrap({ res, app });
return false;
}
const eventService = services.get('event');
const accessCheckEvent = {
appUid: app.uid,
userUid: identity.userUid ?? null,
requestHost: req.hostname,
requestPath,
result: {
allowed: false,
},
};
try {
await eventService.emit('app.privateAccess.check', accessCheckEvent);
} catch (e) {
logPrivateAccessEvent('private_access.entitlement_check_error', {
appUid: app.uid,
userUid: identity.userUid ?? null,
requestHost: req.hostname,
requestPath,
source: identity.source,
error: e?.message || String(e),
});
console.error('private app access check failed', e);
}
if ( ! accessCheckEvent.result.allowed ) {
const redirectUrl = getPrivateDeniedRedirectUrl(
app,
accessCheckEvent.result.redirectUrl,
);
logPrivateAccessEvent('private_access.denied', {
appUid: app.uid,
userUid: identity.userUid ?? null,
requestHost: req.hostname,
requestPath,
source: identity.source,
reason: accessCheckEvent.result.reason ?? null,
redirectUrl,
hasPrivateCookie: identity.hasPrivateCookie,
hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie,
});
const marketplaceAppUrl = getMarketplaceAppUrl(app);
appendLinkHeader(
res,
marketplaceAppUrl ? `<${marketplaceAppUrl}>; rel="alternate"` : null,
);
res.redirect(redirectUrl);
return false;
}
const shouldRefreshPrivateCookie = identity.userUid && !identity.hasValidPrivateCookie;
if ( identity.userUid && !identity.hasValidPrivateCookie ) {
const authService = services.get('auth');
const privateToken = authService.createPrivateAssetToken({
appUid: identity.tokenAppUid || app.uid,
userUid: identity.userUid,
sessionUuid: identity.sessionUuid,
subdomain: identity.subdomain,
privateHost: identity.privateHost,
});
res.cookie(
authService.getPrivateAssetCookieName(),
privateToken,
authService.getPrivateAssetCookieOptions({
requestHostname: req.hostname,
}),
);
const sanitizedUrl = stripBootstrapAuthTokenFromOriginalUrl(req.originalUrl);
const shouldKeepBootstrapTokenInUrl = hasAppInstanceIdQueryParam(req);
if ( sanitizedUrl && !shouldKeepBootstrapTokenInUrl ) {
logPrivateAccessEvent('private_access.allowed_cookie_redirect', {
appUid: app.uid,
userUid: identity.userUid ?? null,
requestHost: req.hostname,
requestPath,
source: identity.source,
redirectUrl: sanitizedUrl,
});
res.redirect(sanitizedUrl);
return false;
}
if ( sanitizedUrl && shouldKeepBootstrapTokenInUrl ) {
logPrivateAccessEvent('private_access.allowed_cookie_redirect_skipped_for_app_instance', {
appUid: app.uid,
userUid: identity.userUid ?? null,
requestHost: req.hostname,
requestPath,
source: identity.source,
redirectUrl: sanitizedUrl,
});
}
}
logPrivateAccessEvent('private_access.allowed', {
appUid: app.uid,
userUid: identity.userUid ?? null,
requestHost: req.hostname,
requestPath,
source: identity.source,
cookieRefreshed: !!shouldRefreshPrivateCookie,
hasPrivateCookie: identity.hasPrivateCookie,
hasInvalidPrivateCookie: identity.hasInvalidPrivateCookie,
});
return true;
}
async function runInternal (req, res, next) {
const isPrivateHostedRequest = hostMatchesPrivateDomain(req.hostname);
const subdomain =
req.is_custom_domain && !isPrivateHostedRequest ? req.hostname :
req.subdomains[0] === 'devtest' ? 'devtest' :
getSubdomainFromHostedRequest(req);
let path = (req.baseUrl + req.path) || 'index.html';
const context = Context.get();
const services = context.get('services');
const getUsernameSite = (async () => {
if ( ! subdomain.endsWith('.at') ) return;
const parts = subdomain.split('.');
if ( parts.length !== 2 ) return;
const username = parts[0];
if ( ! username.match(usernameRegex) ) {
return;
}
const filesystemService = services.get('filesystem');
const indexNode = await filesystemService.node(new NodePathSelector(`/${username}/Public/index.html`));
const node = await filesystemService.node(new NodePathSelector(`/${username}/Public`));
if ( ! await indexNode.exists() ) return;
return {
name: `${username }.at`,
uuid: uuidv5(username, AT_DIRECTORY_NAMESPACE),
root_dir_id: await node.get('mysql-id'),
};
});
if ( req.hostname === staticHostingDomain || req.hostname === staticHostingDomainAlt || subdomain === 'www' ) {
// redirect to information page about static hosting
return res.redirect(staticHostingBaseDomainRedirect);
}
const site =
await getUsernameSite() ||
await (async () => {
const puterSiteService = services.get('puter-site');
const site = await puterSiteService.get_subdomain(subdomain, {
is_custom_domain: req.is_custom_domain && !isPrivateHostedRequest,
});
return site;
})();
if ( site === null ) {
return res.status(404).send('Subdomain not found');
}
const subdomainOwner = await get_user({ id: site.user_id });
if ( subdomainOwner?.suspended ) {
// This used to be "401 Account suspended", but this implies
// the client user is suspended, which is not the case.
// Instead we simply return 404, indicating that this page
// doesn't exist without further specifying that the owner's
// account is suspended. (the client user doesn't need to know)
return res.status(404).send('Subdomain not found');
}
const associatedApp = site.associated_app_id
? await get_app({ id: site.associated_app_id })
: null;
const privateApp = await resolvePrivateAppForHostedSite({
req,
site,
services,
associatedApp,
});
const privateAppEnabled = isPrivateApp(privateApp);
const privateAccessGateEnabled = isPrivateAccessGateEnabled();
if ( privateAppEnabled ) {
setReferrerPolicyHeader(res);
}
if (
privateAccessGateEnabled
&& privateAppEnabled
&& !hostMatchesPrivateDomain(req.hostname)
) {
const privateHostRedirect = buildPrivateHostRedirectUrl(req, privateApp);
if ( privateHostRedirect ) {
logPrivateAccessEvent('private_access.host_redirect', {
appUid: privateApp?.uid ?? null,
requestHost: req.hostname,
requestPath: req.path,
redirectUrl: privateHostRedirect,
});
const marketplaceAppUrl = getMarketplaceAppUrl(privateApp);
appendLinkHeader(
res,
marketplaceAppUrl
? `<${marketplaceAppUrl}>; rel="alternate"`
: null,
);
return res.redirect(privateHostRedirect);
}
logPrivateAccessEvent('private_access.host_mismatch_denied', {
appUid: privateApp?.uid ?? null,
requestHost: req.hostname,
requestPath: req.path,
});
return res.status(403).send('Private app host mismatch');
}
if (
site.associated_app_id &&
!privateAppEnabled &&
!req.query['puter.app_instance_id'] &&
( path === '' || path.endsWith('/') )
) {
const app = associatedApp || await get_app({ id: site.associated_app_id });
return res.redirect(`${originUrl}/app/${app.name}/`);
}
if ( path === '' ) path += '/index.html';
else if ( path.endsWith('/') ) path += 'index.html';
const resolvedUrlPath =
resolve('/', path);
const filesystemService = services.get('filesystem');
let subdomainRootPath = '';
if ( site.root_dir_id !== null && site.root_dir_id !== undefined ) {
const node = await filesystemService.node(new NodeInternalIDSelector('mysql', site.root_dir_id));
if ( ! await node.exists() ) {
return res.status(502).send('subdomain is pointing to deleted directory');
}
if ( await node.get('type') !== TYPE_DIRECTORY ) {
return res.status(502).send('subdomain is pointing to non-directory');
}
// Verify subdomain owner permission
const subdomainActor = Actor.adapt(subdomainOwner);
const aclService = services.get('acl');
if ( ! await aclService.check(subdomainActor, node, 'read') ) {
res.status(502).send('subdomain owner does not have access to directory');
return;
}
subdomainRootPath = await node.get('path');
}
if ( ! subdomainRootPath ) {
return respondHtmlError({
html: dedent(`
Subdomain or site is not pointing to a directory.
`),
}, req, res, next);
}
if ( !subdomainRootPath || subdomainRootPath === '/' ) {
throw APIError.create('forbidden');
}
req.__puterSiteRootPath = subdomainRootPath;
if ( ! privateAppEnabled ) {
try {
const actorContextReady = await evaluatePublicHostedActorContext({
req,
res,
services,
appUid: privateApp?.uid || associatedApp?.uid,
});
if ( ! actorContextReady ) return;
} catch (e) {
logPrivateAccessEvent('public_actor.evaluate_failed', {
appUid: privateApp?.uid || associatedApp?.uid || null,
requestHost: req.hostname,
reason: getPrivateAccessRejectionReason(e),
});
}
}
if ( privateAccessGateEnabled && privateAppEnabled ) {
const accessAllowed = await evaluatePrivateAppAccess({
req,
res,
services,
app: privateApp,
requestPath: req.path,
});
if ( ! accessAllowed ) return;
}
const filepath = subdomainRootPath + decodeURIComponent(resolvedUrlPath);
const targetNode = await filesystemService.node(new NodePathSelector(filepath));
await targetNode.fetchEntry();
if ( ! await targetNode.exists() ) {
return await respond404({ path }, req, res, next, subdomainRootPath);
}
const targetIsDir = await targetNode.get('type') === TYPE_DIRECTORY;
if ( targetIsDir && !resolvedUrlPath.endsWith('/') ) {
return res.redirect(`${resolvedUrlPath }/`);
}
if ( targetIsDir ) {
return await respond404({ path }, req, res, next, subdomainRootPath);
}
const contentType = contentTypeFromMime(await targetNode.get('name'));
res.set('Content-Type', contentType);
const aclConfig = {
no_acl: true,
actor: null,
};
if ( site.protected ) {
const authService = req.services.get('auth');
const getSiteActorFromToken = async () => {
const siteToken = req.cookies['puter.site.token'];
if ( ! siteToken ) return;
let failed = false;
let siteActor;
try {
siteActor =
await authService.authenticate_from_token(siteToken);
} catch (e) {
failed = true;
}
if ( failed ) return;
if ( ! siteActor ) return;
// security measure: if 'puter.site.token' is set
// to a different actor type, someone is likely
// trying to exploit the system.
if ( ! (siteActor.type instanceof SiteActorType) ) {
return;
}
aclConfig.actor = siteActor;
// Refresh the token if it's been 30 seconds since
// the last request
if (
(Date.now() - siteActor.type.iat * 1000)
>
1000 * 30
) {
const siteToken = authService.get_site_app_token({
site_uid: site.uuid,
});
res.cookie('puter.site.token', siteToken);
}
return true;
};
const makeSiteActorFromAppToken = async () => {
const token = req.query['puter.auth.token'];
aclConfig.no_acl = false;
if ( ! token ) {
const e = APIError.create('token_missing');
return respondError({ req, res, e });
}
const appActor =
await authService.authenticate_from_token(token);
const userActor =
appActor.get_related_actor(UserActorType);
const permissionService = req.services.get('permission');
const perm = await (async () => {
if ( userActor.type.user.id === site.user_id ) {
return {};
}
const reading = await permissionService.scan(userActor, `site:uid#${site.uuid}:access`);
const options = PermissionUtil.reading_to_options(reading);
return options.length > 0;
})();
if ( ! perm ) {
const e = APIError.create('forbidden');
respondError({ req, res, e });
return false;
}
const siteActor = await Actor.create(SiteActorType, { site });
aclConfig.actor = siteActor;
// This subdomain is allowed to keep the site actor token,
// so we send it here as a cookie so other html files can
// also load.
const siteToken = authService.get_site_app_token({
site_uid: site.uuid,
});
res.cookie('puter.site.token', siteToken);
return true;
};
let ok = await getSiteActorFromToken();
if ( ! ok ) {
ok = await makeSiteActorFromAppToken();
}
if ( ! ok ) return;
Object.freeze(aclConfig);
}
// Helper function to parse Range header
const parseRangeHeader = (rangeHeader) => {
// Check if this is a multipart range request
if ( rangeHeader.includes(',') ) {
// For now, we'll only serve the first range in multipart requests
// as the underlying storage layer doesn't support multipart responses
const firstRange = rangeHeader.split(',')[0].trim();
const matches = firstRange.match(/bytes=(\d+)-(\d*)/);
if ( ! matches ) return null;
const start = parseInt(matches[1], 10);
const end = matches[2] ? parseInt(matches[2], 10) : null;
return { start, end, isMultipart: true };
}
// Single range request
const matches = rangeHeader.match(/bytes=(\d+)-(\d*)/);
if ( ! matches ) return null;
const start = parseInt(matches[1], 10);
const end = matches[2] ? parseInt(matches[2], 10) : null;
return { start, end, isMultipart: false };
};
if ( req.headers['range'] ) {
res.status(206);
// Parse the Range header and set Content-Range
const rangeInfo = parseRangeHeader(req.headers['range']);
if ( rangeInfo ) {
const { start, end, isMultipart } = rangeInfo;
// For open-ended ranges, we need to calculate the actual end byte
let actualEnd = end;
let fileSize = null;
try {
fileSize = await targetNode.get('size');
if ( end === null ) {
actualEnd = fileSize - 1; // File size is 1-based, end byte is 0-based
}
} catch (e) {
// If we can't get file size, we'll let the storage layer handle it
// and not set Content-Range header
actualEnd = null;
fileSize = null;
}
if ( actualEnd !== null ) {
const totalSize = fileSize !== null ? fileSize : '*';
const contentRange = `bytes ${start}-${actualEnd}/${totalSize}`;
res.set('Content-Range', contentRange);
}
// If this was a multipart request, modify the range header to only include the first range
if ( isMultipart ) {
req.headers['range'] = end !== null
? `bytes=${start}-${end}`
: `bytes=${start}-`;
}
}
} else {
if ( targetNode.entry.size ) {
res.set('x-expected-entity-length', targetNode.entry.size);
}
}
res.set({ 'Accept-Ranges': 'bytes' });
const llRead = new LLRead();
// const actor = Actor.adapt(req.user);
const stream = await llRead.run({
no_acl: aclConfig.no_acl,
actor: aclConfig.actor,
fsNode: targetNode,
...(req.headers['range'] ? { range: req.headers['range'] } : { }),
});
// Destroy the stream if the client disconnects
req.on('close', () => {
stream.destroy();
});
try {
return stream.pipe(res);
} catch (e) {
const handled = await respondSiteError({
path,
req,
res,
next,
subdomainRootPath,
});
if ( handled ) return;
return res.status(500).send(`Error reading file: ${ e.message}`);
}
}
async function respondSiteError ({ path, html, req, res, next, subdomainRootPath }) {
const handled = await maybeRespondWithSiteConfig({
path,
html,
req,
res,
next,
subdomainRootPath,
errorStatus: 500,
});
return handled;
}
async function getSiteErrorConfig (req, subdomainRootPath) {
if ( ! subdomainRootPath ) return null;
req.__puterSiteErrorConfigCache ??= Object.create(null);
if ( req.__puterSiteErrorConfigCache[subdomainRootPath] !== undefined ) {
return req.__puterSiteErrorConfigCache[subdomainRootPath];
}
try {
const context = Context.get();
const services = context.get('services');
const filesystemService = services.get('filesystem');
const configPath = `${subdomainRootPath}/${puterSiteConfigFilename}`;
const configNode = await filesystemService.node(new NodePathSelector(configPath));
await configNode.fetchEntry();
if ( ! await configNode.exists() ) {
req.__puterSiteErrorConfigCache[subdomainRootPath] = null;
return null;
}
if ( await configNode.get('type') === TYPE_DIRECTORY ) {
req.__puterSiteErrorConfigCache[subdomainRootPath] = null;
return null;
}
const size = Number(await configNode.get('size') ?? 0);
if ( Number.isFinite(size) && size > puterSiteConfigMaxSize ) {
req.__puterSiteErrorConfigCache[subdomainRootPath] = null;
return null;
}
const llRead = new LLRead();
const stream = await llRead.run({
no_acl: true,
actor: null,
fsNode: configNode,
});
const buffer = await streamToBuffer(stream);
const text = buffer.toString('utf8');
const parsed = parseSiteErrorConfig(text);
req.__puterSiteErrorConfigCache[subdomainRootPath] = parsed;
return parsed;
} catch {
req.__puterSiteErrorConfigCache[subdomainRootPath] = null;
return null;
}
}
async function getSiteFileNode (subdomainRootPath, sitePath) {
const context = Context.get();
const services = context.get('services');
const filesystemService = services.get('filesystem');
const fullPath = `${subdomainRootPath}${sitePath}`;
const node = await filesystemService.node(new NodePathSelector(fullPath));
await node.fetchEntry();
if ( ! await node.exists() ) return null;
if ( await node.get('type') === TYPE_DIRECTORY ) return null;
return node;
}
async function maybeRespondWithSiteConfig ({
path,
html,
req,
res,
next,
subdomainRootPath,
errorStatus,
}) {
if ( ! subdomainRootPath ) return false;
const parsedConfig = await getSiteErrorConfig(req, subdomainRootPath);
if ( ! parsedConfig ) return false;
const rule = getSiteErrorRule(parsedConfig, errorStatus);
if ( ! rule ) return false;
const responseStatus = rule.status ?? errorStatus;
if ( rule.file ) {
const node = await getSiteFileNode(subdomainRootPath, rule.file);
if ( node ) {
await streamSiteFile({
req,
res,
fsNode: node,
status: responseStatus,
});
return true;
}
}
if ( rule.status !== null && rule.status !== undefined ) {
respondHtmlError({ path, html, status: responseStatus }, req, res, next);
return true;
}
return false;
}
async function streamSiteFile ({ req, res, fsNode, status }) {
res.status(status);
const contentType =
contentTypeFromMime(await fsNode.get('name')) ||
'application/octet-stream';
res.set('Content-Type', contentType);
const llRead = new LLRead();
const stream = await llRead.run({
no_acl: true,
actor: null,
fsNode,
});
req.on('close', () => {
stream.destroy();
});
return stream.pipe(res);
}
async function respond404 ({ path, html }, req, res, next, subdomainRootPath) {
const handled = await maybeRespondWithSiteConfig({
path,
html,
req,
res,
next,
subdomainRootPath,
errorStatus: 404,
});
if ( handled ) return;
if ( subdomainRootPath ) {
const custom404Node = await getSiteFileNode(subdomainRootPath, '/404.html');
if ( custom404Node ) {
return streamSiteFile({
req,
res,
fsNode: custom404Node,
status: 404,
});
}
}
return respondHtmlError({ path, html, status: 404 }, req, res, next);
}
function respondHtmlError ({ path, html, status = 404 }, req, res, _next) {
res.status(status);
res.set('Content-Type', 'text/html; charset=UTF-8');
res.write(``);
res.write(`
${status} `);
res.write('
');
if ( status === 404 && path ) {
if ( path === '/index.html' ) {
res.write('index.html Not Found');
} else {
res.write('Not Found');
}
} else {
res.write(html || 'Request failed');
}
res.write('
');
res.write('
${user.username} . Please check your email for instructions on how to reset your password.` });
}
else
{
return res.send({ message: `Password recovery email sent to ${user.email} . Please check your email for instructions on how to reset your password.` });
}
} catch (e) {
console.log(e);
return res.status(400).send(e);
}
});
module.exports = router;
================================================
FILE: src/backend/src/routers/set-desktop-bg.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } The created user, or null on failure (e.g. email already registered).
*/
async function signup_create_new_user (services, options) {
const { providerId, userinfo } = options;
if ( !providerId || !userinfo ) {
// Form signup: to be refactored from signup.js; not implemented here yet.
return null;
}
const db = await services.get('database').get(DB_WRITE, 'auth');
const svc_group = services.get('group');
const svc_user = services.get('user');
const svc_oidc = services.get('oidc');
if ( ! svc_oidc ) return null;
const claims = userinfo;
let username = (claims.name || claims.email || '').toString().trim();
if ( username ) {
username = username.replace(/\s+/g, '_').replace(/[^a-zA-Z0-9_-]/g, '');
if ( username.length > 45 ) username = username.slice(0, 45);
}
if ( !username || !/^\w+$/.test(username) ) {
let candidate;
do {
candidate = generate_identifier();
const [r] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [candidate]);
if ( ! r ) username = candidate;
} while ( !username );
} else {
const [existing] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [username]);
if ( existing ) {
let suffix = 1;
while ( true ) {
const candidate = `${username}${suffix}`;
const [r] = await db.pread('SELECT 1 FROM user WHERE username = ? LIMIT 1', [candidate]);
if ( ! r ) {
username = candidate; break;
}
suffix++;
}
}
}
const email = (claims.email || '').toString().trim() || null;
const clean_email = email ? email.toLowerCase().trim() : null;
if ( clean_email ) {
const [existingEmail] = await db.pread('SELECT 1 FROM user WHERE clean_email = ? LIMIT 1', [clean_email]);
if ( existingEmail ) {
return null; // email already registered; caller should return error
}
}
const user_uuid = uuidv4();
const email_confirm_code = String(Math.floor(100000 + Math.random() * 900000));
const email_confirm_token = uuidv4();
await db.write(`INSERT INTO user (
username, email, clean_email, password, uuid, referrer,
email_confirm_code, email_confirm_token, free_storage,
referred_by, email_confirmed, requires_email_confirmation
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[
username,
email,
clean_email,
null,
user_uuid,
null,
email_confirm_code,
email_confirm_token,
config.storage_capacity,
null,
1,
0,
]);
const [inserted] = await db.pread('SELECT id FROM user WHERE uuid = ? LIMIT 1', [user_uuid]);
const user_id = inserted.id;
await svc_oidc.linkProviderToUser(user_id, providerId, claims.sub, null);
await svc_group.add_users({
uid: config.default_user_group,
users: [username],
});
const [user] = await db.pread('SELECT * FROM user WHERE id = ? LIMIT 1', [user_id]);
if ( user && user.metadata && typeof user.metadata === 'string' ) {
user.metadata = JSON.parse(user.metadata);
} else if ( user && !user.metadata ) {
user.metadata = {};
}
await svc_user.generate_default_fsentries({ user });
return user;
}
export default signup_create_new_user;
================================================
FILE: src/backend/src/routers/sites.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see (
name: T
): T extends `${infer R extends keyof ServicesMap}`
? ServicesMap[R]
: unknown;
};
config: Record & { services?: Record; server_id?: string };
name?: string;
args?: any;
context: { get (key: string): any };
}
export type EventHandler = (id: string, ...args: any[]) => any;
export interface Logger {
debug: (...args: any[]) => any;
info: (...args: any[]) => any;
[key: string]: any;
}
export class BaseService {
constructor (service_resources: ServiceResources, ...a: any[]);
args: any;
service_name: string;
services: ServiceResources['services'];
config: Record;
global_config: ServiceResources['config'];
context: ServiceResources['context'];
log: Logger;
errors: any;
as (interfaceName: string): Record;
run_as_early_as_possible (): Promise;
construct (): Promise;
init (): Promise;
__on (id: string, args: any[]): Promise;
protected __get_event_handler (id: string): EventHandler;
protected _run_as_early_as_possible? (args?: any): any;
protected _construct? (args?: any): any;
protected _init? (args?: any): any;
protected _get_merged_static_object? (key: string): Record;
static LOG_DEBUG?: boolean;
static CONCERN?: string;
}
export default BaseService;
================================================
FILE: src/backend/src/services/BaseService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves when construction is complete.
*/
async construct () {
const useapi = this.context.get('useapi');
const use = this._get_merged_static_object('USE');
for ( const [key, value] of Object.entries(use) ) {
this[key] = useapi.use(value);
}
await (this._construct || NOOP).call(this, this.args);
}
/**
* Performs the initialization phase of the service lifecycle.
* This method sets up logging and error handling for the service,
* then calls the service-specific initialization logic if defined.
*
* @async
* @memberof BaseService
* @instance
* @returns {Promise} A promise that resolves when initialization is complete.
*/
async init () {
const services = this.services;
const log_fields = {};
if ( this.constructor.CONCERN ) {
log_fields.concern = this.constructor.CONCERN;
}
this.log = services.get('log-service').create(this.service_name, log_fields);
// INFO logs are treated as DEBUG logs instead if...
if (
// The configuration file explicitly says to do so
this.config.log_debug ||
// The class has `static LOG_DEBUG = true`; AND,
// the configuration file does NOT explicitly say NOT to do this
(!this.config.log_info && this.constructor.LOG_DEBUG)
) {
this.log.info = this.log.debug;
}
this.errors = services.get('error-service').create(this.log);
await (this._init || NOOP).call(this, this.args);
}
/**
* Handles an event by retrieving the appropriate event handler
* and executing it with the provided arguments.
*
* @param {string} id - The identifier of the event to handle.
* @param {Array} args - The arguments to pass to the event handler.
* @returns {Promise} The result of the event handler execution.
*/
async __on (id, args) {
const handler = this.__get_event_handler(id);
return await handler(id, ...args);
}
__get_event_handler (id) {
return this[`__on_${id}`]?.bind?.(this)
|| this.constructor[`__on_${id}`]?.bind?.(this.constructor)
|| NOOP;
}
}
module.exports = BaseService;
module.exports.BaseService = BaseService;
================================================
FILE: src/backend/src/services/BootScriptService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
async '__on_boot.ready' () {
const args = Context.get('args');
if ( ! args['boot-script'] ) return;
const script_name = args['boot-script'];
const require = this.require;
const fs = require('fs');
const boot_json_raw = fs.readFileSync(script_name, 'utf8');
const boot_json = JSON.parse(boot_json_raw);
await this.run_script(boot_json);
}
/**
* Executes a series of commands defined in a JSON boot script.
*
* This method processes each command in the boot_json array.
* If the command is recognized within the predefined scope, it will be executed.
* If not, an error is thrown.
*
* @param {Array} boot_json - An array of commands to execute.
* @throws {Error} Thrown if an unknown command is encountered.
*/
async run_script (boot_json) {
const scope = {
runner: 'boot-script',
'end-puter-process': ({ args }) => {
const svc_shutdown = this.services.get('shutdown');
svc_shutdown.shutdown(args[0]);
},
};
for ( const statement of boot_json ) {
const [cmd, ...args] = statement;
if ( ! scope[cmd] ) {
throw new Error(`Unknown command: ${cmd}`);
}
await scope[cmd]({ scope, args });
}
}
}
module.exports = {
BootScriptService,
};
================================================
FILE: src/backend/src/services/ChatAPIService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
async '__on_install.routes' (_, { app }) {
// Create a router for chat API endpoints
const router = (() => {
const require = this.require;
const express = require('express');
return express.Router();
})();
// Register the router with the Express app
app.use('/puterai', router);
// Install endpoints
this.install_chat_endpoints_({ router });
}
/**
* Installs chat API endpoints on the provided router
* @param {Object} options Options object
* @param {express.Router} options.router Express router to install endpoints on
* @private
*/
install_chat_endpoints_ ({ router }) {
const Endpoint = this.require('Endpoint');
router.use(require('../routers/puterai/openai/completions'));
router.use(require('../routers/puterai/openai/chat_completions'));
// Endpoint to list available AI chat models
Endpoint({
route: '/chat/models',
methods: ['GET'],
handler: async (req, res) => {
try {
// Use SUService to access AIChatService as system user
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const svc_aiChat = this.services.get('ai-chat');
// Return the simple model list which contains basic model information
return svc_aiChat.list();
});
// Return the list of models
res.json({ models: models.filter(e => !['costly', 'fake', 'abuse', 'model-fallback-test-1'].includes(e)) });
} catch ( error ) {
this.log.error('Error fetching models:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
// Endpoint to get detailed information about available AI chat models
Endpoint({
route: '/chat/models/details',
methods: ['GET'],
handler: async (req, res) => {
try {
// Use SUService to access AIChatService as system user
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const svc_aiChat = this.services.get('ai-chat');
// Return the detailed model list which includes cost and capability information
return svc_aiChat.models();
});
// Return the detailed list of models
res.json({ models: models.filter((e) => !['costly', 'fake', 'abuse', 'model-fallback-test-1'].includes(e.id)) });
} catch ( error ) {
this.log.error('Error fetching model details:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
Endpoint({
route: '/image/models',
methods: ['GET'],
handler: async (req, res) => {
try {
// Use SUService to access AIImageGenerationService as system user
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const svc_imageGen = this.services.get('ai-image');
// Return the simple model list which contains basic model information
return svc_imageGen.list();
});
// Return the list of models
res.json({ models });
} catch ( error ) {
this.log.error('Error fetching image models:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
Endpoint({
route: '/image/models/details',
methods: ['GET'],
handler: async (req, res) => {
try {
// Use SUService to access AIImageGenerationService as system user
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const svc_imageGen = this.services.get('ai-image');
// Return the detailed model list which includes cost and capability information
return svc_imageGen.models();
});
// Return the detailed list of models
res.json({ models });
} catch ( error ) {
this.log.error('Error fetching image model details:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
Endpoint({
route: '/video/models/details',
methods: ['GET'],
handler: async (req, res) => {
try {
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const items = [];
if ( this.services.has('openai-video-generation') ) {
const svc_video = this.services.get('openai-video-generation');
if ( typeof svc_video.models === 'function' ) {
items.push(...await svc_video.models());
}
}
if ( this.services.has('together-video-generation') ) {
const svc_video = this.services.get('together-video-generation');
if ( typeof svc_video.models === 'function' ) {
items.push(...await svc_video.models());
}
}
return items;
});
res.json({ models });
} catch ( error ) {
this.log.error('Error fetching video model details:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
Endpoint({
route: '/video/models',
methods: ['GET'],
handler: async (req, res) => {
try {
const svc_su = this.services.get('su');
const models = await svc_su.sudo(async () => {
const items = [];
if ( this.services.has('openai-video-generation') ) {
const svc_video = this.services.get('openai-video-generation');
if ( typeof svc_video.models === 'function' ) {
items.push(...(await svc_video.models()).map(model => model.puterId || model.id));
}
}
if ( this.services.has('together-video-generation') ) {
const svc_video = this.services.get('together-video-generation');
if ( typeof svc_video.models === 'function' ) {
items.push(...(await svc_video.models()).map(model => model.id));
}
}
return items;
});
res.json({ models });
} catch ( error ) {
this.log.error('Error fetching video models:', error);
throw APIError.create('internal_server_error');
}
},
}).attach(router);
}
}
module.exports = {
ChatAPIService,
};
================================================
FILE: src/backend/src/services/ChatAPIService.test.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
*/
_construct () {
this.named_rules = this.constructor.NAMED_RULES;
this.providers = this.constructor.PROVIDERS;
this.domain_to_provider = this.constructor.DOMAIN_TO_PROVIDER;
this.domain_nondistinct = this.constructor.DOMAIN_NONDISTINCT;
}
/**
* Cleans an email address by applying provider-specific rules and standardizations
* @param {string} email - The email address to clean
* @returns {string} The cleaned email address with applied rules and standardizations
*
* Splits email into local and domain parts, applies provider-specific rules like:
* - Removing dots for certain providers (Gmail, iCloud)
* - Handling subaddressing (removing +suffix)
* - Normalizing domains (e.g. googlemail.com -> gmail.com)
*/
clean (email) {
const eml = (() => {
const [local, domain] = email.split('@');
return { local, domain };
})();
if ( this.domain_nondistinct[eml.domain] ) {
eml.domain = this.domain_nondistinct[eml.domain];
}
const rules = [
'remove_subaddressing',
];
const provider = this.domain_to_provider[eml.domain] || eml.domain;
const provider_info = this.providers[provider];
if ( provider_info ) {
provider_info.rules = provider_info.rules || [];
provider_info.rmrules = provider_info.rmrules || [];
for ( const rule_name of provider_info.rules ) {
rules.push(rule_name);
}
for ( const rule_name of provider_info.rmrules ) {
const idx = rules.indexOf(rule_name);
if ( idx !== -1 ) {
rules.splice(idx, 1);
}
}
}
for ( const rule_name of rules ) {
const rule = this.named_rules[rule_name];
rule.rule({ eml });
}
return `${eml.local }@${ eml.domain}`;
}
/**
* Validates an email address against blocked domains and custom validation rules
* @param {string} email - The email address to validate
* @returns {Promise} True if email is valid, false if blocked or invalid
* @description First cleans the email, then checks against blocked domains from config.
* Emits 'email.validate' event to allow custom validation rules. Event handlers can
* set event.allow=false to reject the email.
*/
async validate (email) {
if ( this?.global_config?.env === 'dev' ) return true;
email = this.clean(email);
const config = this.global_config;
if ( Array.isArray(config.blocked_email_domains) ) {
for ( const suffix of config.blocked_email_domains ) {
if ( email.endsWith(suffix) ) {
return false;
}
}
}
const svc_event = this.services.get('event');
const event = { allow: true, email };
await svc_event.emit('email.validate', event);
if ( ! event.allow ) return false;
return true;
}
}
module.exports = { CleanEmailService };
================================================
FILE: src/backend/src/services/CleanEmailService.test.ts
================================================
import { describe, expect, it } from 'vitest';
import { createTestKernel } from '../../tools/test.mjs';
import { CleanEmailService } from './CleanEmailService.js';
describe('CleanEmailService', () => {
it('should clean email addresses correctly', async () => {
const testKernel = await createTestKernel({
serviceMap: {
'clean-email': CleanEmailService,
},
});
const cleanEmailService = testKernel.services!.get('clean-email') as CleanEmailService;
const cases = [
{
email: 'bob.ross+happy-clouds@googlemail.com',
expected: 'bobross@gmail.com',
},
{
email: 'under.rated+email-service@yahoo.com',
expected: 'under.rated+email-service@yahoo.com',
},
{
email: 'the-absolute+best@protonmail.com',
expected: 'the-absolute@protonmail.com',
},
];
for ( const { email, expected } of cases ) {
const cleaned = cleanEmailService.clean(email);
expect(cleaned).toBe(expected);
}
});
});
================================================
FILE: src/backend/src/services/ClientOperationService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves to the created ClientOperationTracker instance.
*/
async add_operation (parameters) {
const tracker = new ClientOperationTracker(parameters);
return tracker;
}
ckey (key) {
return `${CONTEXT_KEY }:${ key}`;
}
}
module.exports = {
ClientOperationService,
};
================================================
FILE: src/backend/src/services/ClientOperationService.test.ts
================================================
import { describe, expect, it } from 'vitest';
import { ClientOperationService } from './ClientOperationService';
describe('ClientOperationService', async () => {
// ClientOperationService doesn't extend BaseService, so we can't use init
// We need to create it directly
const services = { _instances: {} };
const clientOperationService = new ClientOperationService({ services });
it('should be instantiated', () => {
expect(clientOperationService).toBeDefined();
expect(clientOperationService.operations_).toBeDefined();
});
it('should have operations array', () => {
expect(clientOperationService.operations_).toBeDefined();
expect(Array.isArray(clientOperationService.operations_)).toBe(true);
});
it('should create operation with default parameters', async () => {
const tracker = await clientOperationService.add_operation({});
expect(tracker).toBeDefined();
expect(tracker.name).toBe('untitled');
expect(Array.isArray(tracker.tags)).toBe(true);
expect(tracker.tags.length).toBe(0);
expect(tracker.frame).toBe(null);
expect(tracker.metadata).toBeDefined();
expect(typeof tracker.metadata).toBe('object');
expect(Array.isArray(tracker.objects)).toBe(true);
});
it('should create operation with name', async () => {
const tracker = await clientOperationService.add_operation({
name: 'test-operation',
});
expect(tracker.name).toBe('test-operation');
});
it('should create operation with tags', async () => {
const tags = ['tag1', 'tag2', 'tag3'];
const tracker = await clientOperationService.add_operation({
tags,
});
expect(tracker.tags).toEqual(tags);
});
it('should create operation with frame', async () => {
const frame = { type: 'test-frame' };
const tracker = await clientOperationService.add_operation({
frame,
});
expect(tracker.frame).toBe(frame);
});
it('should create operation with metadata', async () => {
const metadata = { key1: 'value1', key2: 'value2' };
const tracker = await clientOperationService.add_operation({
metadata,
});
expect(tracker.metadata).toEqual(metadata);
});
it('should create operation with objects', async () => {
const objects = [{ id: 1 }, { id: 2 }];
const tracker = await clientOperationService.add_operation({
objects,
});
expect(tracker.objects).toEqual(objects);
});
it('should create operation with all parameters', async () => {
const params = {
name: 'full-operation',
tags: ['full', 'test'],
frame: { type: 'frame' },
metadata: { meta: 'data' },
objects: [{ obj: 1 }],
};
const tracker = await clientOperationService.add_operation(params);
expect(tracker.name).toBe(params.name);
expect(tracker.tags).toEqual(params.tags);
expect(tracker.frame).toBe(params.frame);
expect(tracker.metadata).toEqual(params.metadata);
expect(tracker.objects).toEqual(params.objects);
});
it('should create multiple operations', async () => {
const tracker1 = await clientOperationService.add_operation({ name: 'op1' });
const tracker2 = await clientOperationService.add_operation({ name: 'op2' });
const tracker3 = await clientOperationService.add_operation({ name: 'op3' });
expect(tracker1.name).toBe('op1');
expect(tracker2.name).toBe('op2');
expect(tracker3.name).toBe('op3');
});
it('should have ckey method', () => {
expect(clientOperationService.ckey).toBeDefined();
expect(typeof clientOperationService.ckey).toBe('function');
});
it('should generate context key with ckey', () => {
const key = clientOperationService.ckey('test-key');
expect(key).toBeDefined();
expect(typeof key).toBe('string');
expect(key).toContain('test-key');
});
it('should generate different keys for different inputs', () => {
const key1 = clientOperationService.ckey('key1');
const key2 = clientOperationService.ckey('key2');
expect(key1).not.toBe(key2);
});
});
================================================
FILE: src/backend/src/services/CommandService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see }
* @throws {Error} Logs any errors that occur during command execution
*/
async execute (args, log) {
log = log ?? console;
const { id, name, description, handler } = this.spec_;
try {
await handler(args, log);
} catch ( err ) {
log.error(`command ${name ?? id} failed: ${err.message}`);
log.error(err.stack);
}
}
completeArgument (args) {
const completer = this.spec_.completer;
if ( completer )
{
return completer(args);
}
return [];
}
}
/**
* CommandService class manages the registration, execution, and handling of commands in the Puter system.
* Extends BaseService to provide command-line interface functionality. Maintains a collection of Command
* objects, supports command registration with namespaces, command execution with arguments, and provides
* command lookup capabilities. Includes built-in help command functionality.
* @extends BaseService
*/
class CommandService extends BaseService {
/**
* Initializes the command service's internal state
* Called during service construction to set up the empty commands array
*/
async _construct () {
this.commands_ = [];
}
/**
* Add the help command to the list of commands on init
*/
async _init () {
this.commands_.push(new Command({
id: 'help',
description: 'show this help',
handler: (args, log) => {
log.log('available commands:');
for ( const command of this.commands_ ) {
log.log(`- ${command.spec_.id}: ${command.spec_.description}`);
}
},
}));
}
async '__on_boot.consolidation' () {
const svc_event = this.services.get('event');
const svc_command = this;
const event = {
createCommand (name, command) {
const serviceName = Context.get('extension_name') ?? '%missing%';
const commandSpec = typeof command === 'function'
? { handler: command }
: command;
if ( typeof commandSpec !== 'object' ) {
throw new Error('command must be either a function or an object');
}
if ( ! (typeof command.handler === 'function') ) {
throw new Error('command should have a handler function');
}
svc_command.registerCommands(serviceName, [{
id: name,
...commandSpec,
}]);
},
};
svc_event.emit('create.commands', event);
}
registerCommands (serviceName, commands) {
if ( ! this.log ) {
/* eslint-disable */
console.error(
'CommandService.registerCommands was called before a logger ' +
'was initialied. This happens when calling registerCommands ' +
'in the "construct" phase instead of the "init" phase. If ' +
'you are migrating a legacy service that does not extend ' +
'BaseService, maybe the _construct hook is calling init()'
);
/* eslint-enable */
process.exit(1);
}
for ( const command of commands ) {
this.log.debug(`registering command ${serviceName}:${command.id}`);
this.commands_.push(new Command({
...command,
id: `${serviceName}:${command.id}`,
}));
}
}
/**
* Executes a command with the given arguments and logging context
* @param {string[]} args - Array of command arguments where first element is command name
* @param {Object} log - Logger object for output (defaults to console if not provided)
* @returns {Promise}
* @throws {Error} If command execution fails
*/
async executeCommand (args, log) {
const [commandName, ...commandArgs] = args;
const command = this.commands_.find(c => c.spec_.id === commandName);
if ( ! command ) {
log.error(`unknown command: ${commandName}`);
return;
}
/**
* Executes a command with the given arguments in a global context
* @param {string[]} args - Array of command arguments where first element is command name
* @param {Object} log - Logger object for output
* @returns {Promise}
* @throws {Error} If command execution fails
*/
await globalThis.root_context.sub({
injected_logger: log,
}).arun(async () => {
await command.execute(commandArgs, log);
});
}
/**
* Executes a raw command string by splitting it into arguments and executing the command
* @param {string} text - Raw command string to execute
* @param {object} log - Logger object for output (defaults to console if not provided)
* @returns {Promise}
* @todo Replace basic whitespace splitting with proper tokenizer (obvious-json)
*/
async executeRawCommand (text, log) {
// TODO: add obvious-json as a tokenizer
const args = text.split(/\s+/);
await this.executeCommand(args, log);
}
/**
* Gets a list of all registered command names/IDs
* @returns {string[]} Array of command identifier strings
*/
get commandNames () {
return this.commands_.map(command => command.id);
}
getCommand (id) {
return this.commands_.find(command => command.id === id);
}
}
module.exports = {
CommandService,
};
================================================
FILE: src/backend/src/services/CommandService.test.ts
================================================
import { describe, expect, it, vi } from 'vitest';
import { createTestKernel } from '../../tools/test.mjs';
import { CommandService } from './CommandService';
describe('CommandService', async () => {
const testKernel = await createTestKernel({
serviceMap: {
commands: CommandService,
},
initLevelString: 'init',
});
const commandService = testKernel.services!.get('commands') as CommandService;
it('should be instantiated', () => {
expect(commandService).toBeInstanceOf(CommandService);
});
it('should have help command registered by default', () => {
expect(commandService.commandNames).toContain('help');
});
it('should register commands', () => {
commandService.registerCommands('test-service', [
{
id: 'test-cmd',
description: 'A test command',
handler: async () => {},
},
]);
expect(commandService.commandNames).toContain('test-service:test-cmd');
});
it('should execute registered commands', async () => {
let executed = false;
commandService.registerCommands('exec-test', [
{
id: 'exec-cmd',
description: 'Execute test',
handler: async () => { executed = true; },
},
]);
const mockLog = { error: vi.fn(), log: vi.fn() };
await commandService.executeCommand(['exec-test:exec-cmd'], mockLog);
expect(executed).toBe(true);
});
it('should pass arguments to command handler', async () => {
let receivedArgs: string[] = [];
commandService.registerCommands('args-test', [
{
id: 'args-cmd',
description: 'Args test',
handler: async (args) => { receivedArgs = args; },
},
]);
const mockLog = { error: vi.fn(), log: vi.fn() };
await commandService.executeCommand(['args-test:args-cmd', 'arg1', 'arg2'], mockLog);
expect(receivedArgs).toEqual(['arg1', 'arg2']);
});
it('should handle unknown commands', async () => {
const mockLog = { error: vi.fn(), log: vi.fn() };
await commandService.executeCommand(['unknown-command'], mockLog);
expect(mockLog.error).toHaveBeenCalledWith('unknown command: unknown-command');
});
it('should execute raw commands', async () => {
let executed = false;
commandService.registerCommands('raw-test', [
{
id: 'raw-cmd',
description: 'Raw test',
handler: async () => { executed = true; },
},
]);
const mockLog = { error: vi.fn(), log: vi.fn() };
await commandService.executeRawCommand('raw-test:raw-cmd', mockLog);
expect(executed).toBe(true);
});
it('should get command by id', () => {
commandService.registerCommands('get-test', [
{
id: 'get-cmd',
description: 'Get test',
handler: async () => {},
},
]);
const cmd = commandService.getCommand('get-test:get-cmd');
expect(cmd).toBeDefined();
expect(cmd?.id).toBe('get-test:get-cmd');
});
it('should execute help command', async () => {
const mockLog = { error: vi.fn(), log: vi.fn() };
await commandService.executeCommand(['help'], mockLog);
expect(mockLog.log).toHaveBeenCalledWith('available commands:');
});
it('should support command completers', () => {
commandService.registerCommands('complete-test', [
{
id: 'complete-cmd',
description: 'Complete test',
handler: async () => {},
completer: (args) => ['option1', 'option2'],
},
]);
const cmd = commandService.getCommand('complete-test:complete-cmd');
const completions = cmd?.completeArgument([]);
expect(completions).toEqual(['option1', 'option2']);
});
});
================================================
FILE: src/backend/src/services/ConfigurableCountingService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves when the database connection is established.
* @memberof ConfigurableCountingService
*/
async _init () {
this.db = this.services.get('database').get(DB_WRITE, 'counting');
}
/**
* Increments the count for a given service based on the provided parameters.
* This method builds an SQL query to update the count and other custom values
* in the database. It handles different SQL dialects (MySQL and SQLite) and
* ensures that the pricing category is correctly hashed and stored.
*
* @param {Object} params - The parameters for incrementing the count.
* @param {string} params.service_name - The name of the service.
* @param {string} params.service_type - The type of the service.
* @param {Object} params.values - The values to be incremented.
* @throws {Error} If the service type is unknown or if there are no more available columns.
* @returns {Promise} A promise that resolves when the count is successfully incremented.
*/
async increment ({ service_name, service_type, values }) {
values = values ? { ...values } : {};
const now = new Date();
const year = now.getUTCFullYear();
const month = now.getUTCMonth() + 1;
const counting_type = this.constructor.counting_types[service_type];
if ( ! counting_type ) {
throw new Error(`unknown counting type ${service_type}`);
}
const available_columns = {};
for ( const k in this.constructor.sql_columns ) {
available_columns[k] = [...this.constructor.sql_columns[k]];
}
const custom_col_names = counting_type.values.map((value, index) => {
const column = available_columns[value.type].shift();
if ( ! column ) {
// TODO: this could be an init check on all the available service types
throw new Error(`no more available columns for type ${value.type}`);
}
return column;
});
const custom_col_values = counting_type.values.map((value, index) => {
return values[value.name];
});
// `pricing_category` is a JSON field. Keys from `values` used for
// the pricing category will be removed from ths `values` object
const pricing_category = {};
for ( const category of counting_type.category ) {
pricing_category[category.name] = values[category.name];
delete values[category.name];
}
// `JSON.stringify` cannot be used here because it does not sort
// the keys.
const pricing_category_str = counting_type.category.map((category) => {
return `${category.name}:${pricing_category[category.name]}`;
}).join(',');
const pricing_category_hash = hash(pricing_category_str);
const actor = Context.get('actor');
const actor_key = actor.uid;
const required_data = {
year,
month,
service_name,
service_type,
actor_key,
pricing_category_hash,
pricing_category: JSON.stringify(pricing_category),
};
const duplicate_update_part =
`count = count + 1${
custom_col_names.length > 0 ? ', ' : ''
} ${
custom_col_names.map((name) => `${name} = ${name} + ?`).join(', ')
}`;
const identifying_keys = [
'year', 'month',
'service_type', 'service_name',
'actor_key',
'pricing_category_hash',
];
const sql =
`INSERT INTO monthly_usage_counts (${
Object.keys(required_data).join(', ')
}, count, ${
custom_col_names.join(', ')
}) ` +
`VALUES (${
Object.keys(required_data).map(() => '?').join(', ')
}, 1, ${custom_col_values.map(() => '?').join(', ')}) ${
this.db.case({
mysql: `ON DUPLICATE KEY UPDATE ${ duplicate_update_part}`,
sqlite: `ON CONFLICT(${
identifying_keys.map(v => `\`${v}\``).join(', ')
}) DO UPDATE SET ${duplicate_update_part}`,
})}`
;
const value_array = [
...Object.values(required_data),
...custom_col_values,
...custom_col_values,
];
await this.db.write(sql, value_array);
}
}
module.exports = {
ConfigurableCountingService,
};
================================================
FILE: src/backend/src/services/ConfigurableCountingService.test.ts
================================================
import { describe, expect, it } from 'vitest';
import { createTestKernel } from '../../tools/test.mjs';
import * as config from '../config';
import { ConfigurableCountingService } from './ConfigurableCountingService';
describe('ConfigurableCountingService', async () => {
config.load_config({
'services': {
'database': {
path: ':memory:',
},
},
});
const testKernel = await createTestKernel({
serviceMap: {
'counting': ConfigurableCountingService,
},
initLevelString: 'init',
testCore: true,
});
const countingService = testKernel.services!.get('counting') as ConfigurableCountingService;
it('should be instantiated', () => {
expect(countingService).toBeInstanceOf(ConfigurableCountingService);
});
it('should have counting types defined', () => {
expect(ConfigurableCountingService.counting_types).toBeDefined();
expect(ConfigurableCountingService.counting_types.gpt).toBeDefined();
expect(ConfigurableCountingService.counting_types.dalle).toBeDefined();
});
it('should have sql columns defined', () => {
expect(ConfigurableCountingService.sql_columns).toBeDefined();
expect(ConfigurableCountingService.sql_columns.uint).toBeDefined();
expect(ConfigurableCountingService.sql_columns.uint.length).toBe(3);
});
it('should validate GPT counting type structure', () => {
const gptType = ConfigurableCountingService.counting_types.gpt;
expect(gptType.category).toBeDefined();
expect(gptType.values).toBeDefined();
expect(gptType.category.length).toBeGreaterThan(0);
expect(gptType.values.length).toBeGreaterThan(0);
});
it('should validate DALL-E counting type structure', () => {
const dalleType = ConfigurableCountingService.counting_types.dalle;
expect(dalleType.category).toBeDefined();
expect(dalleType.category.length).toBeGreaterThan(0);
expect(dalleType.category.some(c => c.name === 'model')).toBe(true);
expect(dalleType.category.some(c => c.name === 'quality')).toBe(true);
expect(dalleType.category.some(c => c.name === 'resolution')).toBe(true);
});
it('should have gpt token value definitions', () => {
const gptType = ConfigurableCountingService.counting_types.gpt;
expect(gptType.values.some(v => v.name === 'input_tokens')).toBe(true);
expect(gptType.values.some(v => v.name === 'output_tokens')).toBe(true);
expect(gptType.values.every(v => v.type === 'uint')).toBe(true);
});
it('should have available sql columns for uint type', () => {
const columns = ConfigurableCountingService.sql_columns.uint;
expect(columns).toBeDefined();
expect(Array.isArray(columns)).toBe(true);
expect(columns.length).toBe(3);
expect(columns.every(col => typeof col === 'string')).toBe(true);
});
it('should have model category for gpt', () => {
const gptType = ConfigurableCountingService.counting_types.gpt;
const modelCategory = gptType.category.find(c => c.name === 'model');
expect(modelCategory).toBeDefined();
expect(modelCategory!.type).toBe('string');
});
});
================================================
FILE: src/backend/src/services/Container.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } A promise that resolves when all services are
* initialized or rejects if any service initialization fails.
*/
async init () {
for ( const k in this.instances_ ) {
if ( ! this.instances_[k]._run_as_early_as_possible ) continue;
await this.instances_[k].run_as_early_as_possible();
}
for ( const k in this.instances_ ) {
await this.instances_[k].construct();
}
const init_failures = [];
const promises = [];
const PARALLEL = config.experimental_parallel_init;
for ( const k in this.instances_ ) {
try {
if ( PARALLEL ) promises.push(this.instances_[k].init());
else await this.instances_[k].init();
} catch (e) {
init_failures.push({ k, e });
}
}
if ( PARALLEL ) await Promise.all(promises);
if ( init_failures.length ) {
console.error('init failures', init_failures);
throw new CompositeError(`failed to initialize these services: ${
init_failures.map(({ k }) => k).join(', ')}`,
init_failures.map(({ k, e }) => e));
}
}
/**
* Emits an event to all registered services.
*
* This method sends an event identified by `id` along with any additional arguments to all
* services registered in the container. If a logger is available, it logs the event.
*
* @param {string} id - The identifier of the event.
* @param {...*} args - Additional arguments to pass to the event handler.
* @returns {Promise} A promise that resolves when all event handlers have completed.
*/
async emit (id, ...args) {
if ( this.logger ) {
this.logger.debug(`services:event ${id}`, { args });
}
const promises = [];
for ( const k in this.instances_ ) {
if ( this.instances_[k].__on ) {
promises.push(Context.arun(() => this.instances_[k].__on(id, args)));
}
}
await Promise.all(promises);
}
}
/**
* @class ProxyContainer
* @classdesc The ProxyContainer class is a proxy for the Container class, allowing for delegation of service management tasks.
* It extends the functionality of the Container class by providing a delegation mechanism.
* This class is useful for scenarios where you need to manage services through a proxy,
* enabling additional flexibility and control over service instances.
*/
class ProxyContainer {
constructor (delegate) {
this.delegate = delegate;
this.instances_ = {};
}
set (name, instance) {
this.instances_[name] = instance;
}
get (name) {
if ( this.instances_.hasOwnProperty(name) ) {
return this.instances_[name];
}
return this.delegate.get(name);
}
/**
* Checks if the container has a service with the specified name.
*
* @param {string} name - The name of the service to check.
* @returns {boolean} - Returns true if the service exists, false otherwise.
*/
has (name) {
if ( this.instances_.hasOwnProperty(name) ) {
return true;
}
return this.delegate.has(name);
}
get values () {
const values = {};
Object.assign(values, this.delegate.values);
for ( const k in this.instances_ ) {
let k2 = k;
// Replace lowerCamelCase with underscores
// (just an idea; more effort than it's worth right now)
// let k2 = k.replace(/([a-z])([A-Z])/g, '$1_$2')
// Replace dashes with underscores
k2 = k2.replace(/-/g, '_');
// Convert to lower case
k2 = k2.toLowerCase();
values[k2] = this.instances_[k];
}
return values;
}
}
module.exports = { Container, ProxyContainer };
================================================
FILE: src/backend/src/services/ContextInitService.js
================================================
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see } The populated output object after all
* providers have been processed.
*/
async get_details (context, out) {
out = out || {};
for ( const provider of this.providers_ ) {
await provider(context, out);
}
return out;
}
}
module.exports = { DetailProviderService };
================================================
FILE: src/backend/src/services/DetailProviderService.test.ts
================================================
import { describe, expect, it } from 'vitest';
import { createTestKernel } from '../../tools/test.mjs';
import { DetailProviderService } from './DetailProviderService';
describe('DetailProviderService', async () => {
const testKernel = await createTestKernel({
serviceMap: {
'detail-provider': DetailProviderService,
},
initLevelString: 'init',
});
const detailProviderService = testKernel.services!.get('detail-provider') as any;
it('should be instantiated', () => {
expect(detailProviderService).toBeInstanceOf(DetailProviderService);
});
it('should have empty providers array initially', () => {
expect(detailProviderService.providers_).toBeDefined();
expect(Array.isArray(detailProviderService.providers_)).toBe(true);
});
it('should register a provider', () => {
const initialLength = detailProviderService.providers_.length;
const provider = async (context: any, out: any) => {
out.test = 'value';
};
detailProviderService.register_provider(provider);
expect(detailProviderService.providers_.length).toBe(initialLength + 1);
});
it('should get details with single provider', async () => {
const service = testKernel.services!.get('detail-provider') as any;
service.register_provider(async (context: any, out: any) => {
out.name = context.input;
});
const result = await service.get_details({ input: 'test-name' });
expect(result.name).toBe('test-name');
});
it('should get details with multiple providers', async () => {
const service = testKernel.services!.get('detail-provider') as any;
service.register_provider(async (context: any, out: any) => {
out.field1 = 'value1';
});
service.register_provider(async (context: any, out: any) => {
out.field2 = 'value2';
});
const result = await service.get_details({});
expect(result.field1).toBe('value1');
expect(result.field2).toBe('value2');
});
it('should allow providers to modify existing output', async () => {
const service = testKernel.services!.get('detail-provider') as any;
service.register_provider(async (context: any, out: any) => {
out.counter = 1;
});
service.register_provider(async (context: any, out: any) => {
out.counter = out.counter + 1;
});
const result = await service.get_details({});
expect(result.counter).toBe(2);
});
it('should use provided output object', async () => {
const service = testKernel.services!.get('detail-provider') as any;
service.register_provider(async (context: any, out: any) => {
out.added = true;
});
const existingOut = { existing: 'value' };
const result = await service.get_details({}, existingOut);
expect(result.existing).toBe('value');
expect(result.added).toBe(true);
});
it('should handle async providers', async () => {
const service = testKernel.services!.get('detail-provider') as any;
service.register_provider(async (context: any, out: any) => {
await new Promise(resolve => setTimeout(resolve, 10));
out.async = true;
});
const result = await service.get_details({});
expect(result.async).toBe(true);
});
});
================================================
FILE: src/backend/src/services/DynamoKVStore/.gitignore
================================================
*.js
*.js.map
================================================
FILE: src/backend/src/services/DynamoKVStore/DynamoKVStore.test.ts
================================================
import { Actor } from '@heyputer/backend/src/services/auth/Actor.js';
import { SUService } from '@heyputer/backend/src/services/SUService';
import { createTestKernel } from '@heyputer/backend/tools/test.mjs';
import { describe, expect, it } from 'vitest';
import { config } from '../../loadTestConfig.js';
import { DynamoKVStore } from './DynamoKVStore.js';
import { DynamoKVStoreWrapper, IDynamoKVStoreWrapper } from './DynamoKVStoreWrapper.js';
describe('DynamoKVStore', async () => {
const TABLE_NAME = 'store-kv-v1';
const makeActor = (userId: number | string, appUid?: string) => ({
type: {
user: { id: userId, uuid: String(userId) },
...(appUid ? { app: { uid: appUid } } : {}),
},
}) as Actor;
const testKernel = await createTestKernel({
serviceMap: {
'puter-kvstore': DynamoKVStoreWrapper,
},
initLevelString: 'init',
testCore: true,
serviceConfigOverrideMap: {
'services': {
'puter-kvstore': { tableName: TABLE_NAME },
},
},
});
const testSubject = testKernel.services!.get('puter-kvstore') as IDynamoKVStoreWrapper;
const kvStore = testSubject.kvStore!;
const su = testKernel.services!.get('su') as SUService;
it('should be instantiated', () => {
expect(testSubject).toBeInstanceOf(DynamoKVStoreWrapper);
});
it('should contain a copy of the public methods of DynamoKVStore too', () => {
const meteringMethods = Object.getOwnPropertyNames(DynamoKVStore.prototype)
.filter((name) => name !== 'constructor');
const wrapperMethods = testSubject as unknown as Record;
const missing = meteringMethods.filter((name) => typeof wrapperMethods[name] !== 'function');
expect(missing).toEqual([]);
});
it('should have DynamoKVStore instantiated', async () => {
expect(testSubject.kvStore).toBeInstanceOf(DynamoKVStore);
});
it('sets and retrieves values for the current actor context', async () => {
const actor = makeActor(1);
const key = 'greeting';
const value = { hello: 'world' };
await su.sudo(actor, () => kvStore.set({ key, value }));
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect(stored).toEqual(value);
});
it('scopes data to the app when provided', async () => {
const userId = 2;
const actorAppOne = makeActor(userId, 'app-one');
const actorAppTwo = makeActor(userId, 'app-two');
const key = 'scoped-key';
await su.sudo(actorAppOne, () => kvStore.set({ key, value: 'one' }));
await su.sudo(actorAppTwo, () => kvStore.set({ key, value: 'two' }));
const fromOne = await su.sudo(actorAppOne, () => kvStore.get({ key }));
const fromTwo = await su.sudo(actorAppTwo, () => kvStore.get({ key }));
expect(fromOne).toBe('one');
expect(fromTwo).toBe('two');
});
it('increments nested numeric paths and persists the aggregated totals', async () => {
const actor = makeActor(3);
const key = 'counter-key';
const first = await su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { 'total': 5, 'nested.count': 2 },
}));
const second = await su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { 'total': 1, 'nested.count': 3 },
}));
expect(first).toMatchObject({ total: 5, nested: { count: 2 } });
expect(second).toMatchObject({ total: 6, nested: { count: 5 } });
const persisted = await su.sudo(actor, () => kvStore.get({ key }));
expect(persisted).toMatchObject({ total: 6, nested: { count: 5 } });
});
it('decrements numeric paths via decr and keeps values in sync', async () => {
const actor = makeActor(4);
const key = 'decr-key';
await su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { total: 5, 'nested.count': 4 },
}));
const afterDecr = await su.sudo(actor, () => kvStore.decr({
key,
pathAndAmountMap: { total: 2, 'nested.count': 1 },
}));
expect(afterDecr).toMatchObject({ total: 3, nested: { count: 3 } });
const persisted = await su.sudo(actor, () => kvStore.get({ key }));
expect(persisted).toMatchObject({ total: 3, nested: { count: 3 } });
});
it('deletes keys with del', async () => {
const actor = makeActor(5);
const key = 'delete-me';
await su.sudo(actor, () => {
return kvStore.set({ key, value: 'bye' });
});
const res = await su.sudo(actor, () => kvStore.del({ key }));
const value = await su.sudo(actor, () => kvStore.get({ key }));
expect(res).toBe(true);
expect(value).toBeNull();
});
it('lists entries, keys, and values while omitting expired rows', async () => {
const actor = makeActor(6);
await su.sudo(actor, () => kvStore.set({ key: 'k1', value: 'v1' }));
await su.sudo(actor, () => kvStore.set({ key: 'expired', value: 'gone', expireAt: Math.floor(Date.now() / 1000) - 10 }));
const entries = await su.sudo(actor, () => kvStore.list({ as: 'entries' }));
const keys = await su.sudo(actor, () => kvStore.list({ as: 'keys' }));
const values = await su.sudo(actor, () => kvStore.list({ as: 'values' }));
expect(entries).toEqual([{ key: 'k1', value: 'v1' }]);
expect(keys).toEqual(['k1']);
expect(values).toEqual(['v1']);
});
it('rejects invalid list selector', async () => {
const actor = makeActor(7);
expect(su.sudo(actor, () => kvStore.list({ as: 'bad' as never })))
.rejects;
});
it('supports paginated list results with cursors', async () => {
const actor = makeActor(71);
await su.sudo(actor, () => kvStore.set({ key: 'a', value: 1 }));
await su.sudo(actor, () => kvStore.set({ key: 'b', value: 2 }));
await su.sudo(actor, () => kvStore.set({ key: 'c', value: 3 }));
const firstPage = await su.sudo(actor, () => kvStore.list({ as: 'keys', limit: 2 })) as { items: string[]; cursor?: string };
expect(firstPage.items).toHaveLength(2);
expect(firstPage.cursor).toBeTypeOf('string');
const secondPage = await su.sudo(actor, () => kvStore.list({ as: 'keys', limit: 2, cursor: firstPage.cursor })) as { items: string[]; cursor?: string };
expect(secondPage.items).toHaveLength(1);
expect(secondPage.cursor).toBeUndefined();
const allKeys = [...firstPage.items, ...secondPage.items].sort();
expect(allKeys).toEqual(['a', 'b', 'c']);
});
it('supports prefix pattern semantics', async () => {
const actor = makeActor(72);
const allKeys = [
'abc',
'abc123',
'abc123xyz',
'ab',
'key*literal',
'key*literal-2',
'k*y',
'k*y-extra',
'other',
];
await Promise.all(allKeys.map((key, idx) => su.sudo(actor, () => kvStore.set({ key, value: idx }))));
const expectedAbc = ['abc', 'abc123', 'abc123xyz'];
const expectedKeyStar = ['key*literal', 'key*literal-2'];
const expectedMiddleStar = ['k*y', 'k*y-extra'];
const abcKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'abc' })) as string[];
expect([...abcKeys].sort()).toEqual([...expectedAbc].sort());
const abcWildcardKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'abc*' })) as string[];
expect([...abcWildcardKeys].sort()).toEqual([...expectedAbc].sort());
const keyStarKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'key**' })) as string[];
expect([...keyStarKeys].sort()).toEqual([...expectedKeyStar].sort());
const middleStarKeys = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: 'k*y*' })) as string[];
expect([...middleStarKeys].sort()).toEqual([...expectedMiddleStar].sort());
const allList = await su.sudo(actor, () => kvStore.list({ as: 'keys', pattern: '*' })) as string[];
expect([...allList].sort()).toEqual([...allKeys].sort());
});
it('returns ordered values for arrays and null for expired keys', async () => {
const actor = makeActor(8);
const now = Math.floor(Date.now() / 1000);
await su.sudo(actor, () => kvStore.set({ key: 'a', value: 1 }));
await su.sudo(actor, () => kvStore.set({ key: 'b', value: 2, expireAt: now - 5 }));
await su.sudo(actor, () => kvStore.set({ key: 'c', value: 3 }));
const results = await su.sudo(actor, () => kvStore.get({ key: ['c', 'b', 'a'] }));
expect(results).toEqual([3, null, 1]);
});
it('flush clears all keys for the actor/app combination', async () => {
const actor = makeActor(9, 'flush-app');
await su.sudo(actor, () => kvStore.set({ key: 'one', value: 1 }));
await su.sudo(actor, () => kvStore.set({ key: 'two', value: 2 }));
const res = await su.sudo(actor, () => kvStore.flush());
const remaining = await su.sudo(actor, () => kvStore.list({ as: 'entries' }));
expect(res).toBe(true);
expect(remaining).toEqual([]);
});
it('expireAt and expire set timestamps that cause reads to return null', async () => {
const actor = makeActor(10);
const keyAt = 'expire-at';
const keyTtl = 'expire-ttl';
await su.sudo(actor, () => kvStore.set({ key: keyAt, value: 'keep' }));
await su.sudo(actor, () => kvStore.set({ key: keyTtl, value: 'keep' }));
await su.sudo(actor, () => kvStore.expireAt({ key: keyAt, timestamp: Math.floor(Date.now() / 1000) - 1 }));
await su.sudo(actor, () => kvStore.expire({ key: keyTtl, ttl: -1 }));
const valAt = await su.sudo(actor, () => kvStore.get({ key: keyAt }));
const valTtl = await su.sudo(actor, () => kvStore.get({ key: keyTtl }));
expect(valAt).toBeNull();
expect(valTtl).toBeNull();
});
it('updates nested paths and creates missing maps', async () => {
const actor = makeActor(12);
const key = 'update-key';
const updated = await su.sudo(actor, () => kvStore.update({
key,
pathAndValueMap: {
'profile.name': 'Ada',
'profile.stats.score': 7,
'active': true,
},
}));
expect(updated).toMatchObject({
profile: { name: 'Ada', stats: { score: 7 } },
active: true,
});
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect(stored).toMatchObject({
profile: { name: 'Ada', stats: { score: 7 } },
active: true,
});
});
it('update can set ttl for the whole object', async () => {
const actor = makeActor(13);
const key = 'update-ttl';
await su.sudo(actor, () => kvStore.update({
key,
pathAndValueMap: { 'count': 1 },
ttl: -1,
}));
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect(stored).toBeNull();
});
it('supports list index paths when updating', async () => {
const actor = makeActor(17);
const key = 'update-list-index';
await su.sudo(actor, () => kvStore.set({
key,
value: { a: { b: [1, 2] } },
}));
const updated = await su.sudo(actor, () => kvStore.update({
key,
pathAndValueMap: { 'a.b[1]': 5 },
}));
expect((updated as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect((stored as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);
});
it('adds values to nested lists and creates missing maps', async () => {
const actor = makeActor(15);
const key = 'add-key';
const first = await su.sudo(actor, () => kvStore.add({
key,
pathAndValueMap: {
'a.b': 1,
},
}));
expect(first).toMatchObject({ a: { b: [1] } });
const second = await su.sudo(actor, () => kvStore.add({
key,
pathAndValueMap: {
'a.b': 2,
'a.c': ['x', 'y'],
},
}));
expect(second).toMatchObject({ a: { b: [1, 2], c: ['x', 'y'] } });
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect(stored).toMatchObject({ a: { b: [1, 2], c: ['x', 'y'] } });
});
it('supports list index paths when appending', async () => {
const actor = makeActor(18);
const key = 'add-list-index';
await su.sudo(actor, () => kvStore.set({
key,
value: { a: { b: [[1], [2]] } },
}));
const updated = await su.sudo(actor, () => kvStore.add({
key,
pathAndValueMap: { 'a.b[1]': 3 },
}));
expect((updated as { a?: { b?: number[][] } }).a?.b).toEqual([[1], [2, 3]]);
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect((stored as { a?: { b?: number[][] } }).a?.b).toEqual([[1], [2, 3]]);
});
it('supports nested list indexing for add, update, remove, and incr', async () => {
const actor = makeActor(21);
const key = 'nested-list-index';
await su.sudo(actor, () => kvStore.set({
key,
value: { a: [1, { b: { c: [1] } }, 2] },
}));
const added = await su.sudo(actor, () => kvStore.add({
key,
pathAndValueMap: { 'a[1].b.c': 2 },
}));
expect((added as { a?: Array }).a).toEqual([1, { b: { c: [1, 2] } }, 2]);
const updated = await su.sudo(actor, () => kvStore.update({
key,
pathAndValueMap: { 'a[1].b.c': [9] },
}));
expect((updated as { a?: Array }).a).toEqual([1, { b: { c: [9] } }, 2]);
const removed = await su.sudo(actor, () => kvStore.remove({
key,
paths: ['a[1].b.c'],
}));
expect((removed as { a?: Array }).a).toEqual([1, { b: {} }, 2]);
await su.sudo(actor, () => kvStore.set({
key,
value: { a: [1, { b: { c: 1 } }, 2] },
}));
const incrRes = await su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { 'a[1].b.c': 3 },
}));
expect((incrRes as { a?: Array }).a).toEqual([1, { b: { c: 4 } }, 2]);
});
it('removes nested values including indexed list paths', async () => {
const actor = makeActor(19);
const key = 'remove-list-index';
await su.sudo(actor, () => kvStore.set({
key,
value: { a: { b: [1, 2, 3], c: { d: 4 }, e: 'keep' } },
}));
const updated = await su.sudo(actor, () => kvStore.remove({
key,
paths: ['a.b[1]', 'a.c'],
}));
expect((updated as { a?: { b?: number[]; e?: string } }).a).toEqual({ b: [1, 3], e: 'keep' });
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect((stored as { a?: { b?: number[]; e?: string } }).a).toEqual({ b: [1, 3], e: 'keep' });
});
it('rejects overlapping parent/child paths in a single request', async () => {
const actor = makeActor(20);
const key = 'overlap-paths';
await su.sudo(actor, () => kvStore.set({
key,
value: { a: { b: { c: 1 } } },
}));
await expect(su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { 'a.b': 1, 'a.b.c': 1 },
}))).rejects.toThrow(/paths overlap/i);
await expect(su.sudo(actor, () => kvStore.add({
key,
pathAndValueMap: { 'a.b': 1, 'a.b.c': 2 },
}))).rejects.toThrow(/paths overlap/i);
await expect(su.sudo(actor, () => kvStore.update({
key,
pathAndValueMap: { 'a.b': 1, 'a.b.c': 2 },
}))).rejects.toThrow(/paths overlap/i);
await expect(su.sudo(actor, () => kvStore.remove({
key,
paths: ['a.b', 'a.b.c'],
}))).resolves.not.toThrow();
});
it('incr initializes nested maps for missing keys', async () => {
const actor = makeActor(14);
const key = 'incr-missing';
const first = await su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { 'a.b.c': 2, 'x': 1 },
}));
expect(first).toMatchObject({ a: { b: { c: 2 } }, x: 1 });
const second = await su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { 'a.b.c': 3 },
}));
expect(second).toMatchObject({ a: { b: { c: 5 } }, x: 1 });
});
it('supports list index paths when incrementing', async () => {
const actor = makeActor(16);
const key = 'incr-list-index';
await su.sudo(actor, () => kvStore.set({
key,
value: { a: { b: [1, 2] } },
}));
const updated = await su.sudo(actor, () => kvStore.incr({
key,
pathAndAmountMap: { 'a.b[1]': 3 },
}));
expect((updated as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);
const stored = await su.sudo(actor, () => kvStore.get({ key }));
expect((stored as { a?: { b?: number[] } }).a?.b).toEqual([1, 5]);
});
it('enforces key and value size limits', async () => {
const actor = makeActor(11);
const oversizedKey = 'a'.repeat(((config as unknown as Record).kv_max_key_size as number) + 1);
const oversizedValue = 'b'.repeat(((config as unknown as Record).kv_max_value_size as number) + 1);
await expect(su.sudo(actor, () => kvStore.set({ key: oversizedKey, value: 'x' })))
.rejects
.toThrow(/1024/i);
await expect(su.sudo(actor, () => kvStore.set({ key: 'ok', value: oversizedValue })))
.rejects
.toThrow(/has exceeded the maximum allowed size/i);
});
});
================================================
FILE: src/backend/src/services/DynamoKVStore/DynamoKVStore.ts
================================================
import { Actor, SystemActorType } from '@heyputer/backend/src/services/auth/Actor.js';
import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js';
import type { MeteringService } from '@heyputer/backend/src/services/MeteringService/MeteringService.js';
import { RecursiveRecord } from '@heyputer/backend/src/services/MeteringService/types.js';
import { Context } from '@heyputer/backend/src/util/context.js';
import murmurhash from 'murmurhash';
import type { DDBClient } from '../../clients/dynamodb/DDBClient.js';
import { PUTER_KV_STORE_TABLE_DEFINITION } from './tableDefinition.js';
import { Span } from '../../util/otelutil.js';
import APIError from '../../api/APIError.js';
export class DynamoKVStore {
static GLOBAL_APP_KEY = 'os-global';
static LEGACY_GLOBAL_APP_KEY = 'global';
#ddbClient: DDBClient;
#sqlClient: BaseDatabaseAccessService;
#meteringService: MeteringService;
#tableName = 'store-kv-v1';
#pathCleanerRegex = /[:\-+/*]/g;
#enableMigrationFromSQL = false;
constructor ({ ddbClient, sqlClient, tableName, meteringService }: { ddbClient: DDBClient, sqlClient: BaseDatabaseAccessService, tableName: string, meteringService: MeteringService }) {
this.#ddbClient = ddbClient;
this.#sqlClient = sqlClient;
this.#tableName = tableName;
this.#meteringService = meteringService;
this.#enableMigrationFromSQL = !this.#ddbClient.config?.aws; // TODO: disable via config after some time passes
}
async createTableIfNotExists () {
if ( ! this.#enableMigrationFromSQL ) return;
await this.#ddbClient.createTableIfNotExists({ ...PUTER_KV_STORE_TABLE_DEFINITION, TableName: this.#tableName }, 'ttl');
}
#getNameSpace (actor: Actor) {
if ( actor.type instanceof SystemActorType ) {
return 'v1:system';
} else {
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
return `v1:${app ? `${user.uuid}:${app.uid}`
: `${user.uuid}:${this.#enableMigrationFromSQL ? DynamoKVStore.LEGACY_GLOBAL_APP_KEY : DynamoKVStore.GLOBAL_APP_KEY}`}`;
}
}
@Span('kv:get')
async get ({ key }: { key: string | string[]; }): Promise {
if ( key === '' ) {
throw APIError.create('field_empty', null, {
key: 'key',
});
}
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
const namespace = this.#getNameSpace(actor);
const multi = Array.isArray(key);
const keys = multi ? key : [key];
const values: unknown[] = [];
let kvEntries;
let usage;
if ( multi ) {
const entriesAndUsage = (await this.#getBatches(namespace, keys));
kvEntries = entriesAndUsage.kvEntries;
usage = entriesAndUsage.usage;
} else {
const res = await this.#ddbClient.get(this.#tableName, { namespace, key });
kvEntries = res.Item ? [res.Item] : [];
usage = res.ConsumedCapacity?.CapacityUnits ?? 0;
}
this.#meteringService.incrementUsage(actor, 'kv:read', usage || 0);
for ( const key of keys ) {
const kv_entry = kvEntries?.find(e => e.key === key);
const time = Date.now() / 1000;
if ( kv_entry?.ttl && kv_entry.ttl <= (time) ) {
values.push(null);
continue;
}
if ( kv_entry?.value ) {
values.push(kv_entry.value);
continue;
}
if ( this.#enableMigrationFromSQL ) {
const key_hash = murmurhash.v3(key);
const kv_row = await this.#sqlClient.read('SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1',
[user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]);
if ( kv_row[0]?.value ) {
// update and delete from this table
(async () => {
await this.set({ key: kv_row[0].key, value: kv_row[0].value });
await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?',
[user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]);
})();
values.push(kv_row[0]?.value);
continue;
}
}
values.push(kv_entry?.value ?? null);
}
return multi ? values : values[0];
}
/**
*
* @param {string} namespace
* @param {string[]} allKeys
* @returns
*/
async #getBatches (namespace: string, allKeys: string[]) {
const batches: string[][] = [];
for ( let i = 0; i < allKeys.length; i += 100 ) {
batches.push(allKeys.slice(i, i + 100));
}
const batchPromises = batches.map(async (keys) => {
const requests = [...new Set(keys)].map(k => ({ table: this.#tableName, items: { namespace, key: k } }));
const res = await this.#ddbClient.batchGet(requests);
const kvEntries = res.Responses?.[this.#tableName];
const usage = res.ConsumedCapacity?.reduce((acc, curr) => acc + (curr.CapacityUnits ?? 0), 0);
return { kvEntries, usage };
});
const batchGets = await Promise.all(batchPromises);
return batchGets.reduce((acc, curr) => {
acc.kvEntries!.push(...curr?.kvEntries ?? []);
acc.usage! += curr.usage || 0;
return acc;
}, { kvEntries: [], usage: 0 });
}
@Span('kv:set')
async set ({ key, value, expireAt }: { key: string; value: unknown; expireAt?: number; }): Promise {
const context = Context.get();
const actor = context.get('actor');
if ( key === '' ) {
throw APIError.create('field_empty', undefined, {
key: 'key',
});
}
key = String(key);
if ( Buffer.byteLength(key, 'utf8') > 1024 ) {
throw new Error(`key is too large. Max size is ${1024}.`);
}
if ( this.#enableMigrationFromSQL ) {
this.get({ key });
}
const namespace = this.#getNameSpace(actor);
const res = await this.#ddbClient.put(this.#tableName, {
namespace,
key,
value,
ttl: expireAt,
});
this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1);
return true;
}
@Span('kv:del')
async del ({ key }: { key: string; }): Promise {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
const namespace = this.#getNameSpace(actor);
const res = await this.#ddbClient.del(this.#tableName, {
namespace,
key,
});
this.#meteringService.incrementUsage(actor, 'kv:write', res?.ConsumedCapacity?.CapacityUnits ?? 1);
if ( this.#enableMigrationFromSQL ) {
const key_hash = murmurhash.v3(key);
await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?',
[user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY, key_hash]);
}
return true;
}
#encodeCursor (pageKey?: Record) {
if ( !pageKey || Object.keys(pageKey).length === 0 ) {
return undefined;
}
return Buffer.from(JSON.stringify(pageKey)).toString('base64');
}
#decodeCursor (cursor?: string | Record) {
if ( ! cursor ) {
return undefined;
}
if ( typeof cursor === 'object' ) {
return cursor;
}
if ( typeof cursor !== 'string' ) {
throw APIError.create('field_invalid', undefined, {
key: 'cursor',
});
}
const trimmed = cursor.trim();
if ( trimmed === '' ) {
return undefined;
}
try {
const decoded = Buffer.from(trimmed, 'base64').toString('utf8');
return JSON.parse(decoded);
} catch ( e ) {
try {
return JSON.parse(trimmed);
} catch ( err ) {
throw APIError.create('field_invalid', undefined, {
key: 'cursor',
});
}
}
}
#normalizeLimit (limit?: number) {
if ( limit === undefined || limit === null ) {
return undefined;
}
const parsed = Number(limit);
if ( !Number.isFinite(parsed) || parsed <= 0 ) {
throw APIError.create('field_invalid', undefined, {
key: 'limit',
expected: 'positive number',
});
}
return Math.floor(parsed);
}
#normalizePattern (pattern?: string) {
if ( pattern === undefined || pattern === null ) {
return undefined;
}
if ( typeof pattern !== 'string' ) {
throw APIError.create('field_invalid', undefined, {
key: 'pattern',
});
}
const trimmed = pattern.trim();
if ( trimmed === '' ) {
return undefined;
}
if ( trimmed.endsWith('*') ) {
const prefix = trimmed.slice(0, -1);
return prefix === '' ? undefined : prefix;
}
return trimmed;
}
@Span('kv:list')
async list ({
as,
limit,
cursor,
pattern,
}: {
as?: 'keys' | 'values' | 'entries';
limit?: number;
cursor?: string | Record;
pattern?: string;
}): Promise<
| string[]
| unknown[]
| { key: string; value: unknown; }[]
| { items: string[]; cursor?: string; }
| { items: unknown[]; cursor?: string; }
| { items: { key: string; value: unknown; }[]; cursor?: string; }
> {
const actor = Context.get('actor');
const app = actor.type?.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
const namespace = this.#getNameSpace(actor);
const normalizedLimit = this.#normalizeLimit(limit);
const pageKey = this.#decodeCursor(cursor);
const normalizedPattern = this.#normalizePattern(pattern);
const paginated = normalizedLimit !== undefined || pageKey !== undefined;
const entriesRes = await this.#ddbClient.query(this.#tableName,
{ namespace },
normalizedLimit ?? 0,
pageKey,
'',
false,
normalizedPattern ? { beginsWith: { key: 'key', value: normalizedPattern } } : undefined);
this.#meteringService.incrementUsage(actor, 'kv:read', entriesRes.ConsumedCapacity?.CapacityUnits ?? 1);
let entries = entriesRes.Items ?? [];
entries = entries?.filter(entry => {
if ( ! entry ) {
return false;
}
if ( entry.ttl && entry.ttl <= (Date.now() / 1000) ) {
return false;
}
return true;
});
if ( this.#enableMigrationFromSQL && !paginated ) {
const oldEntries = await this.#sqlClient.read('SELECT * FROM kv WHERE user_id=? AND app=?',
[user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY]);
oldEntries.forEach(oldEntry => {
if ( normalizedPattern && !oldEntry.kkey?.startsWith(normalizedPattern) ) {
return;
}
if ( ! entries.find(e => e.key === oldEntry.kkey) ) {
if ( oldEntry.ttl && oldEntry.ttl <= (Date.now() / 1000) ) {
entries.push({ key: oldEntry.kkey, value: oldEntry.value });
}
}
});
}
entries = entries?.map(entry => ({
key: entry.key,
value: entry.value,
}));
as = as || 'entries';
if ( ! ['keys', 'values', 'entries'].includes(as) ) {
throw APIError.create('field_invalid', undefined, {
key: 'as',
expected: '"keys", "values", or "entries"',
});
}
let items: string[] | unknown[] | { key: string; value: unknown; }[] = entries;
if ( as === 'keys' ) items = entries.map(entry => entry.key);
else if ( as === 'values' ) items = entries.map(entry => entry.value);
if ( paginated ) {
const nextCursor = this.#encodeCursor(entriesRes.LastEvaluatedKey as Record | undefined);
if ( nextCursor ) {
return { items, cursor: nextCursor };
}
return { items };
}
return items;
}
@Span('kv:flush')
async flush () {
const actor = Context.get('actor');
const app = actor.type.app ?? undefined;
const user = actor.type?.user ?? undefined;
if ( ! user ) throw new Error('User not found');
const namespace = this.#getNameSpace(actor);
// Query all keys
const entriesRes = await this.#ddbClient.query(this.#tableName,
{ namespace });
const entries = entriesRes.Items ?? [];
const readUsage = entriesRes?.ConsumedCapacity?.CapacityUnits ?? 0;
// meter usage
this.#meteringService.incrementUsage(actor, 'kv:read', readUsage);
// TODO DS: implement batch delete so its faster and less demanding on server
const allRes = (await Promise.all(entries.map(entry => {
try {
return this.#ddbClient.del(this.#tableName, {
namespace,
key: entry.key,
});
} catch ( e ) {
console.error('Error deleting key', entry.key, e);
}
}))).filter(Boolean);
const writeUsage = allRes.reduce((acc, curr) => acc + (curr?.ConsumedCapacity?.CapacityUnits ?? 0), 0);
// meter usage
this.#meteringService.incrementUsage(actor, 'kv:write', writeUsage);
if ( this.#enableMigrationFromSQL ) {
await this.#sqlClient.write('DELETE FROM kv WHERE user_id=? AND app=?',
[user.id, app?.uid ?? DynamoKVStore.LEGACY_GLOBAL_APP_KEY]);
}
return !!allRes;
}
@Span('kv:expireAt')
async expireAt ({ key, timestamp }: { key: string; timestamp: number; }): Promise {
if ( key === '' ) {
throw APIError.create('field_empty', null, {
key: 'key',
});
}
timestamp = Number(timestamp);
return await this.#expireAt(key, timestamp);
}
@Span('kv:expire')
async expire ({ key, ttl }: { key: string; ttl: number; }): Promise {
if ( key === '' ) {
throw APIError.create('field_empty', null, {
key: 'key',
});
}
ttl = Number(ttl);
// timestamp in seconds
let timestamp = Math.floor(Date.now() / 1000) + ttl;
return await this.#expireAt(key, timestamp);
}
async #createPaths ( namespace: string, key: string, pathList: string[]) {
const nestedMapValue = (() => {
const valueRoot: Record = {};
let hasPaths = false;
pathList.forEach((valPath) => {
if ( ! valPath ) return;
hasPaths = true;
const chunks = valPath.split('.').filter(Boolean);
let cursor: Record = valueRoot;
for ( let i = 0; i < chunks.length - 1; i++ ) {
const chunk = chunks[i];
const existing = cursor[chunk];
if ( !existing || typeof existing !== 'object' || Array.isArray(existing) ) {
cursor[chunk] = {};
}
cursor = cursor[chunk] as Record;
}
});
return hasPaths ? valueRoot : null;
})();
if ( ! nestedMapValue ) {
return 0;
}
const isPlainObject = (value: unknown): value is Record => {
return !!value && typeof value === 'object' && !Array.isArray(value);
};
const objectsEqual = (left: unknown, right: unknown): boolean => {
if ( left === right ) return true;
if ( !isPlainObject(left) || !isPlainObject(right) ) return false;
const leftKeys = Object.keys(left);
const rightKeys = Object.keys(right);
if ( leftKeys.length !== rightKeys.length ) return false;
for ( const key of leftKeys ) {
if ( ! rightKeys.includes(key) ) return false;
if ( ! objectsEqual(left[key], right[key]) ) return false;
}
return true;
};
// Collect all intermediate map paths for all entries
const allIntermediatePaths = new Set();
pathList.forEach((valPath) => {
const chunks = ['value', ...valPath.split('.')].filter(Boolean);
// For each intermediate map (excluding the leaf)
for ( let i = 1; i < chunks.length; i++ ) {
const subPath = chunks.slice(0, i).join('.');
allIntermediatePaths.add(subPath);
}
});
let writeUnits = 0;
// Ensure each intermediate map layer exists by issuing a separate DynamoDB update for each
const orderedPaths = [...allIntermediatePaths]
.sort((left, right) => left.split('.').length - right.split('.').length);
for ( const layerPath of orderedPaths ) {
// Build attribute names for the layer
const chunks = layerPath.split('.');
const attrName = chunks.map((chunk) => `#${chunk}`.replaceAll(this.#pathCleanerRegex, '')).join('.');
const expressionNames: Record = {};
chunks.forEach((chunk) => {
const cleanedChunk = chunk.split(/\[\d*\]/g)[0];
expressionNames[`#${cleanedChunk}`.replaceAll(this.#pathCleanerRegex, '')] = cleanedChunk;
});
const isRootLayer = layerPath === 'value';
const expressionValues = isRootLayer
? { ':nestedMap': nestedMapValue }
: { ':emptyMap': {} };
const valueToken = isRootLayer ? ':nestedMap' : ':emptyMap';
// Issue update to set layer to {} if not exists
const layerUpsertRes = await this.#ddbClient.update(this.#tableName,
{ key, namespace },
`SET ${attrName} = if_not_exists(${attrName}, ${valueToken})`,
expressionValues,
expressionNames);
writeUnits += layerUpsertRes.ConsumedCapacity?.CapacityUnits ?? 0;
if ( isRootLayer && objectsEqual(layerUpsertRes.Attributes?.value, nestedMapValue) ) {
return writeUnits;
}
}
return writeUnits;
}
// Ideally the paths support syntax like "a.b[2].c"
@Span('kv:incr')
async incr>({ key, pathAndAmountMap }: { key: string; pathAndAmountMap: T; }): Promise
