diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b5e43f2c..13fd8208f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ See [STATUS.md](server/STATUS.md) to learn more about which features will remain ## Unreleased - The download endpoint can now optimize images on the fly. This is controlled via query parameters. #257 +- Added export endpoint for exporting resources to other formats. Currently only supports exporting tables to csv. [#925](https://github.com/atomicdata-dev/atomic-server/issues/925) ## [v0.38.0] - 2024-06-08 diff --git a/Cargo.lock b/Cargo.lock index 215557cba..145af25f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -576,6 +576,7 @@ dependencies = [ "sled", "toml 0.8.8", "tracing", + "ulid", "ureq", "url", "urlencoding", @@ -1664,8 +1665,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -4594,7 +4597,7 @@ dependencies = [ "tracing-core", "tracing-log", "tracing-subscriber", - "web-time", + "web-time 0.2.4", ] [[package]] @@ -4627,6 +4630,17 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "ulid" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f903f293d11f31c0c29e4148f6dc0d033a7f80cebc0282bea147611667d289" +dependencies = [ + "getrandom 0.2.11", + "rand 0.8.5", + "web-time 1.1.0", +] + [[package]] name = "unicase" version = "2.6.0" @@ -4894,6 +4908,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webp" version = "0.3.0" diff --git a/browser/CHANGELOG.md b/browser/CHANGELOG.md index f26f7ee41..950d54329 100644 --- a/browser/CHANGELOG.md +++ b/browser/CHANGELOG.md @@ -21,6 +21,7 @@ This changelog covers all five packages, as they are (for now) updated as a whol - [#861](https://github.com/atomicdata-dev/atomic-server/issues/861) Fix long usernames overflowing on the share page. - [#906](https://github.com/atomicdata-dev/atomic-server/issues/906) Reset changes after clicking the cancel button in a form or navigating away. - [#914](https://github.com/atomicdata-dev/atomic-server/issues/914) Fix an issue where changing the subject in a new resource form could update the parent of existing resources if their subject matched the new subject. +- [#925](https://github.com/atomicdata-dev/atomic-server/issues/925) Added export to CSV option to tables. - [#919](https://github.com/atomicdata-dev/atomic-server/issues/919) Automatically sort classes and properties in the ontology editor. - [#936](https://github.com/atomicdata-dev/atomic-server/issues/936) Updated the address bar to make it clearer it's also search bar. diff --git a/browser/data-browser/src/components/Button.tsx b/browser/data-browser/src/components/Button.tsx index 31be00e89..9f59ca612 100644 --- a/browser/data-browser/src/components/Button.tsx +++ b/browser/data-browser/src/components/Button.tsx @@ -165,10 +165,9 @@ export const ButtonDefault = styled(ButtonBase)` --button-text-color: ${p => p.theme.colors.bg}; --button-text-color-hover: ${p => p.theme.colors.bg}; - padding: 0.4rem; border-radius: ${p => p.theme.radius}; - padding-left: ${p => p.theme.margin}rem; - padding-right: ${p => p.theme.margin}rem; + padding-block: 0.4rem; + padding-inline: ${p => p.theme.margin}rem; display: inline-flex; background-color: var(--button-bg-color); color: var(--button-text-color); diff --git a/browser/data-browser/src/components/ButtonLink.tsx b/browser/data-browser/src/components/ButtonLink.tsx new file mode 100644 index 000000000..0f19cfe2c --- /dev/null +++ b/browser/data-browser/src/components/ButtonLink.tsx @@ -0,0 +1,17 @@ +import React from 'react'; +import { ButtonDefault } from './Button'; +import styled from 'styled-components'; + +export function ButtonLink( + props: React.AnchorHTMLAttributes, +): React.JSX.Element { + return ( + + {props.children} + + ); +} + +const StyledButtonDefault = styled(ButtonDefault)` + text-decoration: none; +`; diff --git a/browser/data-browser/src/components/Dropdown/index.tsx b/browser/data-browser/src/components/Dropdown/index.tsx index d8698bf28..dab7c210a 100644 --- a/browser/data-browser/src/components/Dropdown/index.tsx +++ b/browser/data-browser/src/components/Dropdown/index.tsx @@ -394,7 +394,6 @@ const MenuItemStyled = styled(Button)` color: ${p => p.theme.colors.text}; padding: 0.4rem 1rem; height: auto; - text-transform: capitalize; background-color: ${p => p.selected ? p.theme.colors.bg1 : p.theme.colors.bg}; text-decoration: ${p => (p.selected ? 'underline' : 'none')}; diff --git a/browser/data-browser/src/components/ResourceContextMenu/index.tsx b/browser/data-browser/src/components/ResourceContextMenu/index.tsx index 43a0327c5..2900db053 100644 --- a/browser/data-browser/src/components/ResourceContextMenu/index.tsx +++ b/browser/data-browser/src/components/ResourceContextMenu/index.tsx @@ -49,6 +49,7 @@ export enum ContextMenuOptions { Import = 'import', UseInCode = 'useInCode', NewChild = 'newChild', + Export = 'export', } export interface ResourceContextMenuProps { @@ -117,14 +118,14 @@ function ResourceContextMenu({ { disabled: location.pathname.startsWith(paths.show), id: ContextMenuOptions.View, - label: 'normal view', + label: 'Normal View', helper: 'Open the regular, default View.', onClick: () => navigate(constructOpenURL(subject)), }, { disabled: location.pathname.startsWith(paths.data), id: ContextMenuOptions.Data, - label: 'data view', + label: 'Data View', helper: 'View the resource and its properties in the Data View.', shortcut: shortcuts.data, onClick: () => navigate(dataURL(subject)), @@ -136,7 +137,7 @@ function ResourceContextMenu({ { // disabled: !canWrite || location.pathname.startsWith(paths.edit), id: ContextMenuOptions.Edit, - label: 'edit', + label: 'Edit', helper: 'Open the edit form.', icon: , shortcut: simple ? '' : shortcuts.edit, @@ -144,7 +145,7 @@ function ResourceContextMenu({ }, { id: ContextMenuOptions.NewChild, - label: 'add child', + label: 'Add child', helper: 'Create a new resource under this resource.', icon: , onClick: handleAddClick, @@ -152,7 +153,7 @@ function ResourceContextMenu({ ), { id: ContextMenuOptions.UseInCode, - label: 'use in code', + label: 'Use in code', helper: 'Usage instructions for how to fetch and use the resource in your code.', icon: , @@ -160,7 +161,7 @@ function ResourceContextMenu({ }, { id: ContextMenuOptions.Scope, - label: 'search children', + label: 'Search children', helper: 'Scope search to resource', icon: , onClick: enableScope, @@ -176,7 +177,7 @@ function ResourceContextMenu({ { id: ContextMenuOptions.History, icon: , - label: 'history', + label: 'History', helper: 'Show the history of this resource', onClick: () => navigate(historyURL(subject)), }, @@ -185,7 +186,7 @@ function ResourceContextMenu({ { id: ContextMenuOptions.Import, icon: , - label: 'import', + label: 'Import', helper: 'Import Atomic Data to this resource', onClick: () => navigate(importerURL(subject)), }, @@ -193,7 +194,7 @@ function ResourceContextMenu({ disabled: !canWrite, id: ContextMenuOptions.Delete, icon: , - label: 'delete', + label: 'Delete', helper: 'Delete this resource.', onClick: () => setShowDeleteDialog(true), }, diff --git a/browser/data-browser/src/components/SideBar/About.tsx b/browser/data-browser/src/components/SideBar/About.tsx index 9da06936d..bfbe63414 100644 --- a/browser/data-browser/src/components/SideBar/About.tsx +++ b/browser/data-browser/src/components/SideBar/About.tsx @@ -15,7 +15,7 @@ const aboutMenuItems: AboutItem[] = [ { icon: , helper: 'Github; View the source code for this application', - href: 'https://github.com/atomicdata-dev/atomic-data-browser', + href: 'https://github.com/atomicdata-dev/atomic-server', }, { icon: , diff --git a/browser/data-browser/src/components/SideBar/AppMenu.tsx b/browser/data-browser/src/components/SideBar/AppMenu.tsx index e94d8cd0a..c108ee548 100644 --- a/browser/data-browser/src/components/SideBar/AppMenu.tsx +++ b/browser/data-browser/src/components/SideBar/AppMenu.tsx @@ -10,7 +10,12 @@ import { constructOpenURL } from '../../helpers/navigation'; import { useCurrentSubject } from '../../helpers/useCurrentSubject'; import { SideBarMenuItem } from './SideBarMenuItem'; import { paths } from '../../routes/paths'; -import { unknownSubject, useCurrentAgent, useResource } from '@tomic/react'; +import { + core, + unknownSubject, + useCurrentAgent, + useResource, +} from '@tomic/react'; // Non standard event type so we have to type it ourselfs for now. type BeforeInstallPromptEvent = { @@ -59,7 +64,11 @@ export function AppMenu({ onItemClick }: AppMenuProps): JSX.Element {
} - label={agent ? agentResource.title : 'Login'} + label={ + agent + ? agentResource.get(core.properties.name) ?? 'User Settings' + : 'Login' + } helper='See and edit the current Agent / User (u)' path={paths.agentSettings} onClick={onItemClick} diff --git a/browser/data-browser/src/views/TablePage/TableExportDialog.tsx b/browser/data-browser/src/views/TablePage/TableExportDialog.tsx new file mode 100644 index 000000000..afc247b75 --- /dev/null +++ b/browser/data-browser/src/views/TablePage/TableExportDialog.tsx @@ -0,0 +1,68 @@ +import { useEffect, useState } from 'react'; +import { useStore, type Store } from '@tomic/react'; +import { FaDownload } from 'react-icons/fa6'; +import { + Dialog, + DialogActions, + DialogContent, + DialogTitle, + useDialog, +} from '../../components/Dialog'; +import { Checkbox, CheckboxLabel } from '../../components/forms/Checkbox'; +import { ButtonLink } from '../../components/ButtonLink'; + +interface TableExportDialogProps { + subject: string; + show: boolean; + bindShow: React.Dispatch; +} + +const buildLink = (subject: string, refAsSubject: boolean, store: Store) => { + const url = new URL(`${store.getServerUrl()}/export`); + + url.searchParams.set('format', 'csv'); + url.searchParams.set('subject', subject); + url.searchParams.set('display_refs_as_name', refAsSubject ? 'false' : 'true'); + + return url.toString(); +}; + +export function TableExportDialog({ + subject, + show, + bindShow, +}: TableExportDialogProps): React.JSX.Element { + const store = useStore(); + const [dialogProps, showDialog] = useDialog({ bindShow }); + const [refAsSubject, setRefAsSubject] = useState(false); + + const url = buildLink(subject, refAsSubject, store); + + useEffect(() => { + if (show) { + showDialog(); + } + }, [show, showDialog]); + + return ( + <> + + +

Export table as CSV

+ + + + {' '} + Reference resources by subject instead of name. + + + + + + Download + + + + + ); +} diff --git a/browser/data-browser/src/views/TablePage/TablePage.tsx b/browser/data-browser/src/views/TablePage/TablePage.tsx index e717c99a1..672315193 100644 --- a/browser/data-browser/src/views/TablePage/TablePage.tsx +++ b/browser/data-browser/src/views/TablePage/TablePage.tsx @@ -16,9 +16,14 @@ import { createResourceDeletedHistoryItem, useTableHistory, } from './helpers/useTableHistory'; +import { Row as FlexRow, Column } from '../../components/Row'; import { useHandleClearCells } from './helpers/useHandleClearCells'; import { useHandleCopyCommand } from './helpers/useHandleCopyCommand'; import { ExpandedRowDialog } from './ExpandedRowDialog'; +import { IconButton } from '../../components/IconButton/IconButton'; +import { FaCode, FaFileCsv } from 'react-icons/fa6'; +import { ResourceCodeUsageDialog } from '../CodeUsage/ResourceCodeUsageDialog'; +import { TableExportDialog } from './TableExportDialog'; const columnToKey = (column: Property) => column.subject; @@ -26,6 +31,8 @@ export function TablePage({ resource }: ResourcePageProps): JSX.Element { const store = useStore(); const titleId = useId(); + const [showCodeUsageDialog, setShowCodeUsageDialog] = useState(false); + const [showExportDialog, setShowExportDialog] = useState(false); const { tableClass, sorting, setSortBy, collection, invalidateCollection } = useTableData(resource); @@ -114,32 +121,60 @@ export function TablePage({ resource }: ResourcePageProps): JSX.Element { return ( - - - {Row} - + + + + + setShowCodeUsageDialog(true)} + > + + + setShowExportDialog(true)} + > + + + + + + {Row} + + + + ); } diff --git a/browser/e2e/tests/ontology.spec.ts b/browser/e2e/tests/ontology.spec.ts index 922b35d14..57f164f30 100644 --- a/browser/e2e/tests/ontology.spec.ts +++ b/browser/e2e/tests/ontology.spec.ts @@ -222,8 +222,8 @@ test.describe('Ontology', async () => { await expect(page.getByText('new arrow-kind')).toBeVisible(); - await expect(page.getByLabel('name')).toBeVisible(); - await page.getByLabel('name').fill(name); + await expect(currentDialog(page).getByLabel('name')).toBeVisible(); + await currentDialog(page).getByLabel('name').fill(name); await currentDialog(page).getByRole('button', { name: 'Save' }).click(); await expect(page.getByRole('heading', { name })).toBeVisible(); diff --git a/browser/svelte/package.json b/browser/svelte/package.json index a53103838..d3f498aa4 100644 --- a/browser/svelte/package.json +++ b/browser/svelte/package.json @@ -16,7 +16,7 @@ "prepublishOnly": "npm run package", "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", - "test": "vitest", + "test": "vitest run", "lint": "prettier --check . && eslint .", "format": "prettier --write ." }, diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index d124ce593..f110d2649 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -12,7 +12,8 @@ - [AtomicServer](atomic-server.md) - [When (not) to use it](atomicserver/when-to-use.md) - [Installation](atomicserver/installation.md) - - [Getting started with the GUI](atomicserver/gui.md) + - [Using the GUI](atomicserver/gui.md) + - [Tables](atomicserver/gui/tables.md) - [API](atomicserver/API.md) - [Creating a JSON-AD file](create-json-ad.md) - [FAQ & troubleshooting](atomicserver/faq.md) diff --git a/docs/src/assets/ui-guide/gui-tables-example.avif b/docs/src/assets/ui-guide/gui-tables-example.avif new file mode 100644 index 000000000..536bc2e49 Binary files /dev/null and b/docs/src/assets/ui-guide/gui-tables-example.avif differ diff --git a/docs/src/assets/ui-guide/ui-guide-fresh-install.avif b/docs/src/assets/ui-guide/ui-guide-fresh-install.avif new file mode 100644 index 000000000..768a64eb5 Binary files /dev/null and b/docs/src/assets/ui-guide/ui-guide-fresh-install.avif differ diff --git a/docs/src/atomicserver/gui.md b/docs/src/atomicserver/gui.md index 33a2d4d71..9176907ba 100644 --- a/docs/src/atomicserver/gui.md +++ b/docs/src/atomicserver/gui.md @@ -1,73 +1,55 @@ # Using the AtomicServer GUI -## Creating the first Agent on AtomicData.dev - -Before you can create new things on AtomicData.dev, you'll need an _Agent_. -This is your virtual User, which can create, sign and own things. - -Simply open the [demo invite](https://atomicdata.dev/invites/1) and press accept. - -Copy the `secret` from the user settings page and save it somewhere safe, like in a password manager. - -## Using your local AtomicServer - After [running the server](installation.md), open it in your browser. By default, that's at [`http://localhost:9883`](http://localhost:9883). - +> Fun fact: `⚛` is HTML entity code for the Atom icon: ⚛ The first screen should show you your main [_Drive_](https://atomicdata.dev/classes/Drive). -You can think of this as your root folder. +You can think of this as the root of the server. It is the resource hosted at the root URL, effectively being the home page of your server. -There's an instruction on the screen about the `/setup` page. -Click this, and you'll get a screen showing an [_Invite_](https://atomicdata.dev/classes/Invite). -Normally, you could `Accept as new user`, but **since you're running on `localhost`, you won't be able to use the newly created Agent on non-local Atomic-Servers**. - -Therefore, it may be best to create an Agent on some _other_ running server, such as the [demo Invite on AtomicData.dev](https://atomicdata.dev/invites/1). -And after that, copy the Secret from the `User settings` panel from AtomicData.dev, go back to your `localhost` version, and press `sign in`. -Paste the Secret, and voila! You're signed in. - -Now, again go to `/setup`. This time, you can `Accept as {user}`. -After clicking, your Agent has gotten `write` rights for the Drive! -You can verify this by hovering over the description field, clicking the edit icon, and making a few changes. -You can also press the menu button (three dots, top left) and press `Data view` to see your agent after the `write` field. -Note that you can now edit every field. -You can also fetch your data now as various formats. - -Try checking out the other features in the menu bar, and check out the `collections`. +In the sidebar you will see a list of resources in the current drive. +At the start these will be: -Again, check out the [README](https://github.com/atomicdata-dev/atomic-server) for more information and guides! +- The setup invite that's used to configure the root agent. +- A resource named `collections`. This is a group of collections that shows collections for all classes in the server, essentially a list of all resources. +- The default ontology. Ontologies are used to define new classes and properties and show to relation between them. -Now, let's create some data. +![The AtomicServer GUI](../assets/ui-guide/ui-guide-fresh-install.avif) -## Creating your first Atomic Data +## Creating an agent +To create data in AtomicServer you'll need an agent. +An agent is like a user account, it signs the changes (commits) you make to data so that others can verify that you made them. +Agents can be used on any AtomicServer as long as they have permission to do so. -Now let's create a [_Table_](https://atomicdata.dev/classes/Table). -A Class represents an abstract concept, such as a `BlogPost` (which we'll do here). -We can do this in a couple of ways: +If your AtomicServer is not reachable from the outside we recommend you create an agent on a public server like [atomicdata.dev](https://atomicdata.dev) as an agent created on a local server can only be used on that server. +The server that hosts your agent cannot do anything on your behalf because your private key is not stored on the server. They can however delete your agent making it unusable. -- Press the `+ icon` button on the left menu (only visible when logged in), and selecting Class -- Opening [Class](https://atomicdata.dev/classes/Class) and pressing `new class` -- Going to the [Classes Collection](https://atomicdata.dev/classes/) and pressing the plus icon +To create an agent on atomicdata.dev you can use the [demo invite](https://atomicdata.dev/invites/1). +If you want to create the agent on your own server you can use the `/setup` invite that was created when you first started the server. -The result is the same: we end up with a form in which we can fill in some details. +Click the "Accept as new user" button and navigate to the User Settings page. +Here you'll find the agent secret. This secret is what you use to login so keep it somewhere safe, like in a password manager. If you lose it you won't be able to recover your account. -Let's add a shortname (singular), and then a description. -After that, we'll add the `required` properties. -This form you're looking at is constructed by using the `required` and `recommended` Properties defined in `Class`. -We can use these same fields to generate our BlogPost resource! -Which fields would be required in a `BlogPost`? -A `name`, and a `description`, probably. +### Setting up the root Agent +Next, we'll set up the root Agent that has write access to the Drive. +If you've chosen to create an Agent on this server using the `/setup` invite, you can skip this step. -So click on the `+ icon` under `requires` and search for these Properties to add them. +Head to the `setup` page by selecting it in the sidebar. +You'll see a button that either says `Accept as ` or `Accept as new user`. +If it says 'as new user`, click on login, paste your secret in the input field and return to the invite page. -Now, we can skip the `recommended` properties, and get right to saving our newly created `BlogPost` class. -So, press save, and now look at what you created. +After clicking the accept button you'll be redirected to the home page and you will have write access to the Drive. +You can verify this by hovering over the description field, clicking the edit icon, and making a few changes. +You can also press the menu button (three dots, top left) and press `Data view` to see your agent after the `write` field. +Note that you can now edit every field. -Notice a couple of things: +The `/setup`-invite can only be used once use and will therefore not work anymore. +If you want to re-enable the invite to change the root agent you can start AtomicServer with the `--initialize` flag. -- Your Class has its own URL. -- It has a `parent`, shown in the top of the screen. This has impact on the visibility and rights of your Resource. We'll get to that [later in the documentation](../hierarchy.md). +## Creating your first Atomic Data -Now, go to the navigation bar, which is by default at the bottom of the window. Use its context menu to open the `Data View`. -This view gives you some more insight into your newly created data, and various ways in which you can serialize it. +Now that everything is up and running you can start creating some resources. +To create a new resource, click the + button in the sidebar. +You will be presented with a list of resource types to choose from like Tables, Folders, Documents etc. +You can also create your own types by using ontologies. diff --git a/docs/src/atomicserver/gui/tables.md b/docs/src/atomicserver/gui/tables.md new file mode 100644 index 000000000..2468d52f0 --- /dev/null +++ b/docs/src/atomicserver/gui/tables.md @@ -0,0 +1,31 @@ +# Tables + +Tables are a way to create and group large amounts of structured data. + +![Table](../../assets/ui-guide/gui-tables-example.avif) + +Tables consist of rows of resources that share the same parent and class. +The properties of that class are represented as columns in the table. +This means that each column is type-safe, a number column can not contain text data for example. + +## Creating a table + +To create a table, click the "+" button in the sidebar or a folder and select "Table". +A dialog will appear prompting you to enter a name. +This name will be used as the title of the table as well as the name for the underlying class of the rows. +This new class will already have a `name` property. Using the `name` property as titles on your resources is a best practice as it helps with compatibility between other tools and makes your resources findable by AtomicServer's search functionality. +If you do not want to use the `name` property, you can remove it by clicking on the three dots in the column header and selecting "Remove". + +While creating a new table you can also choose to use an existing class by selecting "Use existing class" in the dialog and selecting the desired class from the dropdown. + +Classes created by tables are automatically added to the default ontology of the drive. Same goes for the columns of the table. +If you chose to use an existing class, any columns created will be added to the ontology containing that class. + +## Features + +- **Rearange columns**: You can drag and drop columns to rearrange them. +- **Resize columns**: You can resize columns by dragging the edges of the column header. +- **Sort rows**: Click on a column header to sort the rows by that column. +- **Fast keyboard navigation**: Use the arrow keys to navigate the table with hotkeys similar to Excel. +- **Copy & paste multiple cells**: You can copy and paste multiple cells by selecting them and using `Ctrl/Cmd + C` and `Ctrl/Cmd + V`. Pasting also works across different tables and even different applications that support HTML Table data (Most spreadsheet applications). +- **Export data to CSV**: You can export the data of a table to a CSV file by clicking the "Export" button in the top right. diff --git a/docs/src/atomicserver/installation.md b/docs/src/atomicserver/installation.md index aafbf7de6..9b133657f 100644 --- a/docs/src/atomicserver/installation.md +++ b/docs/src/atomicserver/installation.md @@ -8,8 +8,7 @@ You can run AtomicServer in different ways: 3. Using [Cargo](https://doc.rust-lang.org/cargo/getting-started/installation.html) from crates.io: `cargo install atomic-server` 4. Manually from source -When you're running AtomicServer, go to [Initial setup and configuration](#Initial-setup-and-configuration). -If you want to run this locally as a developer / contributor, check out [the Contributors guide](https://github.com/atomicdata-dev/atomic-server/blob/develop/CONTRIBUTING.md). +If you want to run AtomicServer locally as a developer / contributor, check out [the Contributors guide](https://github.com/atomicdata-dev/atomic-server/blob/develop/CONTRIBUTING.md). ## 1. Run using docker @@ -59,7 +58,7 @@ sudo apt-get install -y build-essential pkg-config libssl-dev --fix-missing - The server loads the `.env` from the current path by default. Create a `.env` file from the default template in your current directory with `atomic-server generate-dotenv` - After running the server, check the logs and take note of the `Agent Subject` and `Private key`. You should use these in the [`atomic-cli`](https://crates.io/crates/atomic-cli) and [atomic-data-browser](https://github.com/atomicdata-dev/atomic-data-browser) clients for authorization. - A directory is made: `~/.config/atomic`, which stores your newly created Agent keys, the HTTPS certificates other configuration. Depending on your OS, the actual data is stored in different locations. See use the `show-config` command to find out where, if you need the files. -- Visit `http://localhost:9883/setup` to **register your first (admin) user**. You can use an existing Agent, or create a new one. Note that if you create a `localhost` agent, it cannot be used on the web (since, well, it's local). +- Visit `http://localhost:9883/setup` to **register your first (admin) user**. You can use an existing Agent, or create a new one. Note that if you create a `localhost` agent, it cannot be used on the web (since, well, it's local). More info and steps in [getting started with the GUI](gui.md). ## Running using a tunneling service (easy mode) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 8d93bdc41..3f6a68bc0 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -36,6 +36,7 @@ tracing = "0.1" ureq = "2" url = "2" urlencoding = "2" +ulid = "1.1.3" [dev-dependencies] criterion = "0.5" diff --git a/lib/src/endpoints.rs b/lib/src/endpoints.rs index bd56bb48d..c3699fd60 100644 --- a/lib/src/endpoints.rs +++ b/lib/src/endpoints.rs @@ -82,6 +82,7 @@ pub fn default_endpoints() -> Vec { plugins::search::search_endpoint(), plugins::files::upload_endpoint(), plugins::files::download_endpoint(), + plugins::export::export_endpoint(), #[cfg(feature = "html")] plugins::bookmark::bookmark_endpoint(), plugins::importer::import_endpoint(), diff --git a/lib/src/plugins/export.rs b/lib/src/plugins/export.rs new file mode 100644 index 000000000..a79daa035 --- /dev/null +++ b/lib/src/plugins/export.rs @@ -0,0 +1,19 @@ +use crate::endpoints::Endpoint; + +pub fn export_endpoint() -> Endpoint { + Endpoint { + path: "/export".to_string(), + params: vec!["subject".into(), "format".into(), "display_refs_as_name".into()], + description: r#"Export table data + +Use with the following parameters +- **subject**: Subject of the resource to export. +- **format**: Format of the export, currently only supports `csv`. +- **display_refs_as_name**: If true, it will display referenced resources by their name instead of subject. +"# + .to_string(), + shortname: "export".to_string(), + handle: None, + handle_post: None, + } +} diff --git a/lib/src/plugins/mod.rs b/lib/src/plugins/mod.rs index 31a2bdcf1..9f383bf51 100644 --- a/lib/src/plugins/mod.rs +++ b/lib/src/plugins/mod.rs @@ -41,6 +41,7 @@ pub mod invite; // Endpoints #[cfg(feature = "html")] pub mod bookmark; +pub mod export; pub mod files; pub mod path; pub mod prunetests; diff --git a/lib/src/resources.rs b/lib/src/resources.rs index 59dc990b3..f9fedbd7f 100644 --- a/lib/src/resources.rs +++ b/lib/src/resources.rs @@ -15,6 +15,7 @@ use crate::{ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use tracing::instrument; +use ulid::Ulid; /// A Resource is a set of Atoms that shares a single Subject. /// A Resource only contains valid Values, but it _might_ lack required properties. @@ -209,11 +210,15 @@ impl Resource { } } + pub fn random_subject(store: &impl Storelike) -> String { + format!("{}/{}", store.get_server_url(), Ulid::new().to_string()) + } + /// Create a new resource with a generated Subject pub fn new_generate_subject(store: &impl Storelike) -> Resource { - let generated = format!("{}/{}", store.get_server_url(), random_string(10)); + let subject = Resource::random_subject(store); - Resource::new(generated) + Resource::new(subject) } /// Create a new instance of some Class. diff --git a/lib/src/urls.rs b/lib/src/urls.rs index 03588c291..7ba27457f 100644 --- a/lib/src/urls.rs +++ b/lib/src/urls.rs @@ -22,6 +22,7 @@ pub const BOOKMARK: &str = "https://atomicdata.dev/class/Bookmark"; pub const ONTOLOGY: &str = "https://atomicdata.dev/class/ontology"; pub const ENDPOINT_RESPONSE: &str = "https://atomicdata.dev/ontology/server/class/endpoint-response"; +pub const TABLE: &str = "https://atomicdata.dev/classes/Table"; // Properties pub const SHORTNAME: &str = "https://atomicdata.dev/properties/shortname"; diff --git a/lib/src/values.rs b/lib/src/values.rs index 0beb32704..b6c75ace9 100644 --- a/lib/src/values.rs +++ b/lib/src/values.rs @@ -363,11 +363,12 @@ impl fmt::Display for Value { Value::Float(float) => write!(f, "{}", float), Value::Markdown(i) => write!(f, "{}", i), Value::ResourceArray(v) => { - let mut s: String = String::new(); - for i in v { - s.push_str(&i.to_string()); - } - write!(f, "{}", s) + let out = v + .iter() + .map(|i| i.to_string()) + .collect::>() + .join(","); + write!(f, "{}", out) } Value::Slug(s) => write!(f, "{}", s), Value::String(s) => write!(f, "{}", s), diff --git a/server/src/handlers/download.rs b/server/src/handlers/download.rs index 272444b90..71d2a2ca3 100644 --- a/server/src/handlers/download.rs +++ b/server/src/handlers/download.rs @@ -28,9 +28,6 @@ pub async fn handle_download( let server_url = &appstate.config.server_url; let store = &appstate.store; - if let Some(quality) = ¶ms.q { - println!("Quality: {}", quality); - } // We replace `/download` with `/` to get the subject of the Resource. let subject = if let Some(pth) = path { let subject = format!("{}/{}", server_url, pth); diff --git a/server/src/handlers/export.rs b/server/src/handlers/export.rs new file mode 100644 index 000000000..aca533609 --- /dev/null +++ b/server/src/handlers/export.rs @@ -0,0 +1,309 @@ +use std::collections::HashMap; + +use crate::{appstate::AppState, errors::AtomicServerResult, helpers::get_client_agent}; +use actix_web::http::header::{ContentDisposition, DispositionParam, DispositionType}; +use actix_web::{web, HttpResponse}; +use atomic_lib::agents::ForAgent; +use atomic_lib::errors::AtomicResult; +use atomic_lib::storelike::Query; +use atomic_lib::values::SubResource; +use atomic_lib::{urls, Db, Resource, Storelike, Value}; +use chrono::{DateTime, NaiveDateTime}; +use serde::Deserialize; + +#[serde_with::serde_as] +#[serde_with::skip_serializing_none] +#[derive(Deserialize, Debug)] +pub struct ExportParams { + pub format: Option, + pub subject: Option, + pub display_refs_as_name: Option, +} + +/// Exports a resource in the specified format. +#[tracing::instrument(skip(appstate, req))] +pub async fn handle_export( + path: Option>, + appstate: web::Data, + params: web::Query, + req: actix_web::HttpRequest, +) -> AtomicServerResult { + let headers = req.headers(); + let store = &appstate.store; + + let Some(subject) = params.subject.clone() else { + return Err("No subject provided".into()); + }; + + let Some(format) = params.format.clone() else { + return Err("No format provided".into()); + }; + + let for_agent = get_client_agent(headers, &appstate, subject.clone())?; + let display_refs_as_name = params.display_refs_as_name.unwrap_or(false); + + match format.as_str() { + "csv" => { + let exporter = CSVExporter { + store, + agent: &for_agent, + display_refs_as_name, + }; + + let (name, csv) = exporter.resource_to_csv(&subject)?; + Ok(HttpResponse::Ok() + .content_type("text/csv") + .insert_header(( + actix_web::http::header::CONTENT_DISPOSITION, + ContentDisposition { + disposition: DispositionType::Attachment, + parameters: vec![DispositionParam::Filename(name)], + }, + )) + .body(csv)) + } + _ => Err(format!("Unsupported format: {}", format).into()), + } +} + +struct CSVExporter<'a> { + store: &'a Db, + agent: &'a ForAgent, + display_refs_as_name: bool, +} + +impl<'a> CSVExporter<'a> { + pub fn resource_to_csv(&self, subject: &str) -> AtomicResult<(String, String)> { + println!("Exporting resource to CSV: {}", subject); + let resource = self + .store + .get_resource_extended(subject, false, self.agent)?; + + let binding = resource.get_classes(self.store)?; + + let classes: Vec<&str> = binding.iter().map(|c| c.subject.as_str()).collect(); + + // Check the classes of the resource to determine how to export it. + if classes.contains(&urls::TABLE) { + let prop_order = self.get_prop_order_from_table(&resource)?; + + let data = self.build_csv_from_children(&resource, Some(prop_order))?; + let Ok(Value::String(name)) = resource.get(urls::NAME) else { + return Err("Resource does not have a name".into()); + }; + + let filename = format!( + "{}.csv", + sanitize_filename::sanitize(name).replace(' ', "-") + ); + Ok((filename, data)) + } else { + Err("Resource does not have any supported classes".into()) + } + } + + fn get_prop_order_from_table(&self, resource: &Resource) -> AtomicResult> { + let class_value = resource.get(urls::CLASSTYPE_PROP)?; + + let propvals = match class_value { + Value::AtomicUrl(subject) => self + .store + .get_resource_extended(subject, false, self.agent)? + .get_propvals() + .clone(), + Value::Resource(resource) => resource.get_propvals().clone(), + Value::NestedResource(nested) => match nested { + SubResource::Resource(resource) => resource.get_propvals().clone(), + SubResource::Subject(subject) => self + .store + .get_resource_extended(subject, false, self.agent)? + .get_propvals() + .clone(), + SubResource::Nested(props) => props.clone(), + }, + _ => return Err("Resource does not have any supported classtype".into()), + }; + + let mut requires = Value::ResourceArray(vec![]); + if let Some(req) = propvals.get(urls::REQUIRES) { + requires = req.clone(); + } + + let mut recommends = Value::ResourceArray(vec![]); + if let Some(rec) = propvals.get(urls::RECOMMENDS) { + recommends = rec.clone(); + } + + match (requires, recommends) { + (Value::ResourceArray(requires), Value::ResourceArray(recommends)) => { + let mut order = vec![]; + for value in requires.iter().chain(recommends.iter()) { + match value { + SubResource::Resource(resource) => { + order.push(resource.get_subject().clone()); + } + SubResource::Subject(subject) => { + order.push(subject.clone()); + } + SubResource::Nested(_) => {} + } + } + + Ok(order) + } + _ => Err("Requires and Recommends must be arrays".into()), + } + } + + fn build_csv_from_children( + &self, + resource: &Resource, + prop_order: Option>, + ) -> AtomicResult { + let query = Query { + property: Some(urls::PARENT.into()), + value: Some(atomic_lib::Value::String(resource.get_subject().clone())), + limit: None, + start_val: None, + end_val: None, + offset: 0, + sort_by: Some(urls::CREATED_AT.into()), + sort_desc: false, + include_external: false, + include_nested: true, + for_agent: self.agent.clone(), + }; + + let results = self.store.query(&query)?; + let mut body_csv = String::new(); + let mut encountered_properties = prop_order.unwrap_or_default(); + + for item in results.resources.iter() { + let mut line_vec: Vec = vec![String::new(); encountered_properties.len()]; + line_vec.insert(0, item.get_subject().to_string()); + + for (prop, value) in item.get_propvals().iter() { + if prop == urls::PARENT || prop == urls::LAST_COMMIT { + continue; + } + + let fixed_value = CSVExporter::escape_csv_value(self.value_to_string(value)); + + if let Some(index) = encountered_properties.iter().position(|p| p == prop) { + line_vec[index + 1] = fixed_value; + } else { + encountered_properties.push(prop.clone()); + line_vec.push(fixed_value); + } + } + + let line = line_vec.join(","); + body_csv.push_str(&format!("\n{}", line)); + } + + let header = self.create_csv_header_from_props(&encountered_properties)?; + let csv = format!("{}{}", header, body_csv); + + Ok(csv) + } + + fn create_csv_header_from_props(&self, props: &[String]) -> AtomicResult { + let mut header = "subject".to_string(); + for prop in props.iter() { + let name: String = + if let Ok(resource) = self.store.get_resource_extended(prop, true, self.agent) { + resource.get(urls::SHORTNAME)?.to_string() + } else { + prop.to_string() + }; + header.push_str(&format!(",{}", name)); + } + + Ok(header) + } + + fn value_to_string(&self, value: &Value) -> String { + match value { + Value::Timestamp(ts) => { + // Convert the timestamp to a NaiveDateTime (no timezone) + let seconds = ts / 1000; + let remaining_nanoseconds = (ts % 1000) * 1_000_000; // Convert remaining milliseconds to nanoseconds + + let Some(naive_datetime) = + NaiveDateTime::from_timestamp_opt(seconds, remaining_nanoseconds as u32) + else { + return ts.to_string(); + }; + + // Convert NaiveDateTime to a DateTime + let datetime_utc: DateTime = + DateTime::::from_utc(naive_datetime, chrono::Utc); + + // Format the DateTime as a string in RFC3339 format (e.g., "2023-03-20T12:34:56Z") + datetime_utc.to_rfc3339() + } + Value::ResourceArray(values) => { + let names: Vec = values + .iter() + .map(|v| match v { + SubResource::Subject(subject) => self.get_name_from_subject(subject), + SubResource::Resource(resource) => self.get_name_from_propvals( + resource.get_propvals(), + resource.get_subject().clone(), + ), + SubResource::Nested(nested) => { + self.get_name_from_propvals(nested, "".to_string()) + } + }) + .collect(); + + names.join(", ") + } + Value::Resource(resource) => { + self.get_name_from_propvals(resource.get_propvals(), resource.get_subject().clone()) + } + Value::AtomicUrl(subject) => self.get_name_from_subject(subject), + _ => value.to_string(), + } + } + + fn get_name_from_subject(&self, subject: &str) -> String { + let Ok(resource) = self.store.get_resource_extended(subject, true, self.agent) else { + return subject.to_string(); + }; + + self.get_name_from_propvals(resource.get_propvals(), resource.get_subject().clone()) + } + + fn get_name_from_propvals(&self, propvals: &HashMap, subject: String) -> String { + if !self.display_refs_as_name { + return subject; + } + + if let Some(value) = propvals.get(urls::DOWNLOAD_URL) { + return value.to_string(); + } + if let Some(value) = propvals.get(urls::NAME) { + return value.to_string(); + } + if let Some(value) = propvals.get(urls::SHORTNAME) { + return value.to_string(); + } + if let Some(value) = propvals.get(urls::FILENAME) { + return value.to_string(); + } + + subject + } + + fn escape_csv_value(value: String) -> String { + let no_quotes = value.replace('"', "\"\""); + let reg = regex::Regex::new(r"\n|,").unwrap(); + + if reg.is_match(&no_quotes) { + format!("\"{}\"", no_quotes) + } else { + no_quotes + } + } +} diff --git a/server/src/handlers/mod.rs b/server/src/handlers/mod.rs index 5bbf7b465..69dcddec5 100644 --- a/server/src/handlers/mod.rs +++ b/server/src/handlers/mod.rs @@ -7,6 +7,7 @@ However, some features reside in atomic-server. pub mod commit; pub mod download; +pub mod export; pub mod get_resource; pub mod post_resource; pub mod search; diff --git a/server/src/routes.rs b/server/src/routes.rs index 2c43b03ea..2ef2f3feb 100644 --- a/server/src/routes.rs +++ b/server/src/routes.rs @@ -20,6 +20,7 @@ include!(concat!(env!("OUT_DIR"), "/generated.rs")); pub fn config_routes(app: &mut actix_web::web::ServiceConfig) { app.service(web::resource("/ws").to(handlers::web_sockets::web_socket_handler)) .service(web::resource("/download/{path:[^{}]+}").to(handlers::download::handle_download)) + .service(web::resource("/export").to(handlers::export::handle_export)) // This `generate` imports the static files from the `app_assets` folder .service( ResourceFiles::new("/", generate())