Skip to content

Commit

Permalink
* Renamed old "JSON (debate-map)" import format to be "json (debate-m…
Browse files Browse the repository at this point in the history
…ap, old)", to match what it actually was. (for importing an old firestore backup)

* Added new "JSON (debate-map)" import format, which is the same format as the "JSON (debate-map)" export format.
* MS you can set an "access-policy override" in the subtree-importer dialog. (fixing errors when importing from one DM instance into another)
* Fixed that subtree-ops dialog, for the export operation, did not respect the selected export fields. (would just use default set)
  • Loading branch information
Venryx committed Jul 24, 2024
1 parent b060a8c commit 5512784
Show file tree
Hide file tree
Showing 22 changed files with 205 additions and 62 deletions.
1 change: 1 addition & 0 deletions Packages/client/Source/Store/main/maps.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ export class ImportSubtreeDialogState {
@O showAutoInsertTools = true;
@O autoInsert_interval = 1000;
//@O hideFoundEntries = false;
@O accessPolicyOverride: string|n;

@O @ignore selectedImportResources = new Set<ImportResource>();
@O @ignore selectFromIndex = -1;
Expand Down
4 changes: 2 additions & 2 deletions Packages/client/Source/UI/@Shared/Maps/MapDetailsUI.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ export class MapDetailsUI extends DetailsUI_Base<Map, MapDetailsUI> {
</RowLR>*/}
<RowLR mt={5} splitAt={splitAt}>
<Pre>Access policy: </Pre>
<PolicyPicker value={newData.accessPolicy} onChange={val=>Change(newData.accessPolicy = val)}>
<PolicyPicker value={newData.accessPolicy} onChange={val=>Change(newData.accessPolicy = val!)}>
{text=><Button enabled={enabled} text={text} style={{width: "100%"}}/>}
</PolicyPicker>
</RowLR>
Expand Down Expand Up @@ -150,7 +150,7 @@ export class MapDetailsUI extends DetailsUI_Base<Map, MapDetailsUI> {
</RowLR> */}
<RowLR mt={5} splitAt={splitAt}>
<TextPlus info="Note that this only applies for new nodes created in this map. (ie. if you change this setting, you must manually update the access-policies of existing nodes)">Node access policy:</TextPlus>
<PolicyPicker value={newData.nodeAccessPolicy} onChange={val=>Change(newData.nodeAccessPolicy = val)}>
<PolicyPicker value={newData.nodeAccessPolicy} onChange={val=>Change(newData.nodeAccessPolicy = val!)}>
<Button enabled={enabled} text={nodeAccessPolicy ? `${nodeAccessPolicy.name} (id: ${nodeAccessPolicy.id})` : "(click to select policy)"} style={{width: "100%"}}/>
</PolicyPicker>
</RowLR>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ class StyleRuleUI extends BaseComponent<StyleRuleUI_Props, {}> {
<Column ml={30}>
{rule.if_accessPolicyDoesNotMatch.policyIDs.map((policyID, index)=>{
return <Row mt={5} key={index}>
<PolicyPicker value={policyID} onChange={val=>ChangeIfBlock(()=>rule.if_accessPolicyDoesNotMatch.policyIDs[index] = val)}>
<PolicyPicker value={policyID} onChange={val=>ChangeIfBlock(()=>rule.if_accessPolicyDoesNotMatch.policyIDs[index] = val!)}>
{text=><Button text={text} style={{width: "100%"}}/>}
</PolicyPicker>
<Button ml={5} text="X" onClick={()=>rule.if_accessPolicyDoesNotMatch.policyIDs.RemoveAt(index)}/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ export class OthersPanel extends BaseComponentPlus({} as {show: boolean, map?: M
accessPolicyButton={
<PolicyPicker containerStyle={{flex: "none"}} value={node.accessPolicy} onChange={async val=>{
//new UpdateNodeAccessPolicy({nodeID: node.id, accessPolicy: val}).RunOnServer();
await RunCommand_UpdateNode({id: node.id, updates: {accessPolicy: val}});
await RunCommand_UpdateNode({id: node.id, updates: {accessPolicy: val!}});
}}>
{/*<Button ml={5} enabled={creatorOrMod} text={accessPolicy ? `${accessPolicy.name} (id: ${accessPolicy.id})` : "(click to select policy)"} style={{width: "100%"}}/>*/}
<Button ml={5} p="3px 7px" enabled={creatorOrMod} text="Change" style={{width: "100%"}}/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ export class RatingsPanel extends BaseComponentPlus({} as RatingsPanel_Props, {}
}}/>
<Text ml={10} mr={5}>Access-policy:</Text>
<PolicyPicker containerStyle={{flex: null}} value={myRating_raw.accessPolicy} onChange={policyID=>{
SetRating(myRating_displayVal, policyID);
SetRating(myRating_displayVal, policyID!);
}}>
<PolicyPicker_Button policyID={myRating_raw.accessPolicy} idTrimLength={3} style={{padding: "3px 10px"}}/>
</PolicyPicker>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ export class RatingsPanel_Old extends BaseComponentPlus({} as RatingsPanel_Props
</RowLR>
<RowLR mt={5} splitAt={splitAt}>
<Pre>Access policy: </Pre>
<PolicyPicker value={newRating_accessPolicyID} onChange={val=>Change(newRating_accessPolicyID = val)}>
<PolicyPicker value={newRating_accessPolicyID} onChange={val=>Change(newRating_accessPolicyID = val!)}>
<PolicyPicker_Button policyID={newRating_accessPolicy?.id} style={{width: "100%"}}/>
</PolicyPicker>
</RowLR>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ export class PermissionsPanel extends BaseComponent<Pick<NodeDetailsUI_SharedPro
<>
<RowLR mt={5} splitAt={splitAt}>
<Pre>Access policy: </Pre>
<PolicyPicker value={newData.accessPolicy} onChange={val=>Change(newData.accessPolicy = val)}>
<PolicyPicker value={newData.accessPolicy} onChange={val=>Change(newData.accessPolicy = val!)}>
<Button enabled={enabled} text={accessPolicy ? `${accessPolicy.name} (id: ${accessPolicy.id})` : "(click to select policy)"} style={{width: "100%"}}/>
</PolicyPicker>
</RowLR>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,8 @@ import {gql} from "web-vcore/nm/@apollo/client";
import {Assert, NN} from "web-vcore/nm/js-vextensions.js";
import {ClassKeys, CreateAccessor} from "web-vcore/nm/mobx-graphlink.js";
import {SubtreeIncludeKeys} from "./SubtreeOpsStructs.js";
import {DMSubtreeData} from "../../../../../../Utils/DataFormats/JSON/DM/DMSubtreeData.js";

export class SubtreeData_Server {
constructor(data?: Partial<SubtreeData_Server>) {
Object.assign(this, data);
}
nodes?: NodeL1[];
nodeLinks?: NodeLink[];
nodeRevisions?: NodeRevision[];
nodePhrasings?: NodePhrasing[];
terms?: Term[];
medias?: Media[];
nodeTags?: NodeTag[];
}
export function GetServerSubtreeData_GQLQuery(rootNodeID: string, maxExportDepth: number, includeKeys: SubtreeIncludeKeys) {
const Fields = (fields: string[], fieldSubfieldsStrings: {[key: string]: string} = {})=>{
if (fields.length == 0) return "__typename";
Expand Down Expand Up @@ -51,8 +40,8 @@ export function GetServerSubtreeData_GQLQuery(rootNodeID: string, maxExportDepth
medias: data.medias.ToMap(a=>a.id, a=>a),
});
});*/
export const ConvertLocalSubtreeDataToServerStructure = CreateAccessor((searchInfo: SubtreeData_Local): SubtreeData_Server=>{
return new SubtreeData_Server({
export const ConvertLocalSubtreeDataToServerStructure = CreateAccessor((searchInfo: SubtreeData_Local): DMSubtreeData=>{
return new DMSubtreeData({
nodes: Array.from(searchInfo.nodes.values()),
nodeLinks: Array.from(searchInfo.nodeLinks.values()),
nodeRevisions: Array.from(searchInfo.nodeRevisions.values()),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import {Media, NodeL1, NodeLink, NodePhrasing, NodeRevision, Term} from "dm_common";
import {makeObservable} from "mobx";
import {O} from "web-vcore";
import {ClassKeys} from "web-vcore/nm/mobx-graphlink.js";

export enum SubtreeOperation {
Expand All @@ -10,12 +12,13 @@ export enum SubtreeOperation {
export class SubtreeIncludeKeys {
constructor(data?: Partial<SubtreeIncludeKeys>) {
Object.assign(this, data);
makeObservable(this);
}
//nodes = ClassKeys<NodeL3>("id", "type", "rootNodeForMap", "c_currentRevision", "multiPremiseArgument", "argumentType");
nodes = ClassKeys<NodeL1>("id", "type", "rootNodeForMap", "c_currentRevision", "multiPremiseArgument", "argumentType");
nodeLinks = ClassKeys<NodeLink>("id", "parent", "child", "form", "polarity");
nodeRevisions = ClassKeys<NodeRevision>("id", "node", "phrasing", "attachments");
nodePhrasings = ClassKeys<NodePhrasing>("id", "node", "type", "text_base", "text_negation", "text_question", "text_narrative", "note", "terms", "references");
terms = ClassKeys<Term>("id", "name", "forms", "disambiguation", "type", "definition", "note");
medias = ClassKeys<Media>("id", "name", "type", "url", "description");
//@O nodes = ClassKeys<NodeL3>("id", "type", "rootNodeForMap", "c_currentRevision", "multiPremiseArgument", "argumentType");
@O nodes = ClassKeys<NodeL1>("id", "type", "rootNodeForMap", "c_currentRevision", "multiPremiseArgument", "argumentType");
@O nodeLinks = ClassKeys<NodeLink>("id", "parent", "child", "form", "polarity");
@O nodeRevisions = ClassKeys<NodeRevision>("id", "node", "phrasing", "attachments");
@O nodePhrasings = ClassKeys<NodePhrasing>("id", "node", "type", "text_base", "text_negation", "text_question", "text_narrative", "note", "terms", "references");
@O terms = ClassKeys<Term>("id", "name", "forms", "disambiguation", "type", "definition", "note");
@O medias = ClassKeys<Media>("id", "name", "type", "url", "description");
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,13 @@ export class SubtreeOpsUI_Export_Left extends BaseComponentPlus(
{} as {} & MI_SharedProps,
{
tab: ExportSubtreeUI_MidTab.Nodes,
includeKeys: new SubtreeIncludeKeys(),
},
) {
render() {
let {} = this.props;
const {tab, includeKeys} = this.state;
const {tab} = this.state;
const dialogState = store.main.maps.subtreeOperationsDialog;

let includeKeys_final = Clone(includeKeys);
// for the export formats below, we only need a specific subset of the data
if (dialogState.targetFormat == DataExchangeFormat.csv_basic) includeKeys_final = csv_basic_includeKeys;
else if (dialogState.targetFormat == DataExchangeFormat.csv_quotes) includeKeys_final = csv_quotes_includeKeys;

const includeKeys = dialogState.export_includeKeys;
return (
<>
<RowLR mt={5} splitAt={splitAt}>
Expand All @@ -64,7 +58,9 @@ export class SubtreeOpsUI_Export_Left extends BaseComponentPlus(
{fieldNames.map(fieldName=>{
return (
<CheckBox key={fieldName} text={fieldName} value={includeKeys[tableName].includes(fieldName)} onChange={val=>{
includeKeys[tableName] = fieldNames.filter(a=>(a == fieldName ? val : fieldNames_oldEnabled.includes(a)));
RunInAction_Set(this, ()=>{
includeKeys[tableName] = fieldNames.filter(a=>(a == fieldName ? val : fieldNames_oldEnabled.includes(a)));
});
this.Update();
}}/>
);
Expand All @@ -84,7 +80,12 @@ export class SubtreeOpsUI_Export_Right extends BaseComponent<{} & MI_SharedProps
const {mapID, node: rootNode, path: rootNodePath} = this.props;
const {retrievalActive} = this.state;
const dialogState = store.main.maps.subtreeOperationsDialog;
const includeKeys = dialogState.export_includeKeys;

const includeKeys_userSet = dialogState.export_includeKeys;
let includeKeys = Clone(includeKeys_userSet);
// for the export formats below, we only need a specific subset of the data (selector ui also not shown for these formats)
if (dialogState.targetFormat == DataExchangeFormat.csv_basic) includeKeys = csv_basic_includeKeys;
else if (dialogState.targetFormat == DataExchangeFormat.csv_quotes) includeKeys = csv_quotes_includeKeys;

const {subtreeData} = useSubtreeRetrievalQueryOrAccessors(rootNode, rootNodePath, includeKeys, dialogState.retrievalMethod, dialogState.maxExportDepth, retrievalActive);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ import {CSV_SL_Row} from "Utils/DataFormats/CSV/CSV_SL/DataModel.js";
import {GetResourcesInImportSubtree_CSV_SL} from "Utils/DataFormats/CSV/CSV_SL/ImportHelpers.js";
import {DataExchangeFormat, DataExchangeFormat_entries_supportedBySubtreeImporter, ImportResource, IR_NodeAndRevision} from "Utils/DataFormats/DataExchangeFormat.js";
import {FS_NodeL3} from "Utils/DataFormats/JSON/DM_Old/FSDataModel/FS_Node.js";
import {GetResourcesInImportSubtree} from "Utils/DataFormats/JSON/DM_Old/FSImportHelpers.js";
import {GetResourcesInImportSubtree as GetResourcesInImportSubtree_JsonDmFs} from "Utils/DataFormats/JSON/DM_Old/FSImportHelpers.js";
import {apolloClient} from "Utils/LibIntegrations/Apollo.js";
import {liveSkin} from "Utils/Styles/SkinManager.js";
import {AddNotificationMessage, ES, InfoButton, O, Observer, RunInAction_Set, UseWindowEventListener} from "web-vcore";
import {AddNotificationMessage, ES, InfoButton, O, Observer, P, RunInAction_Set, UseWindowEventListener} from "web-vcore";
import {gql} from "web-vcore/nm/@apollo/client";
import {E, FromJSON, GetEntries, ModifyString, SleepAsync, Timer} from "web-vcore/nm/js-vextensions.js";
import {makeObservable} from "web-vcore/nm/mobx";
Expand All @@ -29,6 +29,9 @@ import {CG_Debate, CG_Node} from "Utils/DataFormats/JSON/ClaimGen/DataModel.js";
import {GetResourcesInImportSubtree_CG} from "Utils/DataFormats/JSON/ClaimGen/ImportHelpers.js";
import {CommandEntry, RunCommandBatch, RunCommandBatchResult} from "Utils/DB/RunCommandBatch.js";
import {MI_SharedProps} from "../NodeUI_Menu.js";
import {DMSubtreeData} from "../../../../../Utils/DataFormats/JSON/DM/DMSubtreeData.js";
import {GetResourcesInImportSubtree_JsonDm} from "../../../../../Utils/DataFormats/JSON/DM/DMImportHelpers.js";
import {PolicyPicker, PolicyPicker_Button} from "../../../../Database/Policies/PolicyPicker.js";

@Observer
export class MI_ImportSubtree extends BaseComponent<MI_SharedProps, {}, ImportResource> {
Expand Down Expand Up @@ -112,7 +115,8 @@ class ImportSubtreeUI extends BaseComponent<
// left panel
sourceText: string,
sourceText_parseError: string|n,
forJSONDM_subtreeData: FS_NodeL3|n,
forJSONDM_subtreeData: DMSubtreeData|n,
forJSONDMFS_subtreeData: FS_NodeL3|n,
forJSONCG_subtreeData: CG_Debate|n,
forCSVSL_subtreeData: CSV_SL_Row[]|n,

Expand All @@ -139,7 +143,7 @@ class ImportSubtreeUI extends BaseComponent<
render() {
const {mapID, map, node, path, controller} = this.props;
const {
sourceText, sourceText_parseError, forJSONDM_subtreeData, forJSONCG_subtreeData, forCSVSL_subtreeData, process,
sourceText, sourceText_parseError, forJSONDM_subtreeData, forJSONDMFS_subtreeData, forJSONCG_subtreeData, forCSVSL_subtreeData, process,
importSelected, selectedIRs_nodeAndRev_atImportStart,
serverImportInProgress, serverImport_commandsCompleted,
leftTab, rightTab, showLeftPanel,
Expand All @@ -154,12 +158,21 @@ class ImportSubtreeUI extends BaseComponent<
let resources: ImportResource[] = [];
if (process) {
if (uiState.sourceType == DataExchangeFormat.json_dm && forJSONDM_subtreeData != null) {
resources = GetResourcesInImportSubtree(forJSONDM_subtreeData);
resources = GetResourcesInImportSubtree_JsonDm(forJSONDM_subtreeData, node);
} else if (uiState.sourceType == DataExchangeFormat.json_dm_fs && forJSONDMFS_subtreeData != null) {
resources = GetResourcesInImportSubtree_JsonDmFs(forJSONDMFS_subtreeData);
} else if (uiState.sourceType == DataExchangeFormat.json_cg && forJSONCG_subtreeData != null) {
resources = GetResourcesInImportSubtree_CG(importContext, forJSONCG_subtreeData);
} else if (uiState.sourceType == DataExchangeFormat.csv_sl && forCSVSL_subtreeData != null) {
resources = GetResourcesInImportSubtree_CSV_SL(forCSVSL_subtreeData);
}
if (uiState.accessPolicyOverride != null) {
for (const resource of resources) {
if (resource instanceof IR_NodeAndRevision) {
resource.node.accessPolicy = uiState.accessPolicyOverride;
}
}
}
}
this.Stash({resources});

Expand Down Expand Up @@ -190,6 +203,17 @@ class ImportSubtreeUI extends BaseComponent<
<>
<Row>
{uiState.sourceType == DataExchangeFormat.json_dm &&
<Row center style={{flex: 1}}>
<Text>Subtree JSON:</Text>
<InfoButton ml={5} text={`
Obtain this subtree-json by:
1) Right-click the node you want to export, and press "Advanced -> Export subtree".
2) Choose your export options (probably enable all fields of nodes, nodeRevisions, and nodeLinks), then save to file.
3) Open the file, copy all text, and paste it into this subtree-json text area.
Note: Currently this can only import the nodes, nodeRevisions, and nodeLinks data.
`.AsMultiline(0)}/>
</Row>}
{uiState.sourceType == DataExchangeFormat.json_dm_fs &&
<Row center style={{flex: 1}}>
<Text>Subtree JSON:</Text>
<InfoButton ml={5} text={`
Expand Down Expand Up @@ -235,16 +259,27 @@ class ImportSubtreeUI extends BaseComponent<
const newState = {sourceText: newSourceText} as ExtractState<ImportSubtreeUI>;

if (uiState.sourceType == DataExchangeFormat.json_dm) {
let subtreeData_new: FS_NodeL3|n = null;
let subtreeData_new: DMSubtreeData|n = null;
try {
subtreeData_new = FromJSON(newSourceText) as FS_NodeL3;
subtreeData_new = FromJSON(newSourceText) as DMSubtreeData;
newState.forJSONDM_subtreeData = subtreeData_new;
newState.sourceText_parseError = null;
} catch (err) {
newState.forJSONDM_subtreeData = null;
newState.sourceText_parseError = err;
}
this.SetState(newState);
} else if (uiState.sourceType == DataExchangeFormat.json_dm_fs) {
let subtreeData_new: FS_NodeL3|n = null;
try {
subtreeData_new = FromJSON(newSourceText) as FS_NodeL3;
newState.forJSONDMFS_subtreeData = subtreeData_new;
newState.sourceText_parseError = null;
} catch (err) {
newState.forJSONDMFS_subtreeData = null;
newState.sourceText_parseError = err;
}
this.SetState(newState);
} else if (uiState.sourceType == DataExchangeFormat.json_cg) {
let subtreeData_new: CG_Debate|n = null;
try {
Expand Down Expand Up @@ -313,8 +348,8 @@ class ImportSubtreeUI extends BaseComponent<
serverImportInProgress && ` [${this.state.serverImport_commandsCompleted}/${resources.length}]`,
].filter(a=>a).join("")}
enabled={
// atm only the resource-extraction code for the json-cg format adds the "insertPath_parentResourceLocalID" field needed for server-side tree-importing
resources.length > 0 && uiState.sourceType == DataExchangeFormat.json_cg &&
// atm only the resource-extraction code for the json-dm and json-cg formats adds the "insertPath_parentResourceLocalID" field needed for server-side tree-importing
resources.length > 0 && uiState.sourceType.IsOneOf(DataExchangeFormat.json_dm, DataExchangeFormat.json_cg) &&
!this.nodeCreationTimer.Enabled && !serverImportInProgress
}
onClick={async()=>{
Expand Down Expand Up @@ -362,6 +397,10 @@ class ImportSubtreeUI extends BaseComponent<
{/*<Text ml={5}>Batch:</Text>
<Spinner ml={5} value={uiState.autoInsert_batchSize} onChange={val=>RunInAction_Set(this, ()=>uiState.autoInsert_batchSize = val)}/>*/}
</>}
<Text ml={10}>Policy override:</Text>
<PolicyPicker allowClear={true} textForNull="(use original policy id)" value={uiState.accessPolicyOverride} onChange={val=>RunInAction_Set(this, ()=>uiState.accessPolicyOverride = val)}>
<PolicyPicker_Button ml={5} policyID={uiState.accessPolicyOverride} idTrimLength={3} enabled={!serverImportInProgress} style={{padding: "3px 10px"}}/>
</PolicyPicker>
</Row>
<ScrollView>
<ReactList type="variable" length={resources.length}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@ import {ExportRetrievalMethod} from "../../../../../Store/main/maps.js";
import {MI_SharedProps} from "../NodeUI_Menu.js";
import {SubtreeOpsUI_Export_Left, SubtreeOpsUI_Export_Right} from "./Dialogs/SubtreeOpsUI_Export.js";
import {SubtreeOpsUI_SetAccessPolicy_Left, SubtreeOpsUI_SetAccessPolicy_Right} from "./Dialogs/SubtreeOpsUI_SetAccessPolicy.js";
import {ConvertLocalSubtreeDataToServerStructure, GetServerSubtreeData_GQLQuery, PopulateLocalSubtreeData, SubtreeData_Server} from "./Dialogs/SubtreeOpsHelpers.js";
import {ConvertLocalSubtreeDataToServerStructure, GetServerSubtreeData_GQLQuery, PopulateLocalSubtreeData} from "./Dialogs/SubtreeOpsHelpers.js";
import {SubtreeIncludeKeys, SubtreeOperation} from "./Dialogs/SubtreeOpsStructs.js";
import {SubtreeOpsUI_Delete_Left, SubtreeOpsUI_Delete_Right} from "./Dialogs/SubtreeOpsUI_Delete.js";
import {DMSubtreeData} from "../../../../../Utils/DataFormats/JSON/DM/DMSubtreeData.js";

@Observer
export class MI_SubtreeOps extends BaseComponentPlus({} as MI_SharedProps, {}) {
Expand Down Expand Up @@ -113,7 +114,7 @@ export function useSubtreeRetrievalQueryOrAccessors(rootNode: NodeL3, rootNodePa
nextFetchPolicy: "no-cache",
});

let subtreeData: SubtreeData_Server|n;
let subtreeData: DMSubtreeData|n;
if (retrievalActive) {
if (retrievalMethod == ExportRetrievalMethod.server) {
subtreeData = queryData?.subtree;
Expand Down
Loading

0 comments on commit 5512784

Please sign in to comment.