chore(loader-utils): split Worker/WorkerWithEncoder types

modules/3d-tiles/src/tile-3d-writer.ts

@@ -1,11 +1,11 @@
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import {VERSION} from './lib/utils/version';
 import encode3DTile from './lib/encoders/encode-3d-tile';
 
 /**
  * Exporter for 3D Tiles
  */
-export const Tile3DWriter: Writer<unknown, never, WriterOptions> = {
+export const Tile3DWriter: WriterWithEncoder<unknown, never, WriterOptions> = {
   name: '3D Tile',
   id: '3d-tiles',
   module: '3d-tiles',

modules/arrow/src/arrow-writer.ts

@@ -1,6 +1,6 @@
 // import type {} from '@loaders.gl/loader-utils';
 
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import {ColumnarTable} from './lib/encode-arrow';
 import {encodeArrowSync} from './lib/encode-arrow';
 
@@ -13,7 +13,7 @@ type ArrowWriterOptions = WriterOptions & {
 };
 
 /** Apache Arrow writer */
-export const ArrowWriter: Writer<ColumnarTable, never, ArrowWriterOptions> = {
+export const ArrowWriter: WriterWithEncoder<ColumnarTable, never, ArrowWriterOptions> = {
   name: 'Apache Arrow',
   id: 'arrow',
   module: 'arrow',

modules/bson/src/bson-writer.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {EncodeBSONOptions} from './lib/encoders/encode-bson';
 import {encodeBSONSync} from './lib/encoders/encode-bson';
 
@@ -13,7 +13,7 @@ export type BSONWriterOptions = WriterOptions & {
   bson?: EncodeBSONOptions
 }
 
-export const BSONWriter: Writer<Record<string, unknown>, never, BSONWriterOptions> = {
+export const BSONWriter: WriterWithEncoder<Record<string, unknown>, never, BSONWriterOptions> = {
   name: 'BSON',
   id: 'bson',
   module: 'bson',

modules/core/src/lib/api/encode-table.ts

@@ -3,10 +3,14 @@
 // Copyright 2022 Foursquare Labs, Inc
 
 /* global TextEncoder, TextDecoder */
-import {concatenateArrayBuffers, Writer, WriterOptionsType} from '@loaders.gl/loader-utils';
+import {
+  concatenateArrayBuffers,
+  WriterOptionsType,
+  WriterWithEncoder
+} from '@loaders.gl/loader-utils';
 import {Table} from '@loaders.gl/schema';
 
-export async function encodeTable<WriterT extends Writer = Writer>(
+export async function encodeTable<WriterT extends WriterWithEncoder = WriterWithEncoder>(
   data: Table,
   writer: WriterT,
   options?: WriterOptionsType<WriterT>
@@ -36,7 +40,7 @@ export async function encodeTable<WriterT extends Writer = Writer>(
   throw new Error('Writer could not encode data');
 }
 
-export async function encodeTableAsText<WriterT extends Writer = Writer>(
+export async function encodeTableAsText<WriterT extends WriterWithEncoder = WriterWithEncoder>(
   data: Table,
   writer: WriterT,
   options?: WriterOptionsType<WriterT>
@@ -52,7 +56,7 @@ export async function encodeTableAsText<WriterT extends Writer = Writer>(
   throw new Error(`Writer ${writer.name} could not encode data as text`);
 }
 
-export function encodeTableInBatches<WriterT extends Writer = Writer>(
+export function encodeTableInBatches<WriterT extends WriterWithEncoder = WriterWithEncoder>(
   data: Table,
   writer: WriterT,
   options?: WriterOptionsType<WriterT>

modules/core/src/lib/api/encode.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import {Writer, WriterOptions, canEncodeWithWorker} from '@loaders.gl/loader-utils';
+import {WriterOptions, WriterWithEncoder, canEncodeWithWorker} from '@loaders.gl/loader-utils';
 import {concatenateArrayBuffers, resolvePath, NodeFile} from '@loaders.gl/loader-utils';
 import {processOnWorker} from '@loaders.gl/worker-utils';
 import {isBrowser} from '@loaders.gl/loader-utils';
@@ -13,7 +13,7 @@ import {getLoaderOptions} from './loader-options';
  */
 export async function encode(
   data: unknown,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): Promise<ArrayBuffer> {
   const globalOptions = getLoaderOptions() as WriterOptions;
@@ -75,7 +75,11 @@ export async function encode(
 /**
  * Encode loaded data into a binary ArrayBuffer using the specified Writer.
  */
-export function encodeSync(data: unknown, writer: Writer, options?: WriterOptions): ArrayBuffer {
+export function encodeSync(
+  data: unknown,
+  writer: WriterWithEncoder,
+  options?: WriterOptions
+): ArrayBuffer {
   if (writer.encodeSync) {
     return writer.encodeSync(data, options);
   }
@@ -90,7 +94,7 @@ export function encodeSync(data: unknown, writer: Writer, options?: WriterOption
  */
 export async function encodeText(
   data: unknown,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): Promise<string> {
   if (writer.text && writer.encodeText) {
@@ -111,7 +115,11 @@ export async function encodeText(
  * It is not optimized for performance. Data maybe converted from text to binary and back.
  * @throws if the writer does not generate text output
  */
-export function encodeTextSync(data: unknown, writer: Writer, options?: WriterOptions): string {
+export function encodeTextSync(
+  data: unknown,
+  writer: WriterWithEncoder,
+  options?: WriterOptions
+): string {
   if (writer.text && writer.encodeTextSync) {
     return writer.encodeTextSync(data, options);
   }
@@ -129,7 +137,7 @@ export function encodeTextSync(data: unknown, writer: Writer, options?: WriterOp
  */
 export function encodeInBatches(
   data: unknown,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): AsyncIterable<ArrayBuffer> {
   if (writer.encodeInBatches) {
@@ -148,7 +156,7 @@ export function encodeInBatches(
 export async function encodeURLtoURL(
   inputUrl: string,
   outputUrl: string,
-  writer: Writer,
+  writer: WriterWithEncoder,
   options?: WriterOptions
 ): Promise<string> {
   inputUrl = resolvePath(inputUrl);

modules/csv/src/csv-writer.ts

@@ -2,7 +2,7 @@
 // Copyright (c) vis.gl contributors
 
 /* global TextEncoder */
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {Table, TableBatch} from '@loaders.gl/schema';
 import {encodeTableAsCSV} from './lib/encoders/encode-csv';
 
@@ -14,7 +14,7 @@ export type CSVWriterOptions = WriterOptions & {
   useDisplayNames?: never;
 };
 
-export const CSVWriter: Writer<Table, TableBatch, CSVWriterOptions> = {
+export const CSVWriter: WriterWithEncoder<Table, TableBatch, CSVWriterOptions> = {
   id: 'csv',
   version: 'latest',
   module: 'csv',

modules/draco/src/draco-writer.ts

@@ -1,4 +1,4 @@
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {DracoMesh} from './lib/draco-types';
 import type {DracoBuildOptions} from './lib/draco-builder';
 import DRACOBuilder from './lib/draco-builder';
@@ -29,7 +29,7 @@ const DEFAULT_DRACO_WRITER_OPTIONS = {
 /**
  * Exporter for Draco3D compressed geometries
  */
-export const DracoWriter: Writer<DracoMesh, unknown, DracoWriterOptions> = {
+export const DracoWriter: WriterWithEncoder<DracoMesh, unknown, DracoWriterOptions> = {
   name: 'DRACO',
   id: 'draco',
   module: 'draco',

modules/gltf/src/glb-writer.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {GLB} from './lib/types/glb-types';
 import type {GLBEncodeOptions} from './lib/encoders/encode-glb';
 import {encodeGLBSync} from './lib/encoders/encode-glb';
@@ -15,7 +15,7 @@ export type GLBWriterOptions = WriterOptions & {
  * GLB exporter
  * GLB is the binary container format for GLTF
  */
-export const GLBWriter: Writer<GLB, never, GLBWriterOptions> = {
+export const GLBWriter: WriterWithEncoder<GLB, never, GLBWriterOptions> = {
   name: 'GLB',
   id: 'glb',
   module: 'gltf',
@@ -45,6 +45,3 @@ function encodeSync(glb, options) {
 
   return arrayBuffer;
 }
-
-// TYPE TESTS - TODO find a better way than exporting junk
-export const _TypecheckGLBLoader: Writer = GLBWriter;

modules/images/src/image-writer.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {ImageDataType} from './types';
 import {VERSION} from './lib/utils/version';
 import {encodeImage} from './lib/encoders/encode-image';
@@ -14,7 +14,7 @@ export type ImageWriterOptions = WriterOptions & {
 };
 
 /** Writer for image data */
-export const ImageWriter: Writer<ImageDataType, never, ImageWriterOptions> = {
+export const ImageWriter: WriterWithEncoder<ImageDataType, never, ImageWriterOptions> = {
   name: 'Images',
   id: 'image',
   module: 'images',

modules/json/src/geojson-writer.ts

@@ -2,7 +2,7 @@
 // Copyright (c) vis.gl contributors
 // Copyright Foursquare, Inc 20222
 
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {Table, TableBatch} from '@loaders.gl/schema';
 import {encodeTableAsGeojsonInBatches} from './lib/encoders/geojson-encoder';
 
@@ -14,7 +14,7 @@ export type GeoJSONWriterOptions = WriterOptions & {
   chunkSize?: number;
 };
 
-export const GeoJSONWriter: Writer<Table, TableBatch, GeoJSONWriterOptions> = {
+export const GeoJSONWriter: WriterWithEncoder<Table, TableBatch, GeoJSONWriterOptions> = {
   id: 'geojson',
   version: 'latest',
   module: 'geojson',

modules/json/src/json-writer.ts

@@ -3,7 +3,7 @@
 // Copyright 2022 Foursquare Labs, Inc.
 
 /* global TextEncoder */
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import type {Table, TableBatch} from '@loaders.gl/schema';
 import {encodeTableAsJSON} from './lib/encoders/json-encoder';
 
@@ -18,7 +18,7 @@ type RowArray = unknown[];
 type RowObject = {[key: string]: unknown};
 type TableJSON = RowArray[] | RowObject[];
 
-export const JSONWriter: Writer<Table, TableBatch, JSONWriterOptions> = {
+export const JSONWriter: WriterWithEncoder<Table, TableBatch, JSONWriterOptions> = {
   id: 'json',
   version: 'latest',
   module: 'json',

modules/loader-utils/src/index.ts

@@ -35,7 +35,7 @@ export {parseFromContext, parseSyncFromContext, parseInBatchesFromContext} from
 
 // writers
 
-export type {Writer, WriterOptions, WriterOptionsType} from './writer-types';
+export type {Writer, WriterWithEncoder, WriterOptions, WriterOptionsType} from './writer-types';
 
 // GENERAL UTILS
 export {assert} from './lib/env-utils/assert';

modules/loader-utils/src/writer-types.ts

@@ -18,22 +18,47 @@ export type WriterOptions = {
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 export type Writer<DataT = unknown, BatchT = unknown, WriterOptionsT = WriterOptions> = {
-  name: string;
+  /** The result type of this loader  */
+  dataType?: DataT;
+  /** The batched result type of this loader  */
+  batchType?: BatchT;
 
+  /** Human readable name */
+  name: string;
+  /** id should be the same as the field used in LoaderOptions */
   id: string;
+  /** module is used to generate worker threads, need to be the module directory name */
   module: string;
+  /** Version should be injected by build tools */
   version: string;
+  /** A boolean, or a URL */
   worker?: string | boolean;
+  // end Worker
 
-  // TODO - are these are needed?
-  extensions?: string[];
+  /** Which category does this loader belong to */
+  category?: string;
+  /** File extensions that are potential matches with this loader. */
+  extensions: string[];
+  /** MIMETypes that indicate a match with this loader. @note Some MIMETypes are generic and supported by many loaders */
   mimeTypes?: string[];
+
+  /** Is the input of this loader binary */
   binary?: boolean;
+  /** Is the input of this loader text */
   text?: boolean;
 
   options: WriterOptionsT;
   deprecatedOptions?: Record<string, string>;
+};
 
+/**
+ * A writer definition that can be used with `@loaders.gl/core` functions
+ */
+export type WriterWithEncoder<
+  DataT = unknown,
+  BatchT = unknown,
+  WriterOptionsT = WriterOptions
+> = Writer<DataT, BatchT, WriterOptionsT> & {
   encode?(data: DataT, options?: WriterOptionsT): Promise<ArrayBuffer>;
   encodeSync?(data: DataT, options?: WriterOptionsT): ArrayBuffer;
 

modules/parquet/src/parquet-wasm-writer.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Writer} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder} from '@loaders.gl/loader-utils';
 import {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';
 import type * as arrow from 'apache-arrow';
 
@@ -10,18 +10,18 @@ import type * as arrow from 'apache-arrow';
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 /** Parquet WASM writer */
-export const ParquetWasmWriter: Writer<arrow.Table, never, ParquetWriterOptions> = {
+export const ParquetWasmWriter: WriterWithEncoder<arrow.Table, never, ParquetWriterOptions> = {
   name: 'Apache Parquet',
   id: 'parquet-wasm',
   module: 'parquet',
   version: VERSION,
   extensions: ['parquet'],
   mimeTypes: ['application/octet-stream'],
-  encode,
   binary: true,
   options: {
     parquet: {
       wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
     }
-  }
+  },
+  encode
 };

modules/parquet/src/parquet-writer.ts

@@ -1,7 +1,7 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Writer} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder} from '@loaders.gl/loader-utils';
 import {Table, TableBatch} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
@@ -10,7 +10,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 export type ParquetWriterOptions = {};
 
-export const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions> = {
+export const ParquetWriter: WriterWithEncoder<Table, TableBatch, ParquetWriterOptions> = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',

modules/textures/src/compressed-texture-writer.ts

@@ -1,4 +1,4 @@
-import type {Writer, WriterOptions} from '@loaders.gl/loader-utils';
+import type {WriterWithEncoder, WriterOptions} from '@loaders.gl/loader-utils';
 import {VERSION} from './lib/utils/version';
 import {encodeImageURLToCompressedTextureURL} from './lib/encoders/encode-texture';
 
@@ -17,7 +17,11 @@ export type CompressedTextureWriterOptions = WriterOptions & {
 /**
  * DDS Texture Container Exporter
  */
-export const CompressedTextureWriter: Writer<unknown, unknown, CompressedTextureWriterOptions> = {
+export const CompressedTextureWriter: WriterWithEncoder<
+  unknown,
+  unknown,
+  CompressedTextureWriterOptions
+> = {
   name: 'DDS Texture Container',
   id: 'dds',
   module: 'textures',