diff --git a/generate.go b/generate.go index c04452cb..105b1453 100644 --- a/generate.go +++ b/generate.go @@ -37,12 +37,14 @@ var generateOptions struct { vmOptions stdout bool + format string } func init() { initAllVMFlags(generateCmd, &generateOptions.vmOptions) generateCmd.PersistentFlags().BoolVar(&generateOptions.stdout, "stdout", false, "print values on stdout") + generateCmd.PersistentFlags().StringVar(&generateOptions.format, "format", "", "force all values to this format") jk.AddCommand(generateCmd) } @@ -51,6 +53,19 @@ func skipException(err error) bool { return strings.Contains(err.Error(), "jk-internal-skip: ") } +var errUnsupportedFormat = errors.New("--format accepts 'json' or 'yaml'") + +func setGenerateFormat(format string, vm *vm) { + switch format { + case "": + return + case "json", "yaml": + vm.parameters.SetString("jk.generate.format", format) + default: + log.Fatal(errUnsupportedFormat) + } +} + func generateArgs(cmd *cobra.Command, args []string) error { if len(args) != 1 { return errors.New("generate requires an input script") @@ -70,6 +85,7 @@ func generate(cmd *cobra.Command, args []string) { vm := newVM(&generateOptions.vmOptions, ".") vm.parameters.SetBool("jk.generate.stdout", generateOptions.stdout) + setGenerateFormat(generateOptions.format, vm) if err := vm.Run("@jkcfg/std/cmd/", fmt.Sprintf(string(std.Module("cmd/generate-module.js")), args[0])); err != nil { if !skipException(err) { diff --git a/std/cmd/generate-module.js b/std/cmd/generate-module.js index 90a6c1b1..f994a0b2 100644 --- a/std/cmd/generate-module.js +++ b/std/cmd/generate-module.js @@ -1,9 +1,10 @@ import * as param from '@jkcfg/std/param'; -import { generate } from '@jkcfg/std/cmd/generate'; +import { generate, OutputFormat, maybeSetFormat } from '@jkcfg/std/cmd/generate'; import generateDefinition from '%s'; -const inputParams = { +let inputParams = { stdout: param.Boolean('jk.generate.stdout', false), }; +maybeSetFormat(inputParams, param.String('jk.generate.format', undefined)); generate(generateDefinition, inputParams); diff --git a/std/cmd/generate.ts b/std/cmd/generate.ts index ba39235a..aaf7472f 100644 --- a/std/cmd/generate.ts +++ b/std/cmd/generate.ts @@ -1,5 +1,6 @@ import * as std from '../index'; import { WriteOptions } from '../write'; +import { splitPath, formatFromPath } from '../read'; import { ValidateFn } from './validate'; import { normaliseResult, formatError } from '../validation'; @@ -16,15 +17,43 @@ export interface File { validate?: ValidateFn; } +/* + * OutputFormat enumerates the values that a "forced format" argument + * can take. + */ +export enum OutputFormat { + JSON = "json", + YAML = "yaml", +} + +const outputFormatToFormat = { + [OutputFormat.JSON]: std.Format.JSON, + [OutputFormat.YAML]: std.Format.YAML, +}; + /* * GenerateParams types the optional arguments to generate. */ export interface GenerateParams { stdout?: boolean; + format?: OutputFormat; overwrite?: std.Overwrite; writeFile?: (v: any, p: string, o?: WriteOptions) => void; } +export function maybeSetFormat(inputParams: GenerateParams, format?: string) { + switch (format) { + case "json": + inputParams.format = OutputFormat.JSON; + break; + case "yaml": + inputParams.format = OutputFormat.YAML; + break; + default: + break; + } +} + const helpMsg = ` To use generate, export a default value with the list of files to generate: @@ -77,25 +106,6 @@ const nth = (n: number): string => { return n + (s[mod(v - 20, 10)] || s[v] || s[0]); }; -function extension(path: string): string { - return path.split('.').pop(); -} - -function formatFromPath(path: string): std.Format { - switch (extension(path)) { - case 'yaml': - case 'yml': - return std.Format.YAML; - case 'json': - return std.Format.JSON; - case 'hcl': - case 'tf': - return std.Format.HCL; - default: - return std.Format.JSON; - } -} - const isString = (s: any): boolean => typeof s === 'string' || s instanceof String; // represents a file spec that has its promise resolved, if necessary @@ -159,6 +169,33 @@ function validateFormat(files: RealisedFile[], params: GenerateParams) { return { valid, showHelp: !valid }; } +function forceFormat(forced: OutputFormat, files: RealisedFile[]) { + for (const file of files) { + const { path, value, format } = file; + // this makes sure the forced file format is a stream if the + // original file is a stream. + switch (fileFormat(file)) { + case std.Format.YAMLStream: + if (forced === OutputFormat.JSON) { + file.format = std.Format.JSONStream; + } + break; + case std.Format.JSONStream: + if (forced == OutputFormat.YAML) { + file.format = std.Format.YAMLStream; + } + break; + default: + file.format = outputFormatToFormat[forced]; + break; + } + const [p, ext] = splitPath(path); + if (ext !== '') { + file.path = [p, forced].join('.'); + } + } +} + function assembleForStdout(values: RealisedFile[]) { // When writing to stdout, we need to // 1. make sure everything is a mutually compatible format (e.g., @@ -267,6 +304,10 @@ export function generate(definition: GenerateArg, params: GenerateParams) { throw new Error('jk-internal-skip: values failed validation'); } + if (params.format !== undefined) { + forceFormat(params.format, files) + } + if (stdout) { const { valid, stdoutFormat, stream } = assembleForStdout(files); if (!valid) { diff --git a/std/cmd/transform.ts b/std/cmd/transform.ts index b1396dca..274c64eb 100644 --- a/std/cmd/transform.ts +++ b/std/cmd/transform.ts @@ -1,20 +1,21 @@ -import { Format, Overwrite } from '../index'; +import { Format, Overwrite, read, stdin, print } from '../index'; import * as host from '@jkcfg/std/internal/host'; // magic module import * as param from '../param'; -import { generate, File, GenerateParams } from './generate'; -import { valuesFormatFromPath } from '../read'; +import { generate, File, GenerateParams, maybeSetFormat } from './generate'; +import { valuesFormatFromPath, valuesFormatFromExtension } from '../read'; type TransformFn = (value: any) => any | void; -const inputParams: GenerateParams = { +const generateParams: GenerateParams = { stdout: param.Boolean('jk.transform.stdout', false), overwrite: param.Boolean('jk.transform.overwrite', false) ? Overwrite.Write : Overwrite.Err, }; +maybeSetFormat(generateParams, param.String('jk.generate.format', undefined)); // NB jk.generate. param // If we're told to overwrite, we need to be able to write to the // files mentioned on the command-line; but not otherwise. -if (inputParams.overwrite == Overwrite.Write) { - inputParams.writeFile = host.write; +if (generateParams.overwrite == Overwrite.Write) { + generateParams.writeFile = host.write; } function transform(fn: TransformFn): void { @@ -27,7 +28,17 @@ function transform(fn: TransformFn): void { const inputFiles = param.Object('jk.transform.input', {}); const outputs = []; + for (const path of Object.keys(inputFiles)) { + if (path === '') { // read from stdin + const stdinFormat = param.String('jk.transform.stdin.format', 'yaml'); + const format = valuesFormatFromExtension(stdinFormat); + const path = `stdin.${stdinFormat}`; // path is a stand-in + const value = read(stdin, { format }).then(v => v.map(transformOne)); + outputs.push({ path, value, format }); + continue; + } + const format = valuesFormatFromPath(path); outputs.push(host.read(path, { format }).then((obj): File => { switch (format) { @@ -43,7 +54,7 @@ function transform(fn: TransformFn): void { } })); } - generate(Promise.all(outputs), inputParams); + generate(Promise.all(outputs), generateParams); } export default transform; diff --git a/std/read.ts b/std/read.ts index a0c44c56..04100032 100644 --- a/std/read.ts +++ b/std/read.ts @@ -29,13 +29,47 @@ export interface ReadOptions { module?: string; } -// valuesFormatFromPath guesses, for a path, the format that will -// return all values in a file. In other words, it prefers YAML -// streams and concatenated JSON. You may need to treat the read value -// differently depending on the format you got here, since YAMLStream -// and JSONStream will both result in an array of values. -export function valuesFormatFromPath(path: string): Format { - const ext = path.split('.').pop(); +// splitPath returns [all-but-extension, extension] for a path. If a +// path does not end with an extension, it will be an empty string. +export function splitPath(path: string): [string, string] { + const parts = path.split('.'); + const ext = parts.pop(); + // When there's no extension, either there will be a single part (no + // dots anywhere), or a path separator in the last part (a dot + // somewhere before the last path segment) + if (parts.length === 0 || ext.includes('/')) { + return [ext, '']; + } + return [parts.join(''), ext]; +} + +function extension(path: string): string { + return splitPath(path)[1]; +} + +// formatFromPath guesses, for a file path, the format in which to +// read the file. It will assume one value per file, so if you have +// files that may have multiple values (e.g., YAML streams), it's +// better to use `valuesFormatFromPath` and be prepared to get +// multiple values. +export function formatFromPath(path: string): Format { + switch (extension(path)) { + case 'yaml': + case 'yml': + return Format.YAML; + case 'json': + return Format.JSON; + case 'hcl': + case 'tf': + return Format.HCL; + default: + return Format.JSON; + } +} + +// valuesFormatFromExtension returns the format implied by a +// particular file extension. +export function valuesFormatFromExtension(ext: string): Format { switch (ext) { case 'yaml': case 'yml': @@ -47,6 +81,16 @@ export function valuesFormatFromPath(path: string): Format { } } +// valuesFormatFromPath guesses, for a path, the format that will +// return all values in a file. In other words, it prefers YAML +// streams and concatenated JSON. You may need to treat the read value +// differently depending on the format you got here, since YAMLStream +// and JSONStream will both result in an array of values. +export function valuesFormatFromPath(path: string): Format { + const ext = extension(path); + return valuesFormatFromExtension(ext); +} + type ReadPath = string | typeof stdin; // read requests the path and returns a promise that will be resolved diff --git a/tests/generate-force-format.js b/tests/generate-force-format.js new file mode 100644 index 00000000..b829aced --- /dev/null +++ b/tests/generate-force-format.js @@ -0,0 +1,19 @@ +import { Format } from '@jkcfg/std'; + +function valueAndFormat(f) { + return { + format: f, + value: [ + { item1: Format[f] }, + { item2: Format[f] }, + { item3: Format[f] }, + ], + }; +} + +export default [ + { path: 'jsonarray.json', ...valueAndFormat(Format.JSON) }, + { path: 'jsonstream.json', ...valueAndFormat(Format.JSONStream) }, + { path: 'yamlarray.yaml', ...valueAndFormat(Format.YAML) }, + { path: 'yamlstream.yaml', ...valueAndFormat(Format.YAMLStream) }, +]; diff --git a/tests/test-generate-force-format-stdout.js.cmd b/tests/test-generate-force-format-stdout.js.cmd new file mode 100644 index 00000000..5bb5ea26 --- /dev/null +++ b/tests/test-generate-force-format-stdout.js.cmd @@ -0,0 +1 @@ +jk generate --format=yaml --stdout ./generate-force-format.js diff --git a/tests/test-generate-force-format-stdout.js.expected b/tests/test-generate-force-format-stdout.js.expected new file mode 100644 index 00000000..fc72afa5 --- /dev/null +++ b/tests/test-generate-force-format-stdout.js.expected @@ -0,0 +1,19 @@ +- item1: JSON +- item2: JSON +- item3: JSON +--- +item1: JSONStream +--- +item2: JSONStream +--- +item3: JSONStream +--- +- item1: YAML +- item2: YAML +- item3: YAML +--- +item1: YAMLStream +--- +item2: YAMLStream +--- +item3: YAMLStream diff --git a/tests/test-generate-force-format.expected/jsonarray.json b/tests/test-generate-force-format.expected/jsonarray.json new file mode 100644 index 00000000..ffb5c19d --- /dev/null +++ b/tests/test-generate-force-format.expected/jsonarray.json @@ -0,0 +1,11 @@ +[ + { + "item1": "JSON" + }, + { + "item2": "JSON" + }, + { + "item3": "JSON" + } +] diff --git a/tests/test-generate-force-format.expected/jsonstream.json b/tests/test-generate-force-format.expected/jsonstream.json new file mode 100644 index 00000000..d2abed0e --- /dev/null +++ b/tests/test-generate-force-format.expected/jsonstream.json @@ -0,0 +1,3 @@ +{"item1":"JSONStream"} +{"item2":"JSONStream"} +{"item3":"JSONStream"} diff --git a/tests/test-generate-force-format.expected/yamlarray.json b/tests/test-generate-force-format.expected/yamlarray.json new file mode 100644 index 00000000..b7b62406 --- /dev/null +++ b/tests/test-generate-force-format.expected/yamlarray.json @@ -0,0 +1,11 @@ +[ + { + "item1": "YAML" + }, + { + "item2": "YAML" + }, + { + "item3": "YAML" + } +] diff --git a/tests/test-generate-force-format.expected/yamlstream.json b/tests/test-generate-force-format.expected/yamlstream.json new file mode 100644 index 00000000..006ae62f --- /dev/null +++ b/tests/test-generate-force-format.expected/yamlstream.json @@ -0,0 +1,3 @@ +{"item1":"YAMLStream"} +{"item2":"YAMLStream"} +{"item3":"YAMLStream"} diff --git a/tests/test-generate-force-format.js.cmd b/tests/test-generate-force-format.js.cmd new file mode 100644 index 00000000..d52fa3af --- /dev/null +++ b/tests/test-generate-force-format.js.cmd @@ -0,0 +1 @@ +jk generate --format=json -o %d ./generate-force-format.js diff --git a/tests/test-transform-force-format.expected/transform-force-format/items.json b/tests/test-transform-force-format.expected/transform-force-format/items.json new file mode 100644 index 00000000..ce2d9b23 --- /dev/null +++ b/tests/test-transform-force-format.expected/transform-force-format/items.json @@ -0,0 +1,2 @@ +{"item":1,"seen":true} +[{"entry":"one"},{"entry":"two"},{"seen":true}] diff --git a/tests/test-transform-force-format.js.cmd b/tests/test-transform-force-format.js.cmd new file mode 100644 index 00000000..2868e10f --- /dev/null +++ b/tests/test-transform-force-format.js.cmd @@ -0,0 +1 @@ +jk transform --format=json --overwrite -o %d ./transform-force-format.js transform-force-format/*.yaml diff --git a/tests/test-transform-stdin.js.cmd b/tests/test-transform-stdin.js.cmd new file mode 100644 index 00000000..306cdcd7 --- /dev/null +++ b/tests/test-transform-stdin.js.cmd @@ -0,0 +1 @@ +jk transform -c "v => v + 1" --stdin-format=json - ./test-transform-files/*.json diff --git a/tests/test-transform-stdin.js.expected b/tests/test-transform-stdin.js.expected new file mode 100644 index 00000000..687d1fcf --- /dev/null +++ b/tests/test-transform-stdin.js.expected @@ -0,0 +1,3 @@ +7 +2 +3 diff --git a/tests/test-transform-stdin.js.in b/tests/test-transform-stdin.js.in new file mode 100644 index 00000000..1e8b3149 --- /dev/null +++ b/tests/test-transform-stdin.js.in @@ -0,0 +1 @@ +6 diff --git a/tests/transform-force-format.js b/tests/transform-force-format.js new file mode 100644 index 00000000..1dfdf1fb --- /dev/null +++ b/tests/transform-force-format.js @@ -0,0 +1,10 @@ +// Reminder: each file will be treated as a stream, and the function +// called on each value. + +export default function (x) { + if (Array.isArray(x)) { + x.push({ seen: true }); + } else { + x.seen = true; + } +} diff --git a/tests/transform-force-format/items.yaml b/tests/transform-force-format/items.yaml new file mode 100644 index 00000000..3f04f09e --- /dev/null +++ b/tests/transform-force-format/items.yaml @@ -0,0 +1,4 @@ +item: 1 +--- +- entry: one +- entry: two diff --git a/transform.go b/transform.go index a9eed998..d8644d8e 100644 --- a/transform.go +++ b/transform.go @@ -18,24 +18,43 @@ var transformCmd = &cobra.Command{ } const transformExamples = ` - running the default export of a module (or file) on each input document + + run the default export of script.js on each YAML value read from + stdin, printing the transformed values to stdout + + jk transform ./script.js - + + run the default export of a module (or file) on each input document, + and write the results back to outputdir/ + jk transform -o outputdir/ ./script.js ./inputdir/*.json - running a function on each input, and printing the results to stdout + + run a function on each input file, and print the results to stdout + jk transform --stdout -c '({ name: n, ...fields }) => ({ name: n + "-dev", ...fields })' inputdir/*.yaml ` var transformOptions struct { vmOptions scriptOptions - stdout bool // print everything to stdout - overwrite bool // permit the overwriting of input files + // print everything to stdout + stdout bool + // permit the overwriting of input files + overwrite bool + // force the format of the output + format string + // when reading from stdin (`-` is supplied as an argument), + // expect a stream of values in the format given + stdinFormat string } func init() { initScriptFlags(transformCmd, &transformOptions.scriptOptions) initExecFlags(transformCmd, &transformOptions.vmOptions) - transformCmd.PersistentFlags().BoolVar(&transformOptions.stdout, "stdout", false, "print the resulting values to stdout") + transformCmd.PersistentFlags().BoolVar(&transformOptions.stdout, "stdout", false, "print the resulting values to stdout (implied if reading from stdin)") transformCmd.PersistentFlags().BoolVar(&transformOptions.overwrite, "overwrite", false, "allow input file(s) to be overwritten by output file(s); otherwise, an error will be thrown") + transformCmd.PersistentFlags().StringVar(&transformOptions.format, "format", "", "force all output values to this format") + transformCmd.PersistentFlags().StringVar(&transformOptions.stdinFormat, "stdin-format", "yaml", "assume this format for values from stdin (either a 'yaml' stream or concatenated 'json' values)") jk.AddCommand(transformCmd) } @@ -56,14 +75,29 @@ func transform(cmd *cobra.Command, args []string) { // now). This is in part to get around the limitations of // parameters (arrays are not supported as values), and partly in // anticipation of there being more information to pass about each - // input. + // input. Stdin is encoded as an empty string, since `"-"` could + // be the name of a file. inputs := make(map[string]interface{}) for _, f := range args[1:] { - inputs[f] = f + if f == "-" { + inputs[""] = "" + transformOptions.stdout = true + } else { + inputs[f] = f + } } + vm.parameters.Set("jk.transform.input", inputs) vm.parameters.Set("jk.transform.stdout", transformOptions.stdout) vm.parameters.Set("jk.transform.overwrite", transformOptions.overwrite) + setGenerateFormat(transformOptions.format, vm) + + switch transformOptions.stdinFormat { + case "json", "yaml": + vm.parameters.Set("jk.transform.stdin.format", transformOptions.stdinFormat) + default: + log.Fatal("--stdin-format must be 'json' or 'yaml'") + } var module string switch {