ÿØÿà JFIF    ÿÛ „ ( %!1!%*+...983,7(-.- index.js000064400000012532152351002620006210 0ustar00'use strict' var Parser = require('jsonparse') , through = require('through') var bufferFrom = Buffer.from && Buffer.from !== Uint8Array.from /* the value of this.stack that creationix's jsonparse has is weird. it makes this code ugly, but his problem is way harder that mine, so i'll forgive him. */ exports.parse = function (path, map) { var header, footer var parser = new Parser() var stream = through(function (chunk) { if('string' === typeof chunk) chunk = bufferFrom ? Buffer.from(chunk) : new Buffer(chunk) parser.write(chunk) }, function (data) { if(data) stream.write(data) if (header) stream.emit('header', header) if (footer) stream.emit('footer', footer) stream.queue(null) }) if('string' === typeof path) path = path.split('.').map(function (e) { if (e === '$*') return {emitKey: true} else if (e === '*') return true else if (e === '') // '..'.split('.') returns an empty string return {recurse: true} else return e }) var count = 0, _key if(!path || !path.length) path = null parser.onValue = function (value) { if (!this.root) stream.root = value if(! path) return var i = 0 // iterates on path var j = 0 // iterates on stack var emitKey = false; var emitPath = false; while (i < path.length) { var key = path[i] var c j++ if (key && !key.recurse) { c = (j === this.stack.length) ? this : this.stack[j] if (!c) return if (! check(key, c.key)) { setHeaderFooter(c.key, value) return } emitKey = !!key.emitKey; emitPath = !!key.emitPath; i++ } else { i++ var nextKey = path[i] if (! nextKey) return while (true) { c = (j === this.stack.length) ? this : this.stack[j] if (!c) return if (check(nextKey, c.key)) { i++; if (!Object.isFrozen(this.stack[j])) this.stack[j].value = null break } else { setHeaderFooter(c.key, value) } j++ } } } // emit header if (header) { stream.emit('header', header); header = false; } if (j !== this.stack.length) return count ++ var actualPath = this.stack.slice(1).map(function(element) { return element.key }).concat([this.key]) var data = value if(null != data) if(null != (data = map ? map(data, actualPath) : data)) { if (emitKey || emitPath) { data = { value: data }; if (emitKey) data["key"] = this.key; if (emitPath) data["path"] = actualPath; } stream.queue(data) } if (this.value) delete this.value[this.key] for(var k in this.stack) if (!Object.isFrozen(this.stack[k])) this.stack[k].value = null } parser._onToken = parser.onToken; parser.onToken = function (token, value) { parser._onToken(token, value); if (this.stack.length === 0) { if (stream.root) { if(!path) stream.queue(stream.root) count = 0; stream.root = null; } } } parser.onError = function (err) { if(err.message.indexOf("at position") > -1) err.message = "Invalid JSON (" + err.message + ")"; stream.emit('error', err) } return stream function setHeaderFooter(key, value) { // header has not been emitted yet if (header !== false) { header = header || {} header[key] = value } // footer has not been emitted yet but header has if (footer !== false && header === false) { footer = footer || {} footer[key] = value } } } function check (x, y) { if ('string' === typeof x) return y == x else if (x && 'function' === typeof x.exec) return x.exec(y) else if ('boolean' === typeof x || 'object' === typeof x) return x else if ('function' === typeof x) return x(y) return false } exports.stringify = function (op, sep, cl, indent) { indent = indent || 0 if (op === false){ op = '' sep = '\n' cl = '' } else if (op == null) { op = '[\n' sep = '\n,\n' cl = '\n]\n' } //else, what ever you like var stream , first = true , anyData = false stream = through(function (data) { anyData = true try { var json = JSON.stringify(data, null, indent) } catch (err) { return stream.emit('error', err) } if(first) { first = false ; stream.queue(op + json)} else stream.queue(sep + json) }, function (data) { if(!anyData) stream.queue(op) stream.queue(cl) stream.queue(null) }) return stream } exports.stringifyObject = function (op, sep, cl, indent) { indent = indent || 0 if (op === false){ op = '' sep = '\n' cl = '' } else if (op == null) { op = '{\n' sep = '\n,\n' cl = '\n}\n' } //else, what ever you like var first = true var anyData = false var stream = through(function (data) { anyData = true var json = JSON.stringify(data[0]) + ':' + JSON.stringify(data[1], null, indent) if(first) { first = false ; this.queue(op + json)} else this.queue(sep + json) }, function (data) { if(!anyData) this.queue(op) this.queue(cl) this.queue(null) }) return stream } bin.js000064400000000373152351002620005651 0ustar00#! /usr/bin/env node var JSONStream = require('./') if(!module.parent && process.title !== 'browser') { process.stdin .pipe(JSONStream.parse(process.argv[2])) .pipe(JSONStream.stringify('[', ',\n', ']\n', 2)) .pipe(process.stdout) } readme.markdown000064400000013132152351002620007541 0ustar00# JSONStream streaming JSON.parse and stringify ![](https://secure.travis-ci.org/dominictarr/JSONStream.png?branch=master) ## install ```npm install JSONStream``` ## example ``` js var request = require('request') , JSONStream = require('JSONStream') , es = require('event-stream') request({url: 'http://isaacs.couchone.com/registry/_all_docs'}) .pipe(JSONStream.parse('rows.*')) .pipe(es.mapSync(function (data) { console.error(data) return data })) ``` ## JSONStream.parse(path) parse stream of values that match a path ``` js JSONStream.parse('rows.*.doc') ``` The `..` operator is the recursive descent operator from [JSONPath](http://goessner.net/articles/JsonPath/), which will match a child at any depth (see examples below). If your keys have keys that include `.` or `*` etc, use an array instead. `['row', true, /^doc/]`. If you use an array, `RegExp`s, booleans, and/or functions. The `..` operator is also available in array representation, using `{recurse: true}`. any object that matches the path will be emitted as 'data' (and `pipe`d down stream) If `path` is empty or null, no 'data' events are emitted. If you want to have keys emitted, you can prefix your `*` operator with `$`: `obj.$*` - in this case the data passed to the stream is an object with a `key` holding the key and a `value` property holding the data. ### Examples query a couchdb view: ``` bash curl -sS localhost:5984/tests/_all_docs&include_docs=true ``` you will get something like this: ``` js {"total_rows":129,"offset":0,"rows":[ { "id":"change1_0.6995461115147918" , "key":"change1_0.6995461115147918" , "value":{"rev":"1-e240bae28c7bb3667f02760f6398d508"} , "doc":{ "_id": "change1_0.6995461115147918" , "_rev": "1-e240bae28c7bb3667f02760f6398d508","hello":1} }, { "id":"change2_0.6995461115147918" , "key":"change2_0.6995461115147918" , "value":{"rev":"1-13677d36b98c0c075145bb8975105153"} , "doc":{ "_id":"change2_0.6995461115147918" , "_rev":"1-13677d36b98c0c075145bb8975105153" , "hello":2 } }, ]} ``` we are probably most interested in the `rows.*.doc` create a `Stream` that parses the documents from the feed like this: ``` js var stream = JSONStream.parse(['rows', true, 'doc']) //rows, ANYTHING, doc stream.on('data', function(data) { console.log('received:', data); }); //emits anything from _before_ the first match stream.on('header', function (data) { console.log('header:', data) // => {"total_rows":129,"offset":0} }) ``` awesome! In case you wanted the contents the doc emitted: ``` js var stream = JSONStream.parse(['rows', true, 'doc', {emitKey: true}]) //rows, ANYTHING, doc, items in docs with keys stream.on('data', function(data) { console.log('key:', data.key); console.log('value:', data.value); }); ``` You can also emit the path: ``` js var stream = JSONStream.parse(['rows', true, 'doc', {emitPath: true}]) //rows, ANYTHING, doc, items in docs with keys stream.on('data', function(data) { console.log('path:', data.path); console.log('value:', data.value); }); ``` ### recursive patterns (..) `JSONStream.parse('docs..value')` (or `JSONStream.parse(['docs', {recurse: true}, 'value'])` using an array) will emit every `value` object that is a child, grand-child, etc. of the `docs` object. In this example, it will match exactly 5 times at various depth levels, emitting 0, 1, 2, 3 and 4 as results. ```js { "total": 5, "docs": [ { "key": { "value": 0, "some": "property" } }, {"value": 1}, {"value": 2}, {"blbl": [{}, {"a":0, "b":1, "value":3}, 10]}, {"value": 4} ] } ``` ## JSONStream.parse(pattern, map) provide a function that can be used to map or filter the json output. `map` is passed the value at that node of the pattern, if `map` return non-nullish (anything but `null` or `undefined`) that value will be emitted in the stream. If it returns a nullish value, nothing will be emitted. `JSONStream` also emits `'header'` and `'footer'` events, the `'header'` event contains anything in the output that was before the first match, and the `'footer'`, is anything after the last match. ## JSONStream.stringify(open, sep, close) Create a writable stream. you may pass in custom `open`, `close`, and `seperator` strings. But, by default, `JSONStream.stringify()` will create an array, (with default options `open='[\n', sep='\n,\n', close='\n]\n'`) If you call `JSONStream.stringify(false)` the elements will only be seperated by a newline. If you only write one item this will be valid JSON. If you write many items, you can use a `RegExp` to split it into valid chunks. ## JSONStream.stringifyObject(open, sep, close) Very much like `JSONStream.stringify`, but creates a writable stream for objects instead of arrays. Accordingly, `open='{\n', sep='\n,\n', close='\n}\n'`. When you `.write()` to the stream you must supply an array with `[ key, data ]` as the first argument. ## unix tool query npm to see all the modules that browserify has ever depended on. ``` bash curl https://registry.npmjs.org/browserify | JSONStream 'versions.*.dependencies' ``` ## numbers numbers will be emitted as numbers. huge numbers that cannot be represented in memory as javascript numbers will be emitted as strings. cf https://github.com/creationix/jsonparse/commit/044b268f01c4b8f97fb936fc85d3bcfba179e5bb for details. ## Acknowlegements this module depends on https://github.com/creationix/jsonparse by Tim Caswell and also thanks to Florent Jaby for teaching me about parsing with: https://github.com/Floby/node-json-streams ## license Dual-licensed under the MIT License or the Apache License, version 2.0 LICENSE.MIT000064400000002100152351002620006166 0ustar00The MIT License Copyright (c) 2011 Dominic Tarr Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. .travis.yml000064400000000073152351002620006651 0ustar00language: node_js node_js: - 4 - 5 - 6 sudo: false examples/all_docs.js000064400000000713152351002620010475 0ustar00var request = require('request') , JSONStream = require('JSONStream') , es = require('event-stream') var parser = JSONStream.parse(['rows', true]) //emit parts that match this path (any element of the rows array) , req = request({url: 'http://isaacs.couchone.com/registry/_all_docs'}) , logger = es.mapSync(function (data) { //create a stream that logs to stderr, console.error(data) return data }) req.pipe(parser) parser.pipe(logger) LICENSE.APACHE2000064400000001112152351002620006542 0ustar00Apache License, Version 2.0 Copyright (c) 2011 Dominic Tarr Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. package.json000064400000003505152351002620007031 0ustar00{ "_from": "JSONStream@1.3.5", "_id": "JSONStream@1.3.5", "_inBundle": false, "_integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", "_location": "/JSONStream", "_phantomChildren": {}, "_requested": { "type": "version", "registry": true, "raw": "JSONStream@1.3.5", "name": "JSONStream", "escapedName": "JSONStream", "rawSpec": "1.3.5", "saveSpec": null, "fetchSpec": "1.3.5" }, "_requiredBy": [ "#USER", "/", "/npm-registry-fetch" ], "_resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", "_shasum": "3208c1f08d3a4d99261ab64f92302bc15e111ca0", "_spec": "JSONStream@1.3.5", "_where": "/Users/aeschright/code/cli", "author": { "name": "Dominic Tarr", "email": "dominic.tarr@gmail.com", "url": "http://bit.ly/dominictarr" }, "bin": { "JSONStream": "./bin.js" }, "bugs": { "url": "https://github.com/dominictarr/JSONStream/issues" }, "bundleDependencies": false, "dependencies": { "jsonparse": "^1.2.0", "through": ">=2.2.7 <3" }, "deprecated": false, "description": "rawStream.pipe(JSONStream.parse()).pipe(streamOfObjects)", "devDependencies": { "assertions": "~2.2.2", "event-stream": "~0.7.0", "it-is": "~1", "render": "~0.1.1", "tape": "~2.12.3", "trees": "~0.0.3" }, "engines": { "node": "*" }, "homepage": "http://github.com/dominictarr/JSONStream", "keywords": [ "json", "stream", "streaming", "parser", "async", "parsing" ], "license": "(MIT OR Apache-2.0)", "name": "JSONStream", "optionalDependencies": {}, "repository": { "type": "git", "url": "git://github.com/dominictarr/JSONStream.git" }, "scripts": { "test": "node test/run.js" }, "version": "1.3.5" }