Merge pull request #296 from bhunjadi/scoped-publish

[WIP] Scoped publications to avoid client side documents overlap

docs/named_queries.md

@@ -426,6 +426,95 @@ query.expose({
 })
 ```
 
+## Scoped publications
+
+Scoped publications add a scope (context) to their published documents with the goal of client being able to distinguish between documents of different publications or different grapher paths in the same publication.
+
+Problems often arise when using only server-side filtering.
+
+Consider the following situation:
+
+```js
+// the query
+const usersQuery = Users.createQuery('getUsers', {
+    name: 1,
+    friends: {
+        name: 1,
+    },
+}, {
+    scoped: true,
+});
+
+// server-side exposure
+usersQuery.expose({
+    embody: {
+        $filter({filters, params}) {
+            filters.name = params.name;
+        }
+    }
+});
+
+// links
+Users.addLinks({
+    friends: {
+        collection: Users,
+        field: 'friendIds',
+        type: 'many'
+    },
+});
+```
+
+Notice that `friends` is a link from Users to Users collection. Also, we have server-side filtering (see exposure).
+On the client, we want to fetch reactively one user by name, but we are going to get all of his friends, too, and that is because of the `friends` link.
+```js
+// querying for user John
+withQuery(props => {
+    return usersQuery.clone({
+        name: 'John',
+    });
+}, {
+    reactive: true,
+})(SomeComponent);
+```
+
+Client receives queried user (John) and all of his friends into the local Users collection.
+By passing `{scoped: true}` query parameter to the `createQuery()`, client-side recursive fetching is now able to distinguish between queried user and his friends.
+
+### Technical details
+Continuing on the example above, there are two pieces on how server and client achieve this functionality.
+
+#### Subscription scope
+Each subscription adds `_sub_<subscriptionId>` field to its documents. For example, a User document could look like this:
+```
+{
+    name: 'John',
+    _sub_1: 1,
+    _sub_2: 1
+}
+```
+This way we ensure that there is no mixup between the subscriptions (i.e. between two reactive queries on the client).
+
+#### Query path scope
+Now suppose Alice is John's friend. Both Alice and John would have the same `_sub_<id>` field for our example query and we would get both instead of only John.
+This part is solved by adding "query path" field to the docs, in format `_query_path_<namespace>` where namespace is path constructed from collection name and link names, for example:
+```
+{
+    name: 'John',
+    _sub_1: 1,
+    _query_path_users: 1,
+},
+{
+    name: 'Alice',
+    _sub_1: 1,
+    // deeper nesting than John's
+    _query_path_users_friends: 1 
+}
+```
+
+where Alice has namespace equal to `users_friends` and client-side recursive fetching can now distinguish between the documents that should be returned as a query result (John) and as a `friends` link results for John (which is Alice).
+
+By adding query path field into the documents, we ensure that there is no mixup between the documents in the same reactive query (i.e. subscription).
+
 ## Conclusion
 
 We can now safely expose our queries to the client, and the client can use it in a simple and uniform way.

lib/namedQuery/expose/extension.js

@@ -154,6 +154,12 @@ _.extend(NamedQuery.prototype, {
         const self = this;
 
         Meteor.publishComposite(this.name, function(params = {}) {
+            const isScoped = !!self.options.scoped;
+
+            if (isScoped) {
+                this.enableScope();
+            }
+
             self._unblockIfNecessary(this);
             self.doValidateParams(params);
             self._callFirewall(this, this.userId, params);
@@ -168,7 +174,7 @@ _.extend(NamedQuery.prototype, {
 
             const rootNode = createGraph(self.collection, body);
 
-            return recursiveCompose(rootNode);
+            return recursiveCompose(rootNode, undefined, {scoped: isScoped});
         });
     },
 

lib/namedQuery/expose/schema.js

@@ -18,5 +18,5 @@ export const ExposeSchema = {
     ),
     validateParams: Match.Maybe(
         Match.OneOf(Object, Function)
-    ),
+    )
 };

lib/namedQuery/namedQuery.client.js

@@ -2,9 +2,11 @@ import CountSubscription from '../query/counts/countSubscription';
 import createGraph from '../query/lib/createGraph.js';
 import recursiveFetch from '../query/lib/recursiveFetch.js';
 import prepareForProcess from '../query/lib/prepareForProcess.js';
-import { _ } from 'meteor/underscore';
+import {_} from 'meteor/underscore';
 import callWithPromise from '../query/lib/callWithPromise';
 import Base from './namedQuery.base';
+import {LocalCollection} from 'meteor/minimongo';
+
 
 export default class extends Base {
     /**
@@ -179,7 +181,10 @@ export default class extends Base {
         }
 
         return recursiveFetch(
-            createGraph(this.collection, body)
-        );
+            createGraph(this.collection, body),
+            undefined, {
+                scoped: this.options.scoped,
+                subscriptionHandle: this.subscriptionHandle
+            });
     }
 }

lib/namedQuery/testing/bootstrap/queries/index.js

@@ -1,15 +1,18 @@
 import postList from './postList';
 import postListCached from './postListCached';
 import postListExposure from './postListExposure';
+import postListExposureScoped from './postListExposureScoped';
 import postListParamsCheck from './postListParamsCheck';
 import postListParamsCheckServer from './postListParamsCheckServer';
 import postListResolver from './postListResolver';
 import postListResolverCached from './postListResolverCached';
+import userListScoped from './userListScoped';
 
 export {
     postList,
     postListCached,
     postListExposure,
+    postListExposureScoped,
     postListParamsCheck,
     postListParamsCheckServer,
     postListResolver,

lib/namedQuery/testing/bootstrap/queries/postList.js

@@ -19,6 +19,8 @@ const postList = createQuery('postList', {
             name: 1
         }
     }
+}, {
+    scoped: true,
 });
 
 export default postList;

lib/namedQuery/testing/bootstrap/queries/postListExposureScoped.js

@@ -0,0 +1,29 @@
+import { createQuery } from 'meteor/cultofcoders:grapher';
+
+const postListExposureScoped = createQuery('postListExposureScoped', {
+    posts: {
+        title: 1,
+        author: {
+            name: 1
+        },
+        group: {
+            name: 1
+        }
+    }
+}, {
+  scoped: true,
+});
+
+if (Meteor.isServer) {
+    postListExposureScoped.expose({
+        firewall(userId, params) {
+        },
+        embody: {
+            $filter({filters, params}) {
+                filters.title = params.title
+            }
+        }
+    });
+}
+
+export default postListExposureScoped;

lib/namedQuery/testing/bootstrap/queries/userListScoped.js

@@ -0,0 +1,29 @@
+import { createQuery } from 'meteor/cultofcoders:grapher';
+
+const userListScoped = createQuery('userListScoped', {
+  users: {
+    name: 1,
+    friends: {
+      name: 1
+    },
+    subordinates: {
+      name: 1,
+    }
+  }
+}, {
+  scoped: true
+});
+
+if (Meteor.isServer) {
+  userListScoped.expose({
+    firewall() {
+    },
+    embody: {
+      $filter({filters, params}) {
+        filters.name = params.name;
+      }
+    }
+  })
+}
+
+export default userListScoped;

lib/namedQuery/testing/client.test.js

@@ -1,5 +1,9 @@
 import postListExposure from './bootstrap/queries/postListExposure.js';
+import postListExposureScoped from './bootstrap/queries/postListExposureScoped';
+import userListScoped from './bootstrap/queries/userListScoped';
 import { createQuery } from 'meteor/cultofcoders:grapher';
+import Posts from '../../query/testing/bootstrap/posts/collection';
+import Users from '../../query/testing/bootstrap/users/collection';
 
 describe('Named Query', function() {
     it('Should return proper values', function(done) {
@@ -114,6 +118,83 @@ describe('Named Query', function() {
         });
     });
 
+    it('Should work with reactive scoped queries', function(done) {
+        const query = postListExposureScoped.clone({ title: 'User Post - 3' });
+
+        const handle = query.subscribe();
+        Tracker.autorun(c => {
+            if (handle.ready()) {
+                c.stop();
+                const data = query.fetch();
+                handle.stop();
+
+                assert.isTrue(data.length > 0);
+
+                const docMap = Posts._collection._docs._map;
+                const scopeField = `_sub_${handle.subscriptionId}`;
+                const queryPathField = '_query_path_posts';
+                data.forEach(post => {
+                    // no scope field returned from find
+                    assert.isUndefined(post[scopeField]);
+                    assert.isObject(docMap[post._id]);
+                    assert.equal(docMap[post._id][scopeField], 1);
+                    assert.equal(docMap[post._id][queryPathField], 1);
+                });
+
+                done();
+            }
+        });
+    });
+
+    it('Should work with reactive recursive scoped queries', function (done) {
+        const query = userListScoped.clone({name: 'User - 3'});
+
+        const handle = query.subscribe();
+        Tracker.autorun(c => {
+            if (handle.ready()) {
+                c.stop();
+                const data = query.fetch();
+                handle.stop();
+
+                assert.equal(data.length, 1);
+                // User 3 has users 0,1,2 as friends and user 2 as subordinate
+                const [user3] = data;
+                assert.equal(user3.friends.length, 3);
+
+                const docMap = Users._collection._docs._map;
+                // users collection on the client should have 4 items (user 3 and friends - user 0,1,2)
+                assert.equal(_.keys(docMap).length, 4);
+
+                const scopeField = `_sub_${handle.subscriptionId}`;
+                const rootQueryPathField = '_query_path_users';
+                const friendsQueryPathField = '_query_path_users_friends';
+                const adversaryQueryPathField = '_query_path_users_subordinates';
+                Object.entries(docMap).forEach(([userId, userDoc]) => {
+                    const isRoot = userId === user3._id;
+                    assert.equal(userDoc[scopeField], 1);
+                    if (isRoot) {
+                        assert.equal(userDoc[rootQueryPathField], 1);
+                        assert.isTrue(!(friendsQueryPathField in userDoc));
+                        assert.isTrue(!(adversaryQueryPathField in userDoc));
+                    }
+                    else {
+                        assert.equal(userDoc[friendsQueryPathField], 1);
+                        assert.isTrue(!(rootQueryPathField in userDoc));
+
+                        if (userDoc.name === 'User - 2') {
+                            assert.equal(userDoc[adversaryQueryPathField], 1);
+                        }
+                        else {
+                            assert.isTrue(!(adversaryQueryPathField in userDoc));
+                        }
+                    }
+                });
+            
+                done();
+            }
+        });
+    });
+
     it('Should work with reactive queries via import', function(done) {
         const query = postListExposure.clone({
             title: 'User Post - 3',

lib/query/lib/createGraph.js

@@ -112,6 +112,24 @@ export function addFieldNode(body, fieldName, root) {
     }
 }
 
+/**
+ * Returns namespace for node when using query path scoping.
+ *
+ * @param node
+ * @returns {String}
+ */
+export function getNodeNamespace(node) {
+    const parts = [];
+    let n = node;
+    while (n) {
+        const name = n.linker ? n.linker.linkName : n.collection._name;
+        parts.push(name);
+        // console.log('linker', node.linker ? node.linker.linkName : node.collection._name);
+        n = n.parent;
+    }
+    return parts.reverse().join('_');
+}
+
 /**
  * @param collection
  * @param body

lib/query/lib/recursiveCompose.js

@@ -1,6 +1,28 @@
 import applyProps from './applyProps.js';
+import {getNodeNamespace} from './createGraph';
 
-function compose(node, userId) {
+/**
+ * Adds _query_path fields to the cursor docs which are used for scoped query filtering on the client.
+ * 
+ * @param cursor 
+ * @param ns 
+ */
+function patchCursor(cursor, ns) {
+    const originalObserve = cursor.observe;
+    cursor.observe = function (callbacks) {
+        const newCallbacks = Object.assign({}, callbacks);
+        if (callbacks.added) {
+            newCallbacks.added = doc => {
+                doc = _.clone(doc);
+                doc[`_query_path_${ns}`] = 1;
+                callbacks.added(doc);
+            };
+        }
+        originalObserve.call(cursor, newCallbacks);
+    };
+}
+
+function compose(node, userId, config) {
     return {
         find(parent) {
             if (parent) {
@@ -18,7 +40,11 @@ function compose(node, userId) {
                     });
                 }
 
-                return accessor.find(filters, options, userId);
+                const cursor = accessor.find(filters, options, userId);
+                if (config.scoped) {
+                    patchCursor(cursor, getNodeNamespace(node));
+                }
+                return cursor;
             }
         },
 
@@ -26,18 +52,22 @@ function compose(node, userId) {
     }
 }
 
-export default (node, userId, config = {bypassFirewalls: false}) => {
+export default (node, userId, config = {bypassFirewalls: false, scoped: false}) => {
     return {
         find() {
             let {filters, options} = applyProps(node);
 
-            return node.collection.find(filters, options, userId);
+            const cursor = node.collection.find(filters, options, userId);
+            if (config.scoped) {
+                patchCursor(cursor, getNodeNamespace(node));
+            }
+            return cursor;
         },
 
         children: _.map(node.collectionNodes, n => {
             const userIdToPass = (config.bypassFirewalls) ? undefined : userId;
 
-            return compose(n, userIdToPass);
+            return compose(n, userIdToPass, config);
         })
     }
 }

lib/query/lib/recursiveFetch.js

@@ -1,16 +1,26 @@
 import applyProps from './applyProps.js';
 import { assembleMetadata, removeLinkStorages, storeOneResults } from './prepareForDelivery';
 import prepareForDelivery from './prepareForDelivery';
+import {getNodeNamespace} from './createGraph';
 
 /**
  * This is always run client side to build the data graph out of client-side collections.
  *
  * @param node
  * @param parentObject
+ * @param fetchOptions
  * @returns {*}
  */
-function fetch(node, parentObject) {
+function fetch(node, parentObject, fetchOptions = {}) {
     let {filters, options} = applyProps(node);
+    // add subscription filter
+    if (fetchOptions.scoped && fetchOptions.subscriptionHandle) {
+        _.extend(filters, fetchOptions.subscriptionHandle.scopeQuery());
+    }
+    // add query path filter
+    if (fetchOptions.scoped) {
+        _.extend(filters, {[`_query_path_${getNodeNamespace(node)}`]: {$exists: true}});
+    }
 
     let results = [];
 
@@ -53,8 +63,8 @@ function fetch(node, parentObject) {
     return results;
 }
 
-export default (node, params) => {
-    node.results = fetch(node);
+export default (node, params, fetchOptions) => {
+    node.results = fetch(node, null, fetchOptions);
 
     prepareForDelivery(node, params);
 

lib/query/testing/bootstrap/fixtures.js

@@ -7,16 +7,19 @@ import Comments from './comments/collection';
 import Posts from './posts/collection';
 import Tags from './tags/collection';
 import Groups from './groups/collection';
+import Users from './users/collection';
 
 Authors.remove({});
 Comments.remove({});
 Posts.remove({});
 Tags.remove({});
 Groups.remove({});
+Users.remove({});
 
 const AUTHORS = 6;
 const POST_PER_USER = 6;
 const COMMENTS_PER_POST = 6;
+const USERS = 4;
 const TAGS = ['JavaScript', 'Meteor', 'React', 'Other'];
 const GROUPS = ['JavaScript', 'Meteor', 'React', 'Other'];
 const COMMENT_TEXT_SAMPLES = [
@@ -81,4 +84,16 @@ _.each(authors, (author) => {
     })
 });
 
+const friendIds = [];
+// each user is created so his friends are previously added users
+_.range(USERS).forEach(idx => {
+    const id = Users.insert({
+        name: `User - ${idx}`,
+        friendIds,
+        subordinateIds: idx === 3 ? [friendIds[2]] : [],
+    });
+
+    friendIds.push(id);
+});
+
 console.log('[ok] fixtures have been loaded.');

lib/query/testing/bootstrap/index.js

@@ -4,10 +4,12 @@ import './authors/links';
 import './tags/links';
 import './groups/links';
 import './security/links';
+import './users/links';
 
 import Posts from './posts/collection';
 import Groups from './groups/collection';
 import Authors from './authors/collection';
+import Users from './users/collection';
 
 if (Meteor.isServer) {
     Posts.expose();

lib/query/testing/bootstrap/users/collection.js

@@ -0,0 +1,2 @@
+const Users = new Mongo.Collection('users');
+export default Users;

lib/query/testing/bootstrap/users/links.js

@@ -0,0 +1,14 @@
+import Users from './collection';
+
+Users.addLinks({
+  friends: {
+      collection: Users,
+      field: 'friendIds',
+      type: 'many'
+  },
+  subordinates: {
+    collection: Users,
+    field: 'subordinateIds',
+    type: 'many'
+  }
+});

package.js

@@ -31,6 +31,7 @@ Package.onUse(function(api) {
         'reywood:publish-composite@1.5.2',
         'dburles:mongo-collection-instances@0.3.5',
         'herteby:denormalize@0.6.5',
+        'peerlibrary:subscription-scope@0.1.0',
     ];
 
     api.use(packages);
@@ -49,6 +50,7 @@ Package.onTest(function(api) {
         'reywood:publish-composite@1.5.2',
         'dburles:mongo-collection-instances@0.3.5',
         'herteby:denormalize@0.6.5',
+        'peerlibrary:subscription-scope@0.1.0',
         'mongo',
     ];