Turn Stream into App

This refactor turns the stream page a part of our main H application

Main features:
* Stream is now accessible from our app at the /stream angular route
* Stream has the same header (same look, same code) as the main app
* Authentication from the stream page is now possible
* The same visualsearch-bar is used both for page- and stream search
* Stream page now uses our viewer.html to show annotation cards.
* Replying to/editing/deleting annotations can be done in the stream
page too.

Details
* streamsearch.pt is obsolete and has been delete because app.pt is
used instead of it.
* streamviewer.html is obsolete and has been deleted because
viewer.html is used instead of it.
* The QueryParser class is moved into the streamfilter.coffee and
available is a new service for the app.
* The visualsearch directive now uses functions living in the
<current scope>.search object that any part of the app can configure it.

h/js/app.coffee

@@ -12,6 +12,7 @@ imports = [
   'h.session'
   'h.services'
   'h.socket'
+  'h.streamsearch'
 ]
 
 
@@ -48,6 +49,10 @@ configure = [
       controller: 'SearchController'
       reloadOnSearch: false
       templateUrl: 'page_search.html'
+    $routeProvider.when '/stream',
+      controller: 'StreamSearchController'
+      reloadOnSearch: false
+      templateUrl: 'viewer.html'
     $routeProvider.otherwise
       redirectTo: '/viewer'
 

h/js/controllers.coffee

@@ -33,11 +33,11 @@ class App
     ]
 
   this.$inject = [
-    '$element', '$filter', '$http', '$location', '$rootScope', '$scope', '$timeout',
+    '$element', '$location', '$q', '$rootScope', '$scope', '$timeout',
     'annotator', 'flash', 'session', 'socket', 'streamfilter', 'viewFilter'
   ]
   constructor: (
-     $element,   $filter,   $http,   $location,   $rootScope,   $scope,   $timeout
+     $element,   $location,   $q,   $rootScope,   $scope,   $timeout
      annotator,   flash,   session,   socket,   streamfilter,   viewFilter
   ) ->
     {plugins, host, providers} = annotator
@@ -255,33 +255,51 @@ class App
     $scope.searchFacets = SEARCH_FACETS
     $scope.searchValues = SEARCH_VALUES
 
-    $scope.search = (searchCollection) ->
-      return unless annotator.discardDrafts()
-      return unless searchCollection.models.length
+    $scope.search = {}
+    $scope.search.update = angular.noop
+    $scope.search.clear = angular.noop
+    $scope.search.query = -> $scope.query
 
-      matched = []
-      query =
-        tags: []
-        quote: []
+    $rootScope.$on '$routeChangeSuccess', (event, next, current) ->
+      unless next.$$route? then return
 
-      for item in searchCollection.models
-        {category, value} = item.attributes
-
-        # Stuff we need to collect
-        switch
-          when category in ['text', 'user', 'time', 'group']
-            query[category] = value
-          when category == 'tags'
-            # Tags are specials, because we collect those into an array
-            query.tags.push value.toLowerCase()
-          when category == 'quote'
-            query.quote = query.quote.concat(value.split(/\s+/))
-
-      $location.path('/page_search').search(query)
-
-    $scope.searchClear = ->
-      $location.url('/viewer')
-      $scope.show_search = false
+      unless next.$$route.originalPath is '/stream'
+        session.$promise.then ->
+          $scope.updater.then (sock) ->
+            $timeout ->
+              entities = Object.keys(plugins.Store?.entities or {})
+              streamfilter.resetFilter().addClause('/uri', 'one_of', entities)
+              filter = streamfilter.getFilter()
+              sock.send(JSON.stringify(filter: filter))
+
+        $scope.search.update = (searchCollection) ->
+          return unless annotator.discardDrafts()
+          return unless searchCollection.models.length
+
+          models = searchCollection.models
+          matched = []
+          query =
+            tags: []
+            quote: []
+
+          for item in models
+            {category, value} = item.attributes
+
+            # Stuff we need to collect
+            switch
+              when category in ['text', 'user', 'time', 'group']
+                query[category] = value
+              when category == 'tags'
+                # Tags are specials, because we collect those into an array
+                query.tags.push value.toLowerCase()
+              when category == 'quote'
+                query.quote = query.quote.concat(value.split(/\s+/))
+
+          $location.path('/page_search').search(query)
+
+        $scope.search.clear = ->
+          $location.url('/viewer')
+          $scope.show_search = false
 
     $scope.reloadAnnotations = ->
       $rootScope.applyView "Screen"
@@ -337,27 +355,25 @@ class App
           annotations = (a for a in annotations when a not in deleted)
           if annotations.length is 0
             annotator.unsubscribe 'annotationsLoaded', cleanup
-            $scope.$broadcast 'ReRenderPageSearch'
+            $scope.$broadcast 'RefreshSearch'
         , 10
       cleanup (a for a in annotations when a.thread)
       annotator.subscribe 'annotationsLoaded', cleanup
 
     $scope.initUpdater = ->
-      filter =
-        streamfilter
-          .setPastDataNone()
-          .setMatchPolicyIncludeAny()
-          .addClause('/uri', 'one_of', Object.keys(plugins.Store.entities))
-          .getFilter()
+      _dfdSock = $q.defer()
+      _sock = socket()
 
-      $scope.updater = socket()
-      $scope.updater.onopen = ->
-        $scope.updater.send(JSON.stringify({filter}))
+      $scope.updater?.then (sock) -> sock.close()
+      $scope.updater = _dfdSock.promise
 
-      $scope.updater.onclose = =>
-        $timeout $scope.initUpdater, 60000
+      _sock.onopen = ->
+        _dfdSock.resolve(_sock)
 
-      $scope.updater.onmessage = (msg) =>
+      _sock.onclose = ->
+        $scope.initUpdater()
+
+      _sock.onmessage = (msg) ->
         #console.log msg
         unless msg.data.type? and msg.data.type is 'annotation-notification'
           return
@@ -376,7 +392,7 @@ class App
         else
           $scope.applyUpdates action, data
 
-    $scope.markAnnotationUpdate = (data) =>
+    $scope.markAnnotationUpdate = (data) ->
       for annotation in data
         # We need to flag the top level
         if annotation.references?
@@ -389,9 +405,30 @@ class App
         else
           annotation._updatedAnnotation = true
 
-    $scope.applyUpdates = (action, data) =>
+    $scope.applyUpdates = (action, data) ->
+      return unless data?.length
+      if action == 'past'
+        action = 'create'
+
+      inRootScope = (annotation) ->
+        for ann in $rootScope.annotations
+          return true if ann.id is annotation.id
+        false
+
       switch action
         when 'create'
+          # Sorting the data for updates.
+          # Because sometimes a reply can arrive in the same package as the
+          # Root annotation, we have to make a len(references, updates sort
+          data.sort (a,b) ->
+            ref_a = a.references?.length or 0
+            ref_b = b.references?.length or 0
+            return ref_a - ref_b if ref_a != ref_b
+
+            a_upd = if a.updated? then new Date(a.updated) else new Date()
+            b_upd = if b.updated? then new Date(b.updated) else new Date()
+            a_upd.getTime() - b_upd.getTime()
+
           # XXX: Temporary workaround until solving the race condition for annotationsLoaded event
           # Between threading and bridge plugins
           for annotation in data
@@ -399,23 +436,37 @@ class App
 
           $scope.markAnnotationUpdate data
 
-          $scope.$apply =>
-            if plugins.Store?
-              plugins.Store._onLoadAnnotations data
-              # XXX: Ugly workaround to update the scope content
+          if plugins.Store?
+            plugins.Store._onLoadAnnotations data
+            # XXX: Ugly workaround to update the scope content
+
+            for annotation in data
               switch $rootScope.viewState.view
                 when 'Document'
                   unless annotator.isComment(annotation) or annotation.references?
-                    $rootScope.annotations.push annotation
+                    $rootScope.annotations.push annotation if not inRootScope(annotation)
                 when 'Comments'
                   if annotator.isComment(annotation)
-                    $rootScope.annotations.push annotation
+                    $rootScope.annotations.push annotation if not inRootScope(annotation)
+                else
+                    $rootScope.annotations.push annotation if not inRootScope(annotation)
         when 'update'
           $scope.markAnnotationUpdate data
           plugins.Store._onLoadAnnotations data
+
+          if $location.path() is '/stream'
+            for annotation in data
+              $rootScope.annotations.push annotation if not inRootScope(annotation)
         when 'delete'
           $scope.markAnnotationUpdate data
+
           for annotation in data
+            # Remove it from the rootScope too
+            for ann, index in $rootScope.annotations
+              if ann.id is annotation.id
+                $rootScope.annotations.splice(index, 1)
+                break
+
             container = annotator.threading.getContainer annotation.id
             if container.message
               # XXX: This is a temporary workaround until real client-side only
@@ -424,6 +475,9 @@ class App
               plugins.Store.annotations[index..index] = [] if index > -1
               annotator.deleteAnnotation container.message
 
+      # Refresh page_search
+      $rootScope.$broadcast 'RefreshSearch' if $location.path() is '/page_search' and data.length
+
       # Finally blink the changed tabs
       $timeout =>
         for p in annotator.providers
@@ -706,7 +760,7 @@ class Editor
 class Viewer
   this.$inject = [
     '$location', '$rootScope', '$routeParams', '$scope',
-    'annotator', 'viewFilter'
+    'annotator'
   ]
   constructor: (
     $location, $rootScope, $routeParams, $scope,
@@ -929,7 +983,7 @@ class Search
       for thread in threads
         $rootScope.focus thread.message, true
 
-    $scope.$on 'ReRenderPageSearch', refresh
+    $scope.$on 'RefreshSearch', refresh
     $scope.$on '$routeUpdate', refresh
 
     $scope.getThreadId = (id) ->

h/js/directives.coffee

@@ -386,15 +386,6 @@ fuzzytime = ['$filter', '$window', ($filter, $window) ->
   template: '<a target="_blank" href="{{shared_link}}" title="{{hint}}">{{ftime | date:mediumDate}}</a>'
 ]
 
-streamviewer = [ ->
-  link: (scope, elem, attr, ctrl) ->
-    return unless ctrl?
-
-  require: '?ngModel'
-  restrict: 'E'
-  templateUrl: 'streamviewer.html'
-]
-
 
 visualSearch = ['$parse', ($parse) ->
   link: (scope, elem, attr, ctrl) ->
@@ -423,6 +414,9 @@ visualSearch = ['$parse', ($parse) ->
           values = _values(scope)?[facet]
           callback(values or [], preserveOrder: true)
 
+    scope.$on 'VSSearch', ->
+      _search(scope, {"this": _vs.searchQuery})
+
     scope.$watch attr.query, (query) ->
       terms =
         for k, v of query
@@ -462,6 +456,5 @@ angular.module('h.directives', ['ngSanitize'])
 .directive('username', username)
 .directive('userPicker', userPicker)
 .directive('repeatAnim', repeatAnim)
-.directive('streamviewer', streamviewer)
 .directive('visualSearch', visualSearch)
 .directive('whenscrolled', whenscrolled)

h/js/streamfilter.coffee

+# This class will process the results of search and generate the correct filter
+# It expects the following dict format as rules
+# { facet_name : {
+#      formatter: to format the value (optional)
+#      path: json path mapping to the annotation field
+#      exact_match: true|false (default: true)
+#      case_sensitive: true|false (default: false)
+#      and_or: and|or for multiple values should it threat them as 'or' or 'and' (def: or)
+#      operator: if given it'll use this operator regardless of other circumstances
+#
+#      options: backend specific options
+#      options.es: elasticsearch specific options
+#      options.es.query_type : can be: simple, query_string, match
+#         defaults to: simple, determines which es query type to use
+#      options.es.cutoff_frequency: if set, the query will be given a cutoff_frequency for this facet
+#      options.es.and_or: match queries can use this, defaults to and
+# }
+# The models is the direct output from visualsearch
+class QueryParser
+  rules:
+    user:
+      formatter: (user) ->
+        'acct:' + user + '@' + window.location.hostname
+      path: '/user'
+      exact_match: true
+      case_sensitive: false
+      and_or: 'or'
+    text:
+      path: '/text'
+      exact_match: false
+      case_sensitive: false
+      and_or: 'and'
+    tags:
+      path: '/tags'
+      exact_match: false
+      case_sensitive: false
+      and_or: 'or'
+    quote:
+      path: "/quote"
+      exact_match: false
+      case_sensitive: false
+      and_or: 'and'
+    uri:
+      formatter: (uri) ->
+        uri.toLowerCase()
+      path: '/uri'
+      exact_match: false
+      case_sensitive: false
+      and_or: 'or'
+      options:
+        es:
+         query_type: 'match'
+         cutoff_frequency: 0.001
+         and_or: 'and'
+    since:
+      formatter: (past) ->
+        seconds =
+          switch past
+            when '5 min' then 5*60
+            when '30 min' then 30*60
+            when '1 hour' then 60*60
+            when '12 hours' then 12*60*60
+            when '1 day' then 24*60*60
+            when '1 week' then 7*24*60*60
+            when '1 month' then 30*24*60*60
+            when '1 year' then 365*24*60*60
+        new Date(new Date().valueOf() - seconds*1000)
+      path: '/created'
+      exact_match: false
+      case_sensitive: true
+      and_or: 'and'
+      operator: 'ge'
+
+  populateFilter: (filter, models) =>
+    # First cluster the different facets into categories
+    categories = {}
+    for searchItem in models
+      category = searchItem.attributes.category
+      value = searchItem.attributes.value
+
+      limit = 50
+      if category is 'results' then limit = value
+      else
+        if category is 'text'
+          # Visualsearch sickly automatically cluster the text field
+          # (and only the text filed) into a space separated string
+          catlist = []
+          catlist.push val for val in value.split ' '
+          categories[category] = catlist
+        else
+          if category of categories then categories[category].push value
+          else categories[category] = [value]
+
+    # Now for the categories
+    for category, values of categories
+      unless @rules[category]? then continue
+      unless values.length then continue
+      rule = @rules[category]
+
+      # Now generate the clause with the help of the rule
+      exact_match = if rule.exact_match? then rule.exact_match else true
+      case_sensitive = if rule.case_sensitive? then rule.case_sensitive else false
+      and_or = if rule.and_or? then rule.and_or else 'or'
+      mapped_field = if rule.path? then rule.path else '/'+category
+
+      if values.length is 1
+        oper_part =
+          if rule.operator? then rule.operator
+          else if exact_match then 'equals' else 'matches'
+        value_part = if rule.formatter then rule.formatter values[0] else values[0]
+        filter.addClause mapped_field, oper_part, value_part, case_sensitive, rule.options
+      else
+        if and_or is 'or'
+          val_list = ''
+          first = true
+          for val in values
+            unless first then val_list += ',' else first = false
+            value_part = if rule.formatter then rule.formatter val else val
+            val_list += value_part
+          oper_part =
+            if rule.operator? then rule.operator
+            else if exact_match then 'one_of' else 'match_of'
+          filter.addClause mapped_field, oper_part, val_list, case_sensitive, rule.options
+        else
+          oper_part =
+            if rule.operator? then rule.operator
+            else if exact_match then 'equals' else 'matches'
+          for val in values
+            value_part = if rule.formatter then rule.formatter val else val
+            filter.addClause mapped_field, oper_part, value_part, case_sensitive, rule.options
+
+    categories['results'] = [limit]
+
+    categories
+
+
 class StreamFilter
   strategies: ['include_any', 'include_all', 'exclude_any', 'exclude_all']
   past_modes: ['none','hits','time']
@@ -103,5 +239,6 @@ class StreamFilter
     this
 
 
-angular.module('h.streamfilter',['bootstrap'])
-  .service('streamfilter', StreamFilter)
\ No newline at end of file
+angular.module('h.streamfilter', [])
+.service('queryparser', QueryParser)
+.service('streamfilter', StreamFilter)