Add new new wildcard operators for web-public streams.

Now that we have a concept of web-public streams, many archive sites
will want to use the set of web-public streams for their archive;
rewrite the stream selection syntax to make that easy to do.

Fixes #80.
Fixes #zulip/zulip/21789.

Co-authored-by: rht <rhtbot@protonmail.com>
Co-authored-by: Tim Abbott <tabbott@zulip.com>

Author: 3 people

default_settings.py

@@ -119,20 +119,22 @@
   - data structures
 
 
-If you want to include all the streams you can use '*'
+A few wildcard operators are supported.
 
 Example
 ---
 
 included:
-  - '*'
+  - 'web-public:*'
 
-Using '*' includes all the **public streams** in Zulip archive.  You
-can make the settings more restrictive than that, but not the opposite
-direction.
+Using 'web-public:*' includes all the **web-public streams** in the
+Zulip organization. Using 'public:*' includes all the **public
+streams** in Zulip archive (`*` will do the same thing, for
+backwards-compatibility).  You can make the settings more restrictive
+than that, but not the opposite direction.
 
-If you want to exclude some public streams, mention them under `excluded`
-category in `streams.yaml`
+If you want to exclude some public streams, mention them in the
+`excluded` category in `streams.yaml`.
 
 Example:
 ---

default_streams.yaml

@@ -1,8 +1,9 @@
 included:
-- '*'
+- 'web-public:*'
 # - general
 # - javascript
 # - data structures
+# - 'public:*'
 
 excluded:
 # - checkins

lib/common.py

@@ -28,12 +28,20 @@ def stream_validator(settings):
     included_streams = set(settings.included_streams)
 
     def validator(stream):
-        if stream in excluded_streams:
+        stream_name = stream["name"]
+        is_web_public = stream["is_web_public"]
+        is_public = not stream["invite_only"]
+
+        if stream_name in excluded_streams:
             return False
 
-        if "*" in included_streams:
+        if "web-public:*" in included_streams and is_web_public:
+            return True
+
+        # The bare * case is for backwards-compatibility.
+        if ("*" in included_streams or "public:*" in included_streams) and is_public:
             return True
 
-        return stream in included_streams
+        return stream_name in included_streams
 
     return validator

lib/populate.py

@@ -100,12 +100,13 @@ def safe_request(cmd, *args, **kwargs):
 
 
 def get_streams(client):
-    # In the future, we may want to change this to
-    # include_web_public=True, for organizations that might want to
-    # use the upcoming web_public flag; but at the very least we
-    # should only include public streams.
+    # Fetch metadata on all streams the current user has access to.
+    # We will filter these for processing via is_valid_stream_name.
     response = safe_request(
-        client.get_streams, include_public=True, include_subscribed=False
+        client.get_streams,
+        include_public=True,
+        include_subscribed=True,
+        include_web_public=True,
     )
     return response["streams"]
 
@@ -117,8 +118,7 @@ def populate_all(
     is_valid_stream_name,
 ):
     all_streams = get_streams(client)
-
-    streams = [s for s in all_streams if is_valid_stream_name(s["name"])]
+    streams = [s for s in all_streams if is_valid_stream_name(s)]
 
     streams_data = {}
 
@@ -199,7 +199,7 @@ def populate_incremental(
     js = json.load(f)
     f.close()
 
-    for s in (s for s in streams if is_valid_stream_name(s["name"])):
+    for s in (s for s in streams if is_valid_stream_name(s)):
         print(s["name"])
         if s["name"] not in js["streams"]:
             js["streams"][s["name"]] = {

tests/testCommon.py

@@ -1,9 +1,7 @@
-# For convenience, just run the tests from its own directory
-import os
+# For convenience, just run the tests in the repo root directory.
 import sys
 
-os.chdir(os.path.dirname(sys.argv[0]))
-sys.path.append("../lib")
+sys.path.append("lib")
 
 import common
 import url
@@ -48,27 +46,49 @@ def test_sanitize():
 
 
 def test_validator():
-    validator = common.stream_validator(Settings(included_streams=["*"]))
-
-    assert_equal(validator("foo"), True)
+    def stream(name, public, web_public):
+        # Returns a minimalist stream dictionary.
+        return {"name": name, "invite_only": not public, "is_web_public": web_public}
+
+    # Test wildcard operator for public streams.
+    for k in ["*", "public:*"]:
+        validator = common.stream_validator(Settings(included_streams=[k]))
+        assert_equal(validator(stream("foo", True, False)), True)
+        assert_equal(validator(stream("foo", True, True)), True)
+        assert_equal(validator(stream("bar", False, False)), False)
+        assert_equal(validator(stream("bar", False, True)), False)
+
+    # Test web-public
+    validator = common.stream_validator(Settings(included_streams=["web-public:*"]))
+    assert_equal(validator(stream("foo", True, False)), False)
+    assert_equal(validator(stream("foo", True, True)), True)
+    assert_equal(validator(stream("bar", False, False)), False)
+    assert_equal(validator(stream("bar", False, True)), True)
 
     validator = common.stream_validator(Settings(included_streams=["foo", "bar"]))
-    assert_equal(validator("foo"), True)
-    assert_equal(validator("bar"), True)
-    assert_equal(validator("baz"), False)
+    assert_equal(validator(stream("foo", True, True)), True)
+    assert_equal(validator(stream("bar", True, True)), True)
+    assert_equal(validator(stream("baz", True, True)), False)
 
+    # Test exclude.
     validator = common.stream_validator(
         Settings(included_streams=["*"], excluded_streams=["bad", "worse"])
     )
-    assert_equal(validator("good"), True)
-    assert_equal(validator("bad"), False)
-    assert_equal(validator("worse"), False)
+    assert_equal(validator(stream("good", True, True)), True)
+    assert_equal(validator(stream("bad", True, True)), False)
+    assert_equal(validator(stream("worse", True, True)), False)
 
     # edge case: excluded takes precedence over included
     validator = common.stream_validator(
         Settings(included_streams=["foo"], excluded_streams=["foo"])
     )
-    assert_equal(validator("foo"), False)
+    assert_equal(validator(stream("foo", False, False)), False)
+
+    validator = common.stream_validator(
+        Settings(included_streams=["baz"], excluded_streams=["bar"])
+    )
+    assert_equal(validator(stream("foo", True, True)), False)
+    assert_equal(validator(stream("bar", False, False)), False)
 
 
 if __name__ == "__main__":