release v0.4.0

docs/dev/architecture.md

@@ -1,52 +0,0 @@
-
-
-
-## Edge Case Debates
-
-* Is an email Inbox/conversation more like a collection or a feed?
-
-
-## Content System
-
-Content sources can be registered to handle IDs. The first to be registered for a pattern will be used.
-
-It should take into account things like connectivity, availability, etc.
-
-Live API, Offline cache, Archive sources.
-
-If the live API only has 7 days of data, we may fall back cached or exported archive.
-
-## Testing
-
-Unit and E2E tests run offline, except integration portions.
-
-We can use the cached responses and responses module to accomplish this.
-
-## Provider template
-
-Twitter is the furthest along, but each has some distinct features.
-
-I was learning Python as I built this project.
-
-## Extensibility
-
-We rely as much as possible on the constituent parts that we rely on.
-
-Trac is an inspiration but invents everything from scratch which is over-engineering for us at this point.
-
-We'll vet the pieces that we use well and rely on them. Trac for the HTTP routing and themes.
-
-Dataclasses for the serialization.
-
-https://stackoverflow.com/questions/7505988/importing-from-a-relative-path-in-python
-
-https://www.geeksforgeeks.org/absolute-and-relative-imports-in-python/
-
-https://stackoverflow.com/questions/40892104/how-to-organize-one-blueprint-with-multiple-routes-files
-
-### Content
-
-### Themes
-
-### Routes
-

docs/dev/syndication_taxonomy.md

@@ -1,61 +0,0 @@
-# Syndication Taxonomy
-
-These items are implemented as Dataclasses and are network serializable.
-
-They are designed as a ViewModel.
-
-The Hogumathi feed syndication will be the real test of whether this is a viable approach...
-
-Strictly the network should have a different ValueObject.
-
-"Actions" will be tricky. I think they're a map currently, and require regular expressions.
-
-## FeedItem
-
-Presently this is implemented as a ViewModel.
-
-## FeedItemMedia
-
-Media item attached to a FeedItem
-
-## User
-
-## Feed
-
-FeedItems - chronologically ordered
-
-## Collection
-
-CollectionItem - ordered by 'user'
-
-
-## CollectionItem
-
-(presently implemented as FeedItem. Could end up nesting FeedItem)
-
-Think of Search and Playlist as a user, maybe read only access
-
-## List
-
-ListMembers - ordered
-
-## ListMember
-
-User
-
-## Conversation
-
-Ordered collection of Messages
-
-## Message
-
-From User. (Participant?)
-
-To Users.
-
-FeedItems can be attached.
-
-
-## CollectionPage
-
-A page of ViewModel collection items with a next_token.

extensions/instagram_embed_facade.py

@@ -0,0 +1,51 @@
+from hogumathi_app import content_system as h_cs
+import hogumathi_app.view_model as h_vm
+
+def register_content_sources ():
+    h_cs.register_content_source('instagram:post:', get_post, id_pattern='([a-zA-Z0-9\_\-]+)')
+
+
+def get_post (post_id, is_embed=True):
+    html = embed_template(post_id)
+    post = h_vm.FeedItem(
+        id = post_id,
+        created_at = 'some time',
+        display_name = 'IG User',
+        handle = 'iguser',
+        
+        html = html
+    )
+
+    return post
+
+def embed_template (post_id):
+    """
+    Pretty sure I made a little demo that fetches oembed to get detailed info.
+    
+    May be mistaken. The endpoint requires an access token.
+    
+    Perhaps I fetched the iframe embed and scraped?
+    
+    https://developers.facebook.com/docs/instagram/oembed/
+    
+    Maybe a token is not required:
+    
+    https://developers.facebook.com/docs/instagram/oembed-legacy
+    
+    Gives 404.
+    
+    I am starting to remember using the iframe embed and messing with params in browser; no scraping yet.
+    
+// for scraping
+var postInfo = {
+    caption: document.querySelector(".Caption").innerText.split("\n\n")[1],
+    tags: Array.from(document.querySelectorAll(".Caption a")).map(a => a.innerText).slice(1), 
+    follower_count: document.querySelector(".FollowerCountText").innerText,
+    image_url: document.querySelector(".EmbeddedMediaImage").src,
+    avatar_src: document.querySelector(".Avatar img").src,
+    username: document.querySelector(".UsernameText").innerText,
+    source_url: document.querySelector(".EmbeddedMedia").href,
+    author_source_url: document.querySelector(".ViewProfileButton").href
+    }
+    """
+    return f'<iframe class="feed_item" width="100%" height="800" src="http://instagram.com/p/{post_id}/embed/captioned" frameborder="0"></iframe>'

extensions/twitter_archive_facade/__init__.py

@@ -0,0 +1 @@
+from .facade import twitter_app, register_content_sources

extensions/twitter_archive_facade.py → extensions/twitter_archive_facade/facade.py

@@ -1,3 +1,4 @@
+from dataclasses import asdict
 from typing import List
 from dacite import from_dict
 
@@ -21,9 +22,10 @@ import dateutil.tz
 import requests
 
 
+from twitter_v2 import types as tv2_types
 from twitter_v2.archive import ArchiveTweetSource
 
-from hogumathi_app.view_model import FeedItem, PublicMetrics
+from hogumathi_app import content_system, view_model as h_vm
 
 ARCHIVE_TWEETS_PATH=os.environ.get('ARCHIVE_TWEETS_PATH', '.data/tweets.json')
 TWEET_DB_PATH=os.environ.get('TWEET_DB_PATH', '.data/tweet.db')
@@ -257,13 +259,13 @@ def tweet_model (tweet_data):
     return t
 
 
-def tweet_model_vm (tweet_data) -> List[FeedItem]:
+def tweet_model_vm (tweet_data) -> List[h_vm.FeedItem]:
     # retweeted_by, avi_icon_url, display_name, handle, created_at, text
     
     """
     {"id": "797839193", "created_at": "2008-04-27T04:00:27", "retweeted": 0, "favorited": 0, "retweet_count": "0", "favorite_count": "0", "full_text": "Putting pizza on. Come over any time!", "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_screen_name": null, "author_id": "14520320"}, {"id": "797849979", "created_at": "2008-04-27T04:27:46", "retweeted": 0, "favorited": 0, "retweet_count": "0", "favorite_count": "0", "full_text": "hijacked!@!!!", "in_reply_to_status_id_str": null, "in_reply_to_user_id": null, "in_reply_to_screen_name": null, "author_id": "14520320"}
     """
-    t = FeedItem(
+    t = h_vm.FeedItem(
         id = tweet_data['id'],
         text = tweet_data['full_text'],
         created_at = tweet_data['created_at'],
@@ -276,16 +278,16 @@ def tweet_model_vm (tweet_data) -> List[FeedItem]:
         display_name = 'Archive User',
         handle = '!archive',
         
-        url = url_for('.get_tweet_html', tweet_id = tweet_data['id']),
+        url = url_for('twitter_archive_facade.get_tweet_html', tweet_id = tweet_data['id']),
         
-        author_url = url_for('.get_profile_html', user_id='0'),
+        author_url = url_for('twitter_archive_facade.get_profile_html', user_id='0'),
         author_id = '0',
         
         source_url = '!source_url',
         source_author_url = '!source_author_url',
         #'is_edited': len(tweet_data['edit_history_tweet_ids']) > 1
         
-        public_metrics = PublicMetrics(
+        public_metrics = h_vm.PublicMetrics(
             like_count = int(tweet_data['favorite_count']),
             retweet_count = int(tweet_data['retweet_count']),
             reply_count = 0,
@@ -295,13 +297,8 @@ def tweet_model_vm (tweet_data) -> List[FeedItem]:
     
     return t
     
-
-@twitter_app.route('/profile/<user_id>.html', methods=['GET'])
-def get_profile_html (user_id):
-
-    pagination_token = request.args.get('pagination_token')
-    #exclude_replies = request.args.get('exclude_replies', '1')
-
+    
+def get_user_feed (user_id, pagination_token=None, me=None):
     tweet_source = ArchiveTweetSource(ARCHIVE_TWEETS_PATH)
     
     db_tweets = tweet_source.get_user_timeline(author_id = user_id,
@@ -312,15 +309,63 @@ def get_profile_html (user_id):
 
     tweets = list(map(tweet_model_vm, db_tweets))
     next_token = db_tweets[-1]['id']
+    
+    collection_page = h_vm.CollectionPage(
+        id = user_id,
+        items = tweets,
+        next_token = next_token
+    )
+    
+    return collection_page
+
+def get_tweets (ids, me = None):
+    tweet_source = ArchiveTweetSource(ARCHIVE_TWEETS_PATH)
+    
+    db_tweets = tweet_source.get_tweets(ids)
+                                                    
+    tweets = list(map(tweet_model_vm, db_tweets))
+    
+    collection_page = h_vm.CollectionPage(
+        id = ','.join(ids),
+        items = tweets
+    )
+    
+    return collection_page
+
+def get_tweet (tweet_id, me = None):
+    ids = [tweet_id]
+    
+    collection_page = get_tweets(ids, me=me)
+    
+    if collection_page.items:
+        return collection_page.items[0]
 
 
+def register_content_sources ():
+    content_system.register_content_source('twitter:feed:user:', get_user_feed, id_pattern='([\d]+)')
+    
+    content_system.register_content_source('twitter:tweets', get_tweets, id_pattern='')
+    content_system.register_content_source('twitter:tweet:', get_tweet, id_pattern='([\d]+)')
+
+@twitter_app.route('/profile/<user_id>.html', methods=['GET'])
+def get_profile_html (user_id):
+    
+    pagination_token = request.args.get('pagination_token')
+    #exclude_replies = request.args.get('exclude_replies', '1')
+    
+    # FIXME we want to use a specific source here...
+    collection_page = content_system.get_content(f'twitter:feed:user:{user_id}', content_source_id = 'twitter_archive_facade.facade:get_user_feed', pagination_token=pagination_token)
+    
+    tweets = collection_page.items
+    next_token = collection_page.next_token
+    
     query = {}
     
     if next_token:
         query = {
             **query,
-            'next_data_url': url_for('.get_profile_html', user_id=user_id , pagination_token=next_token),
-            'next_page_url': url_for('.get_profile_html', user_id=user_id , pagination_token=next_token)
+            'next_data_url': url_for('twitter_archive_facade.get_profile_html', user_id=user_id , pagination_token=next_token),
+            'next_page_url': url_for('twitter_archive_facade.get_profile_html', user_id=user_id , pagination_token=next_token)
         }
     
     profile_user = {
@@ -344,15 +389,15 @@ def get_tweet_html (tweet_id = None):
     
     if not tweet_id:
         ids = request.args.get('ids').split(',')
+        
+        collection_page = content_system.get_content('twitter:tweets', ids=ids, content_source_id='twitter_archive_facade.facade:get_tweets')
+        
+        tweets = collection_page.items
     else:
-        ids = [tweet_id]
+        tweet = content_system.get_content(f'twitter:tweet:{tweet_id}', content_source_id='twitter_archive_facade.facade:get_tweet')
         
-    tweet_source = ArchiveTweetSource(ARCHIVE_TWEETS_PATH)
+        tweets = [tweet]
     
-    db_tweets = tweet_source.get_tweets(ids)
-                                                    
-
-    tweets = list(map(tweet_model_vm, db_tweets))
     query = {}
     profile_user = {}
     
@@ -528,12 +573,21 @@ def post_tweets ():
 
 def tweet_to_card (tweet, includes):
     
+    if type(tweet) == t_vm.FeedItem:
+        tweet = asdict(tweet)
+        
+    if type(includes) == tv2_types.TweetExpansions:
+        includes = t_vm.cleandict(asdict(includes))
+    
+    
     user = list(filter(lambda u: u.get('id') == tweet['author_id'], includes.get('users')))[0]
     
     tweet_url = 'https://twitter.com/{}/status/{}'.format(user['username'], tweet['id'])
     
     content = tweet['text'] + "\n\n[view tweet]({})".format(tweet_url)
     
+
+    
     card = {
       'id': 'tweet-' + tweet['id'],
       'content': content,
@@ -550,6 +604,9 @@ def tweet_to_card (tweet, includes):
 
 
 def response_to_cards (response_json, add_included = True):
+    """
+    Seems to be unused
+    """
     tweets = response_json.get('data')
     includes = response_json.get('includes')
 

extensions/twitter_v2_facade/__init__.py

@@ -1,7 +1,6 @@
-from . import content_source
-
+from .content_source import register_content_sources
 from .facade import twitter_app
 
+def init_app (app, url_prefix='/twitter'):
+    app.register_blueprint(twitter_app, url_prefix=url_prefix)
 
-def register_content_sources ():
-    content_source.register_content_sources()

extensions/twitter_v2_facade/test/unit/twitter_v2_facade_test/__init__.py

@@ -0,0 +1,4 @@
+import sys
+
+sys.path.append('lib')
+sys.path.append('extensions')

test/unit/twitter_v2_facade_test/test_view_model.py → extensions/twitter_v2_facade/test/unit/twitter_v2_facade_test/test_view_model.py

@@ -1,4 +1,7 @@
-import twitter_v2_facade as t2f
+import dacite
+import twitter_v2.types as tv2_types
+import twitter_v2_facade.view_model as vm
+
 
 """
 
@@ -11,14 +14,18 @@ def mock_url_for (route, *args, **kwargs):
 
 
 def test_tweet_model ():
+    """
+    This should be testing against twitter_v2 or tweet_source instead of an HTTP response. 
+    Very old code.
+    """
     tweet_api_resp = {
         'data': [{
-            'id': 12345,
+            'id': "12345",
             'text': 'A tweet!',
             'created_at': 'a time',
-            'author_id': 456,
+            'author_id': "456",
             
-            'conversation_id': 12345
+            'conversation_id': "12345"
             
         }
         
@@ -26,7 +33,7 @@ def test_tweet_model ():
         'includes': {
             'tweets': [],
             'users': [{
-                'id': 456,
+                'id': "456",
                 'username': 'user456',
                 'name': 'User 456',
                 'verified': False,
@@ -40,14 +47,14 @@ def test_tweet_model ():
         }
     }
     
-    includes = tweet_api_resp.get('includes')
-    tweet_data = tweet_api_resp.get('data')[0]
     
-    tweet = t2f.tweet_model (includes, tweet_data, 'TEST', my_url_for = mock_url_for)
+    tweet_response = dacite.from_dict(data_class=tv2_types.TweetSearchResponse, data=tweet_api_resp)
+    
+    tweet = vm.tweet_model_dc_vm (tweet_response.includes, tweet_response.data[0], 'TEST', my_url_for = mock_url_for, my_g={})
     
-    assert(tweet['id'] == 12345)
-    assert(tweet['text'] == 'A tweet!')
+    assert(tweet.id == "12345")
+    assert(tweet.text == 'A tweet!')
     
-    assert(tweet['author_id'] == 456)
-    assert(tweet['display_name'] == 'User 456')
+    assert(tweet.author_id == "456")
+    assert(tweet.display_name == 'User 456')
     

extensions/twitter_v2_facade/view_model.py

@@ -10,7 +10,7 @@ from . import oauth2_login
 
 url_for = oauth2_login.url_for_with_me
 
-def user_model_dc (user):
+def user_model_dc (user, my_url_for=url_for):
     
     fsu = FeedServiceUser(
         id = user.id,
@@ -24,7 +24,7 @@ def user_model_dc (user):
         is_protected = user.protected,
         location = user.location,
         
-        url = url_for('twitter_v2_facade.get_profile_html', user_id=user.id),
+        url = my_url_for('twitter_v2_facade.get_profile_html', user_id=user.id),
         source_url = f'https://twitter.com/{user.username}',
         
         raw_user = user
@@ -33,19 +33,19 @@ def user_model_dc (user):
     return fsu
 
 
-def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=url_for, reply_depth=0, expand_path=None) -> FeedItem:
+def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=url_for, my_g=g, reply_depth=0, expand_path=None) -> FeedItem:
     
     # retweeted_by, avi_icon_url, display_name, handle, created_at, text
     
     
     user = list(filter(lambda u: u.id == tweet.author_id, includes.users))[0]
     
-    published_by = user_model_dc(user)
+    published_by = user_model_dc(user, my_url_for=my_url_for)
     
     url = my_url_for('twitter_v2_facade.get_tweet_html', tweet_id=tweet.id, view='tweet')
     source_url = 'https://twitter.com/{}/status/{}'.format(user.username, tweet.id)
     
-    avi_icon_url = url_for('get_image', url=user.profile_image_url)
+    avi_icon_url = my_url_for('get_image', url=user.profile_image_url)
     
     retweet_of = None
     quoted = None
@@ -71,9 +71,9 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
     
     if reply_depth:
         vr = actions['view_replies']
-        url = url_for(vr.route, **vr.route_params)
+        url = my_url_for(vr.route, **vr.route_params)
     
-    if g.get('twitter_user'):
+    if my_g.get('twitter_user'):
         actions.update(
             bookmark = FeedItemAction('twitter_v2_facade.post_tweet_bookmark', {'tweet_id': tweet.id}),
             delete_bookmark = FeedItemAction('twitter_v2_facade.delete_tweet_bookmark', {'tweet_id': tweet.id}),
@@ -81,7 +81,7 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
             retweet = FeedItemAction('twitter_v2_facade.post_tweet_retweet', {'tweet_id': tweet.id})
             )
     
-    if g.twitter_live_enabled:
+    if my_g.get('twitter_live_enabled'):
         actions.update(
             view_activity = FeedItemAction('twitter_v2_live_facade.get_tweet_activity_html', {'tweet_id': tweet.id})
             )
@@ -136,8 +136,8 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
                 if url.images:
                     print(url.images)
                     card = replace(card, 
-                        preview_image_url = url_for('get_image', url=url.images[1].url),
-                        image_url = url_for('get_image', url=url.images[0].url)
+                        preview_image_url = my_url_for('get_image', url=url.images[1].url),
+                        image_url = my_url_for('get_image', url=url.images[0].url)
                         )
                 
                 t = replace(t, card = card)
@@ -147,7 +147,10 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
             reply_count = tweet.public_metrics.reply_count,
             quote_count = tweet.public_metrics.quote_count,
             retweet_count = tweet.public_metrics.retweet_count,
-            like_count = tweet.public_metrics.like_count
+            like_count = tweet.public_metrics.like_count,
+            impression_count = tweet.public_metrics.impression_count,
+            bookmark_count = tweet.public_metrics.bookmark_count,
+            
             )
         
         t = replace(t, public_metrics = public_metrics)
@@ -173,7 +176,7 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
             retweeted_tweet_id = retweet_of[0].id,
             source_retweeted_by_url = 'https://twitter.com/{}'.format(user.username),
             retweeted_by = user.name,
-            retweeted_by_url = url_for('.get_profile_html', user_id=user.id)
+            retweeted_by_url = my_url_for('.get_profile_html', user_id=user.id)
             )
     
     
@@ -196,8 +199,8 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
             photo_media = map(lambda p: MediaItem(
                 media_key = p.media_key,
                 type = 'photo',
-                preview_image_url = url_for('get_image', url=p.url + '?name=tiny&format=webp'),
-                url = url_for('get_image', url=p.url),
+                preview_image_url = my_url_for('get_image', url=p.url + '?name=tiny&format=webp'),
+                url = my_url_for('get_image', url=p.url),
                 width = p.width, 
                 height = p.height
                 ), photos)
@@ -223,7 +226,7 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
                         print(variants)
                     variant = variants[0]
                     
-                    url = url_for('get_image', url=variant.url)
+                    url = my_url_for('get_image', url=variant.url)
                     content_type = variant.content_type
                     size = int(v.duration_ms / 1000 * variant.bit_rate)
                 
@@ -236,8 +239,8 @@ def tweet_model_dc_vm (includes: TweetExpansions, tweet: Tweet, me, my_url_for=u
                 mi = MediaItem(
                     media_key = v.media_key,
                     type = 'video',
-                    preview_image_url = url_for('get_image', url=v.preview_image_url + '?name=tiny&format=webp'),
-                    image_url = url_for('get_image', url=v.preview_image_url),
+                    preview_image_url = my_url_for('get_image', url=v.preview_image_url + '?name=tiny&format=webp'),
+                    image_url = my_url_for('get_image', url=v.preview_image_url),
                     width = v.width,
                     height = v.height,
                     url=url,

hogumathi_app/__main__.py

@@ -2,15 +2,18 @@ import os
 import sys
 from importlib.util import find_spec
 from configparser import ConfigParser
-from hashlib import sha256
+
 import json
 
+
 import requests
 
 from flask import Flask, g, redirect, url_for, render_template, jsonify, request, send_from_directory
 from flask_cors import CORS
 
-from .content_system import get_content
+from .web import api
+
+from . import item_collections, view_model as h_vm
 from .item_collections import item_collections_bp
 
 theme_bootstrap5_enabled = False
@@ -19,7 +22,6 @@ if find_spec('theme_bootstrap5'):
     theme_bootstrap5_enabled = True
     
 if find_spec('twitter_v2_facade'):
-    # FIXME get_brand needs a refactor
     import twitter_v2_facade
     from twitter_v2_facade import twitter_app as twitter_v2
     from twitter_v2_facade import oauth2_login
@@ -30,11 +32,12 @@ else:
     twitter_enabled = False
 
 if find_spec('twitter_v2_live_facade'):
+    import twitter_v2_live_facade
     from twitter_v2_live_facade import twitter_app as twitter_v2_live
     
     
     import brands
-    from brands import brands_bp, get_brand
+    from brands import brands_bp
     
     twitter_live_enabled = True
 else:
@@ -42,6 +45,7 @@ else:
     twitter_live_enabled = False
 
 if find_spec('twitter_archive_facade'):
+    import twitter_archive_facade
     from twitter_archive_facade import twitter_app as twitter_archive
     archive_enabled = True
 else:
@@ -56,26 +60,41 @@ else:
     mastodon_enabled = False
 
 if find_spec('feeds_facade'):
+    import feeds_facade
     from feeds_facade import twitter_app as feeds
     feeds_enabled = True
 else:
     print('feeds module not found.')
     feeds_enabled = False
 
-if find_spec('youtube_facade'):
-    from youtube_facade import youtube_app as youtube
+if find_spec('youtube_v3_facade'):
+    import youtube_v3_facade
+    from youtube_v3_facade import youtube_app as youtube
     youtube_enabled = True
 else:
     print('youtube module not found.')
     youtube_enabled = False
 
-if find_spec('email_facade'):
-    from email_facade import messages_facade as messages
+if find_spec('messages_facade'):
+    from messages_facade import messages_facade as messages
     messages_enabled = True
 else:
     print('messages module not found.')
     messages_enabled = False
 
+if find_spec('instagram_embed_facade'):
+    import instagram_embed_facade
+    instagram_embed_enabled = True
+else:
+    print('instagram_embed module not found.')
+    instagram_embed_enabled = False
+
+if find_spec('instagram_facade'):
+    import instagram_facade
+    instagram_enabled = True
+else:
+    print('instagram module not found.')
+    instagram_enabled = False
 
 if find_spec('videojs'):
     from videojs import videojs_bp
@@ -92,6 +111,14 @@ else:
     visjs_enabled = False
 
 
+"""
+By default we use embed because it's less likely to get banned, because it does not
+use scraping. Once we've used scraping for a while and not been banned we can switch to that.
+
+Biz note: scraping could also be an early access feature.
+"""
+print('disabling instagram_facade to avoid scraping... enable at your own risk')
+instagram_enabled = False
 
 #messages_enabled = False
 #twitter_live_enabled = True
@@ -122,7 +149,7 @@ if __name__ == '__main__':
     if not os.path.exists('.data/cache'):
         os.mkdir('.data/cache')
     
-    api = Flask(__name__, static_url_path='')
+    
     
     
     # HACK - environ from .env isn't set yet when the import happens. We should call an init function somewhere.
@@ -165,6 +192,8 @@ if __name__ == '__main__':
         if messages_app_url:
             g.messages_app_url = messages_app_url
             
+         
+            
         if theme_bootstrap5_enabled:
             g.theme_bootstrap5_enabled = theme_bootstrap5_enabled
     
@@ -205,30 +234,14 @@ if __name__ == '__main__':
         return config
     
     
-    @api.context_processor
-    def add_nav_items_to_template_context ():
-        nav_items = []
-        
-        route_nav = g.get('route_nav')
-        
-        if route_nav:
-            nav_items += route_nav
-            
-        module_nav = g.get('module_nav')
-        
-        if module_nav:
-            nav_items += module_nav
-            
-        #nav_items.sort(key = lambda ni: ni['order'])
-        
-        return dict(
-            nav_items = nav_items
-        )
     
     api.secret_key = os.environ.get('FLASK_SECRET')
     
     api.config['TEMPLATES_AUTO_RELOAD'] = True
     
+    # the new way
+    #api.config['notes_app_url'] = notes_app_url
+    
     if theme_bootstrap5_enabled:
         api.register_blueprint(hogumathi_theme_bootstrap5_bp)
     
@@ -242,95 +255,47 @@ if __name__ == '__main__':
     api.register_blueprint(twitter_v2, url_prefix='/twitter')
     
     if archive_enabled:
+        twitter_archive_facade.register_content_sources()
         api.register_blueprint(twitter_archive, url_prefix='/twitter-archive')
     
     if mastodon_enabled:
         api.register_blueprint(mastodon, url_prefix='/mastodon')
         
-    if feeds_enabled:
-        if not os.path.exists('.data/cache/feeds'):
-            os.mkdir('.data/cache/feeds')
-        
-        api.register_blueprint(feeds, url_prefix='/feeds')
-    
+
     if youtube_enabled:
+        youtube_v3_facade.register_content_sources()
         api.register_blueprint(youtube, url_prefix='/youtube')
     
     if messages_enabled:
         api.register_blueprint(messages, url_prefix='/messages')
         
+    
+    item_collections.register_content_sources()
+    api.register_blueprint(item_collections_bp, url_prefix='/collections')
+    
     if twitter_live_enabled:
-        brands.register_content_sources()
-        
+        twitter_v2_live_facade.register_content_sources()
         api.register_blueprint(twitter_v2_live, url_prefix='/twitter-live')
-        api.register_blueprint(item_collections_bp, url_prefix='/collections')
-        api.register_blueprint(brands_bp, url_prefix='/')
         
-    #CORS(api)
+        brands.register_content_sources()
+        api.register_blueprint(brands_bp, url_prefix='/')
     
-    @api.get('/login.html')
-    def get_login_html ():
-        opengraph_info = dict(
-            type = 'webpage', # threads might be article
-            url = g.app_url,
-            title = 'Hogumathi',
-            description = 'An app for Twitter, Mastodon, YouTube, etc; Open Source.'
-        )
-        
-        return render_template('login.html', opengraph_info=opengraph_info)
+    if instagram_enabled:
+        instagram_facade.register_content_sources()
     
-    @api.get('/')
-    def index ():
-        return redirect(url_for('.get_login_html'))
+    if instagram_embed_enabled:
+        instagram_embed_facade.register_content_sources()
     
-    @api.get('/img')
-    def get_image ():
-        url = request.args['url']
-        url_hash = sha256(url.encode('utf-8')).hexdigest()
-        path = f'.data/cache/media/{url_hash}'
-        
-        if not os.path.exists(path):
-            resp = requests.get(url)
-            if resp.status_code >= 200 and resp.status_code < 300:
-                with open(path, 'wb') as f:
-                    f.write(resp.content)
-                with open(f'{path}.meta', 'w') as f:
-                    headers = dict(resp.headers)
-                    json.dump(headers, f)
-            else:
-                return 'not found.', 404
-                
-        with open(f'{path}.meta', 'r') as f: 
-            headers = json.load(f)
-        
-        #print(url)
-        #print(url_hash)
-        #print(headers)
-        
-        # not sure why some responses use lower case.
-        mimetype = headers.get('Content-Type') or headers.get('content-type')
+    if feeds_enabled:
+        if not os.path.exists('.data/cache/feeds'):
+            os.mkdir('.data/cache/feeds')
         
-        return send_from_directory('.data/cache/media', url_hash, mimetype=mimetype)
+        feeds_facade.register_content_sources()
+        api.register_blueprint(feeds, url_prefix='/feeds')
     
-    @api.get('/content/abc123.html')
-    def get_abc123_html ():
-
-        return 'abc123'
     
-    @api.get('/content/<content_id>.html')
-    def get_content_html (content_id, content_kwargs=None):
-        
-        if not content_kwargs:
-            content_kwargs = filter(lambda e: e[0].startswith('content:'), request.args.items())
-            content_kwargs = dict(map(lambda e: [e[0][len('content:'):], e[1]], content_kwargs))
-        
-        content = get_content(content_id, **content_kwargs)
-        
-        return jsonify(content)
+    #CORS(api)
     
-    @api.get('/content/def456.html')
-    def get_def456_html ():
 
-        return get_content_html('brand:ispoogedaily')
     
     api.run(port=PORT, host=HOST)

hogumathi_app/content_system.py

@@ -26,138 +26,222 @@ https://www.bonobo-project.org/
 import re
 import inspect
 
-content_sources = {}
 
-hooks = {}
 
-def register_content_source (id_prefix, content_source_fn, id_pattern='(\d+)', source_id=None):
-    if not source_id:
-        source_id=f'{inspect.getmodule(content_source_fn).__name__}:{content_source_fn.__name__}'
-    
-    print(f'register_content_source: {id_prefix}: {source_id} with ID pattern {id_pattern}')
+class ContentSystem:
+    def __init__ (self):
+        self.content_sources = {}
+        self.hooks = {}
+        
+    def register_content_source (self, id_prefix, content_source_fn, id_pattern='(\d+)', source_id=None, weight=None):
+        if not source_id:
+            source_id=f'{inspect.getmodule(content_source_fn).__name__}:{content_source_fn.__name__}'
+        
+        if weight == None:
+            weight = 1000 - len(self.content_sources)
+        
+        print(f'register_content_source: {id_prefix}: {source_id} with ID pattern {id_pattern} (weight={weight})')
 
-    content_sources[ id_prefix ] = [content_source_fn, id_pattern, source_id]
+        self.content_sources[ source_id ] = [id_prefix, content_source_fn, id_pattern, source_id, weight]
 
 
-def find_content_id_args (id_pattern, content_id):
-    id_args = re.fullmatch(id_pattern, content_id)
-    if not id_args:
-        return [], {}
-    
-    args = []
-    kwargs = id_args.groupdict()
-    if not kwargs:
-        args = id_args.groups()
-    
-    return args, kwargs
-    
-def get_content (content_id, *extra_args, **extra_kwargs):
-    id_prefixes = list(content_sources.keys())
-    id_prefixes.sort(key=lambda id_prefix: len(id_prefix), reverse=True)
-    
-    for id_prefix in id_prefixes:
-        [content_source_fn, id_pattern, source_id] = content_sources[ id_prefix ]
-        if not content_id.startswith(id_prefix):
-            continue
+    def find_content_id_args (self, id_pattern, content_id):
+        id_args = re.fullmatch(id_pattern, content_id)
+        if not id_args:
+            return [], {}
+        
+        args = []
+        kwargs = id_args.groupdict()
+        if not kwargs:
+            args = id_args.groups()
+        
+        return args, kwargs
+        
+    def get_content (self, content_id, content_source_id=None, *extra_args, **extra_kwargs):
+        print(f'get_content {content_id}')
+        #source_ids = list(self.content_sources.keys())
+        #source_ids.sort(key=lambda id_prefix: len(id_prefix), reverse=True)
+        
+        source_ids = list(self.content_sources.keys())
+        source_ids.sort(key=lambda id_prefix: self.content_sources[id_prefix][4], reverse=True) # 4 = weight
+        
+        #print(source_ids)
+        
+        for source_id in source_ids:
+            if content_source_id and source_id != content_source_id:
+                continue
+            
+            [id_prefix, content_source_fn, id_pattern, source_id, weight] = self.content_sources[ source_id ]
+            if not content_id.startswith(id_prefix):
+                continue
+            
+            source_content_id = content_id[len(id_prefix):]
+            
+            print(f'get_content {content_id} from source {source_id}, resolves to {source_content_id} ( weight={weight})')
+            
+            args, kwargs = self.find_content_id_args(id_pattern, source_content_id)
+            
+            if id_prefix.endswith(':') and not args and not kwargs:
+                continue
+            
+            if extra_args:
+                args += extra_args
+            
+            if extra_kwargs:
+                kwargs = {**extra_kwargs, **kwargs}
+            
+            # if we're calling a bulk source and only get back partial results...
+            # we'd want to remove the found content IDs and merge until
+            # we find them all...
+            # yet we don't want intelligence about the type of content returned.
+            # idea: class BulkResponse(dict): pass
+            content = content_source_fn(*args, **kwargs)
+            
+            if content:
+                self.invoke_hooks('got_content', content_id, content)
+                
+                return content
+
+    def get_all_content (self, content_ids):
+        """
+        Get content from all sources, using a grouping call if possible.
+        
+        Returns a map of source_id to to result; the caller needs
+        to have the intelligence to merge and paginate.
+        
+        Native implementation is to juse make one call to get_content per ID,
+        but we need to figure out a way to pass a list of IDs and pagination
+        per source; for exampe a list of 100+ Tweet IDs and 100+ YT videos
+        from a Swipe file.
+        """
+        
+        return self.get_all_content2(content_ids)
         
-        source_content_id = content_id[len(id_prefix):]
+    def get_all_content2 (self, content_collection_ids, content_args = None, max_results = None):
+        """
+        Takes a list of collection IDs and content_args is a map of (args, kwargs) keyed by collection ID.
         
-        print(f'get_content {content_id} from source {source_id}, resolves to {source_content_id}')
+        We could just use keys from content_args with empty values but that's a little confusing.
         
-        args, kwargs = find_content_id_args(id_pattern, source_content_id)
+        Interleaving the next page of a source into existing results is a problem.
         
-        if id_prefix.endswith(':') and not args and not kwargs:
-            continue
+        Gracefully degraded could simply get the next page at the end of all pages and then
+        view older content.
         
-        if extra_args:
-            args += extra_args
+        We also need intelligence about content types, meaning perhaps some lambdas pass in.
+        Eg. CollectionPage.
         
-        if extra_kwargs:
-            kwargs = {**extra_kwargs, **kwargs}
+        See feeds facade for an example of merging one page.
         
-        content = content_source_fn(*args, **kwargs)
+        Seems like keeping feed items in a DB is becoming the way to go, serving things in order.
         
-        if content:
-            invoke_hooks('got_content', content_id, content)
+        Client side content merging might work to insert nodes above, eg. HTMx.
+        
+        Might be jarring to reader, so make optional. Append all new or merge.
+        
+        Cache feed between requests on disk, merge in memory, send merge/append result.
+        
+        """
+        
+        bulk_prefixes = {
+            #'twitter:tweet:': 'twitter:tweets',
+            #'youtube:video:': 'youtube:videos',
+        }
+        bulk_requests = {}
+        
+        result = {}
+        
+        for content_id in content_collection_ids:
             
-            return content
-
-def get_all_content (content_ids):
-    """
-    Get content from all sources, using a grouping call if possible.
-    
-    Returns a map of source_id to to result; the caller needs
-    to have the intelligence to merge and paginate.
-    
-    Native implementation is to juse make one call to get_content per ID,
-    but we need to figure out a way to pass a list of IDs and pagination
-    per source; for exampe a list of 100+ Tweet IDs and 100+ YT videos
-    from a Swipe file.
-    """
-    
-    return get_all_content2(content_ids)
-    
-def get_all_content2 (content_collection_ids, content_args = None, max_results = None):
-    """
-    Takes a list of collection IDs and content_args is a map of (args, kwargs) keyed by collection ID.
-    
-    We could just use keys from content_args with empty values but that's a little confusing.
-    
-    Interleaving the next page of a source into existing results is a problem.
-    
-    Gracefully degraded could simply get the next page at the end of all pages and then
-    view older content.
-    
-    We also need intelligence about content types, meaning perhaps some lambdas pass in.
-    Eg. CollectionPage.
-    
-    See feeds facade for an example of merging one page.
-    
-    Seems like keeping feed items in a DB is becoming the way to go, serving things in order.
-    
-    Client side content merging might work to insert nodes above, eg. HTMx.
-    
-    Might be jarring to reader, so make optional. Append all new or merge.
-    
-    Cache feed between requests on disk, merge in memory, send merge/append result.
-    
-    """
-    
-    result = {}
-    
-    for content_id in content_collection_ids:
-        if content_args and content_id in content_args:
-            extra_args, extra_kwargs = content_args[content_id]
+            is_bulk = False
+            for bulk_prefix in bulk_prefixes:
+                if content_id.startswith(bulk_prefix):
+                    bulk_content_id = bulk_prefixes[ bulk_prefix ]
+                    if not bulk_content_id in bulk_requests:
+                        bulk_requests[ bulk_content_id ] = []
+                    bulk_requests[ bulk_content_id ].append(content_id)
+                    
+                    # max size for a content source...
+                    
+                    is_bulk = True
+                    
+            if is_bulk:
+                continue
             
-        result[ content_id ] = get_content(content_id, *extra_args, **extra_kwargs)
-    
-    return result
+            if content_args and content_id in content_args:
+                extra_args, extra_kwargs = content_args[content_id]
+            else:
+                extra_args, extra_kwargs = [], {}
+            
+            result[ content_id ] = self.get_content(content_id, *extra_args, **extra_kwargs)
+        
+        for bulk_content_id, content_ids in bulk_requests.items():
+            print(f'bulk: {bulk_content_id}, content_ids: {content_ids}')
+            
+            
+            
+            bulk_response = self.get_content(bulk_content_id, content_ids=content_ids) # FIXME me=... workaround, provide bulk id in args map
+            
+            print(f'bulk_response: {bulk_response}')
+            
+            # we're not supposed to be smart about get_content response type...
+            # does it return a map by convention? better than iterating something else.
+            if bulk_response:
+                for content_id, content in bulk_response.items():
+                    if content:
+                        self.invoke_hooks('got_content', content_id, content)
+                
+                result.update(bulk_response)
+        
+        return result
 
-def register_hook (hook_type, hook_fn, *extra_args, **extra_kwargs):
-    if not hook_type in hooks:
-        hooks[hook_type] = []
-    
-    hooks[hook_type].append([hook_fn, extra_args, extra_kwargs])
+    def register_hook (self, hook_type, hook_fn, *extra_args, **extra_kwargs):
+        if not hook_type in self.hooks:
+            self.hooks[hook_type] = []
+        
+        self.hooks[hook_type].append([hook_fn, extra_args, extra_kwargs])
 
 
-def invoke_hooks (hook_type, *args, **kwargs):
-    if not hook_type in hooks:
-        return
+    def invoke_hooks (self, hook_type, *args, **kwargs):
+        if not hook_type in self.hooks:
+            return
+        
+        for hook, extra_args, extra_kwargs in self.hooks[hook_type]:
+            hook_args = args
+            hook_kwargs = kwargs
+            if extra_args:
+                hook_args = args + extra_args
+            if extra_kwargs:
+                hook_kwargs = {**extra_kwargs, **hook_kwargs}
+                
+            hook(*hook_args, **hook_kwargs)
+            
+            #try:
+            #    hook(*args, **kwargs)
+            #except TypeError as e:
+            #    print ('tried to call a hook with wrong args. no problem')
+            #    continue
+                
+# The app was coded before we turned this into a class...
+# so we proxy calls with the old interface to this default instance.
+DEFAULT = ContentSystem()
+
+def reset ():
+    print ('compat resetting content system')
+    DEFAULT = ContentSystem()
+
+def register_content_source (id_prefix, content_source_fn, id_pattern='(\d+)', source_id=None, weight=None):
+    print('compat register_content_source')
+    return DEFAULT.register_content_source(id_prefix, content_source_fn, id_pattern, source_id)
+    
+def get_content (content_id, content_source_id=None, *extra_args, **extra_kwargs):
+    print('compat get_content')
+    return DEFAULT.get_content(content_id, content_source_id, *extra_args, **extra_kwargs)
+    
+def get_all_content (content_ids):
+    print('compat get_all_content')
+    return DEFAULT.get_all_content(content_ids)
     
-    for hook, extra_args, extra_kwargs in hooks[hook_type]:
-        hook_args = args
-        hook_kwargs = kwargs
-        if extra_args:
-            hook_args = args + extra_args
-        if extra_kwargs:
-            hook_kwargs = {**extra_kwargs, **hook_kwargs}
-            
-        hook(*hook_args, **hook_kwargs)
-        
-        #try:
-        #    hook(*args, **kwargs)
-        #except TypeError as e:
-        #    print ('tried to call a hook with wrong args. no problem')
-        #    continue
-            
-            
+def register_hook (hook_type, hook_fn, *extra_args, **extra_kwargs):
+    print('compat register_hook')
+    return DEFAULT.register_hook(hook_type, hook_fn, *extra_args, **extra_kwargs)

hogumathi_app/item_collections.py

@@ -1,3 +1,4 @@
+from ulid import ULID
 from dataclasses import asdict, replace
 
 from importlib.util import find_spec
@@ -9,9 +10,9 @@ import json
 from flask import request, g, jsonify, render_template,  Blueprint, url_for, session
 
 from twitter_v2.api import ApiV2TweetSource
-from .view_model import FeedItem, cleandict
+from .view_model import FeedItem, CollectionPage, cleandict
 
-from .content_system import get_all_content, register_content_source
+from .content_system import get_content, get_all_content, register_content_source
 
 twitter_enabled = False
 if find_spec('twitter_v2_facade'):
@@ -19,8 +20,8 @@ if find_spec('twitter_v2_facade'):
     twitter_enabled = True
     
 youtube_enabled = False
-if find_spec('youtube_facade'):
-    from youtube_facade import youtube_model, get_youtube_builder
+if find_spec('youtube_v3_facade'):
+    from youtube_v3_facade.content_source import youtube_model, get_youtube_builder
     youtube_enabled = True
 
 DATA_DIR=".data"
@@ -32,7 +33,11 @@ item_collections_bp = Blueprint('item_collections', 'item_collections',
 
 
 def get_tweet_collection (collection_id):
-    with open(f'{DATA_DIR}/collection/{collection_id}.json', 'rt', encoding='utf-8') as f:
+    json_path = f'{DATA_DIR}/collection/{collection_id}.json'
+    if not os.path.exists(json_path):
+        return
+        
+    with open(json_path, 'rt', encoding='utf-8') as f:
         collection = json.loads(f.read())
         
     return collection
@@ -195,9 +200,8 @@ def get_collection_html (collection_id):
                 query['next_page_url'] = url_for('.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
             return render_template('tweet-collection.html', tweets = feed_items, user = {}, query = query)
 
-# pagination token is the next tweet_ID
-@item_collections_bp.get('/collections.html')
-def get_collections_html ():
+
+def get_collection_list (me = None):
     me = request.args.get('me')
     acct = session.get(me)
     
@@ -213,17 +217,99 @@ def get_collections_html ():
                     continue
                     
                 collection_id = collection_file.name[:-len('.json')]
+                version = coll.get('_version')
+                
+                if not version: # legacy
+                    version = str(ULID())
                 
                 coll_info = dict(
-                    collection_id = collection_id,
+                    id = collection_id,
+                    _version = version,
                     href = url_for('.get_collection_html', collection_id=collection_id)
                 )
                 
                 collections.append(coll_info)
+                
+    return collections
+
+# pagination token is the next tweet_ID
+@item_collections_bp.get('/collections.html')
+def get_collections_html ():
+    me = request.args.get('me')
     
+    collections = get_content('collections:list', me=me)
+            
     return jsonify(collections)
 
 
+def update_collection (collection_id, new_collection, op='replace', version=None, me = None):
+    path = f'.data/collection/{collection_id}.json'
+    
+    existing_collection = None
+    if os.path.exists(path):
+        with open(path, 'rt', encoding='utf-8') as f:
+            existing_collection = json.load(f)
+    
+    existing_version = existing_collection and existing_collection.get('_version')
+    if existing_collection and existing_version != version:
+        raise Error('updating with a wrong version. probably using a stale copy. fetch and retry op.')
+    
+    
+    if op == 'insert':
+        after_id = request.form.get('after_id')
+        raise Error('not supported yet')
+    
+    elif op == 'append':
+        existing_collection['items'] += new_collection['items']
+        new_collection = existing_collection
+    
+    elif op == 'prepend':
+        existing_collection['items'] = new_collection['items'] + existing_collection['items']
+        new_collection = existing_collection
+        
+    
+    new_version = str(ULID()) # content addressable hash of json w/o this key or similar
+    new_collection['_version'] = new_version
+    
+    with open(path, 'wt', encoding='utf-8') as f:
+        json.dump(new_collection, f)
+    
+    return new_version
+
+@item_collections_bp.post('/collection/<collection_id>.html')
+def post_collection_html (collection_id):
+    
+    
+    op = request.form.get('op', 'replace')
+    version = request.form.get('version')
+    
+    new_collection = request.form.get('collection.json')
+    new_collection = json.loads(new_collection) # FIXME probably wrong
+    
+    new_version = get_content('collection:update', collection_id, new_collection, op=op, me=me)
+    
+    return jsonify({'_version': new_version})
+
+
+@item_collections_bp.get('/collection/test-update/<collection_id>.html')
+def get_collection_test_update_html (collection_id):
+    
+    me = None
+    op = 'prepend'
+    version = request.args.get('version')
+    
+    new_collection = {
+        'items': [{'id': 'zzz999'}]
+    }
+    
+    new_version = get_content(f'collections:update:{collection_id}', 
+        new_collection, 
+        version=version, 
+        op=op,
+        me=me)
+    
+    return jsonify({'_version': new_version})
+
 @item_collections_bp.post('/data/collection/create/from-cards')
 def post_data_collection_create_from_cards ():
     """
@@ -270,54 +356,88 @@ def post_data_collection_create_from_cards ():
     
     return jsonify(collection)
 
-
-
-def expand_item2 (item, me, tweet_contents = None, includes = None, youtube_contents = None):
-    if 'id' in item:
-        tweets_response = tweet_contents[ 'twitter:tweet:' + item['id'] ]
-        tweets = tweets_response.items
+def get_item_id (obj_or_dict, id_key='id'):
+    if type(obj_or_dict) == dict:
+        return obj_or_dict[id_key]
+    else:
+        return getattr(obj_or_dict, id_key)
+
+def expand_item2 (item, me, content_responses):
+
+    content_id = item['id']
+
+    content_response = content_responses[ content_id ]
+    
+    if type(content_response) == CollectionPage:
+        tweets = content_response.items
+    elif type(content_response) == list:
+        tweets = content_response
+    else:        
+        tweets = [content_response]
+    
+    # endswith is a hack. Really FeedItems should return a full ID with prefix.
+    t = list(filter(lambda t: content_id.endswith(f':{get_item_id(t)}'), tweets))
+    
+    if not len(t):
+        print("no tweet for item: " + item['id'])
+        feed_item = FeedItem(
+            id = item['id'],
+            text = "(Deleted, suspended or blocked)",
+            created_at = "",
+            handle = "error",
+            display_name = "Error"
+        )
+        # FIXME 1) put this in relative order to the collection
+        # FIXME 2) we can use the tweet link to get the user ID...
         
-        t = list(filter(lambda t: item['id'] == t.id, tweets))
+    else:
+        feed_item = t[0]
+
+        note = item.get('note')
         
-        if not len(t):
-            print("no tweet for item: " + item['id'])
-            feed_item = FeedItem(
-                id = item['id'],
-                text = "(Deleted, suspended or blocked)",
-                created_at = "",
-                handle = "error",
-                display_name = "Error"
-            )
-            # FIXME 1) put this in relative order to the collection
-            # FIXME 2) we can use the tweet link to get the user ID...
-            
+        if type(feed_item) == dict:
+            feed_item.update(note = note)
         else:
-            feed_item = t[0]
-
-            note = item.get('note')
             feed_item = replace(feed_item, note = note)
-            
-    elif 'yt_id' in item:
-        yt_id = item['yt_id']
         
-        yt_videos = youtube_contents[ 'youtube:video:' + yt_id ]
-        
-        feed_item = list(filter(lambda v: v['id'] == yt_id, yt_videos))[0]
-        
-        note = item.get('note')
-        feed_item.update({'note': note})
-    
+
         
     
     return feed_item
 
 
-def get_collection (collection_id, pagination_token=None, max_results=10):
+def get_collection (collection_id, me=None, pagination_token:str = None, max_results:int =10):
     collection = get_tweet_collection(collection_id)
     
+    if not collection:
+        return
+    
+    first_idx = int(pagination_token or 0)
+    last_idx = first_idx + max_results
+    
+    
+    items = collection['items'][first_idx:last_idx]
+    content_ids = list(map(lambda item: item['id'], items))
+    
+    content_responses = get_all_content( content_ids )
+    
+    feed_items = list(map(lambda item: expand_item2(item, me, content_responses), items))
+    
+    collection['items'] = feed_items
+    
+    if len(collection['items']) == max_results:
+        collection['next_token'] = str(last_idx)
+    
     return collection
 
-register_content_source("collection:", get_collection, id_pattern="([^:]+)")
+
+
+def register_content_sources ():
+    register_content_source("collection:", get_collection, id_pattern="([^:]+)")
+    register_content_source("collections:list", get_collection_list, id_pattern="")
+    register_content_source("collections:update:", update_collection, id_pattern="([A-Za-z0-9\-\_\.]+)")
+
+
 
 # pagination token is the next tweet_ID
 @item_collections_bp.get('/collection2/<collection_id>.html')
@@ -325,38 +445,26 @@ def get_collection2_html (collection_id):
     me = request.args.get('me')
     acct = session.get(me)
     
-    max_results = int(request.args.get('max_results', 10))
+    max_results = int(request.args.get('limit', 1))
     
-    pagination_token = int(request.args.get('pagination_token', 0))
+    pagination_token = request.args.get('pagination_token', 0)
     
     #collection = get_tweet_collection(collection_id)
     collection = get_content(f'collection:{collection_id}',
+        me=me,
         pagination_token=pagination_token,
         max_results=max_results)
     
     if 'authorized_users' in collection and (not acct or not me in collection['authorized_users']):
         return 'access denied.', 403
     
-    items = collection['items'][pagination_token:(pagination_token + max_results)]
+    feed_items = collection['items']
+    pagination_token = collection.get('next_token')
     
-    if not len(items):
+    if not len(feed_items):
         return 'no tweets', 404
     
-    tweet_ids = filter(lambda i: 'id' in i, items)
-    tweet_ids = list(map(lambda item: 'twitter:tweet:' + item['id'], tweet_ids))
-    
-    tweet_contents = get_all_content( tweet_ids )
-    
-    
-    yt_ids = filter(lambda i: 'yt_id' in i, items)
-    yt_ids = list(map(lambda item: 'youtube:video:' + item['yt_id'], yt_ids))
     
-    youtube_contents = get_all_content( yt_ids )
-    
-    
-    includes = None
-    
-    feed_items = list(map(lambda item: expand_item2(item, me, tweet_contents, includes, youtube_contents), items))
     
     if request.args.get('format') == 'json':
         return jsonify({'ids': tweet_ids,
@@ -368,11 +476,12 @@ def get_collection2_html (collection_id):
         query = {}
         
         if pagination_token:
-            query['next_data_url'] = url_for('.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
+            query['next_data_url'] = url_for('.get_collection2_html', collection_id=collection_id, pagination_token=pagination_token, limit=max_results, me=me)
+            query['next_page_url'] = url_for('.get_collection2_html', collection_id=collection_id, pagination_token=pagination_token, limit=max_results, me=me)
         
         if 'HX-Request' in request.headers:
             return render_template('partial/tweets-timeline.html', tweets = feed_items, user = {}, query = query)
         else:
             if pagination_token:
-                query['next_page_url'] = url_for('.get_collection_html', collection_id=collection_id, pagination_token=pagination_token)
+                query['next_page_url'] = url_for('.get_collection2_html', me=me, collection_id=collection_id, pagination_token=pagination_token)
             return render_template('tweet-collection.html', tweets = feed_items, user = {}, query = query)

hogumathi_app/templates/brand-page.html

@@ -1,24 +0,0 @@
-{% extends "base-bs.html" %}
-
-{% block head %}
-	<title>{{ brand.display_name }} on Hogumathi</title>
-
-{% endblock %}
-
-{% block content %}
-	<div class="w-100" style="height: 80px;">
-		{% include "partial/page-nav-bs.html" %}
-	</div>
-	
-	<nav class="nav nav-pills flex-column flex-sm-row">
-		<a class="flex-sm-fill text-sm-center nav-link active" aria-current="page" href="#">Products &amp; Socials</a>
-		<a class="flex-sm-fill text-sm-center nav-link" href="#">Feeds</a>
-		<a class="flex-sm-fill text-sm-center nav-link" href="#">Partners</a>
-		<a class="flex-sm-fill text-sm-center nav-link disabled">Analytics</a>
-	</nav>
-	
-	{% if brand %}
-	{% include "partial/brand-info-bs.html" %}
-	{% endif %}
-	
-{% endblock %}

hogumathi_app/templates/conversations.html

@@ -1,14 +0,0 @@
-{% extends "base.html" %}
-
-{% block head %}
-	<title>Messages: {{ user.id }}</title>
-{% endblock %}
-
-
-{% block content %}
-<code><pre>
-
-{{ dm_events | tojson(indent=4) | safe }}
-
-</pre></code>
-{% endblock %}

hogumathi_app/templates/login.html

@@ -31,13 +31,13 @@
 	<li><form method="POST" action="{{ url_for('messages_facade.post_api_login') }}">
 		<button type="submit">Email</button> <input name="host" placeholder="smtp.example.com">
 		<input name="user" placeholder="me@example.com">
-		<input name="pass" type="password" placeholder="pa$$word123">
+		<input name="password" type="password" placeholder="pa$$word123">
 		</form>
 		<p>This will use IMap and SMTP over SSL.</p>
 		</li>
 {% endif %}
 {% if youtube_enabled %}
-	<li><a href="{{ url_for('youtube_facade.get_api_login') }}">YouTube</a></li>
+	<li><a href="{{ url_for('youtube_v3_facade.get_api_login') }}">YouTube</a></li>
 {% endif %}
 
 </ul>

hogumathi_app/templates/partial/brand-info-bs.html

@@ -1,367 +0,0 @@
-<div class="container-fluid w-100 ">
-
-<style>
-
-.brand-info ul {
-	list-style: none;
-	padding-left: 0;
-}
-
-.brand-info ul li {
-	padding-left: 0;
-}
-
-.brand-info ul li a.btn {
-	text-align: left;
-}
-
-.brand-info ul li .form-control {
-	overflow-wrap: break-word;
-}
-
-@media (min-width: 992px) {
-	.ms-lg-35 {
-		margin-left: 35% !important;
-	}
-}
-
-
-
-</style>
-
-<div class="brand-info w-100 p-2 bg-body border-black">
-
-<div class="bg-body border-bottom border-dark sticky-lg-top mb-5 mb-lg-0" style="z-index: 9999">
-{% if 'logo_href' in brand %}
-
-<center>
-<img class="m-4 w-25 w-lg-50" src="{{ brand.logo_href }}" style="float: left">
-
-</center>
-
-{% endif %}
-
-
-<h1>{{ brand.display_name }}</h1>
-
-
-
-{% if brand.description %}
-{% for para in brand.description.split('\n') %}
-<p>{{ para }}</p>
-{% endfor %}
-{% endif %}
-
-</div>
-
-<div class="ms-0 ms-lg-35">
-{% if gumroad and gumroad.products %}
-
-	<h2>Products</h2>
-	
-	
-	<div class="row row-cols-2 row-cols-lg-3 g-2 mb-3 mt-1">
-		{% for product in gumroad.products %}
-		<div class="col-sm">
-			<div class="card">
-				<img src="{{ product.cover_image_uri }}" class="card-img-top">
-				<h5 class="card-title">{{ product.name }}</h5>
-				<p class="card-text">({{ product.price }})</p>
-				
-				<a href="{{ product.href }}" class="btn btn-primary">
-					<i class="bi-box2-heart"></i>
-					On Gumroad
-				</a>
-				
-			</div>
-		</div>
-		{% endfor %}
-	</div>
-
-	
-
-
-{% endif %}
-
-
-
-
-{% if patreon or (gumroad and gumroad.subscriptions and len(gumroad.subscriptions)) %}
-	<h2>Subscriptions</h2>
-
-	<ul>
-
-	{% if patreon %}
-	<li>
-		<div class="input-group mb-1" style="width: fit-content">
-			<a class="btn btn-primary" href="{{ patreon.href }}">
-				<i class="bi-calendar-heart"></i>
-				Patreon
-			</a>
-			<span class="form-control px-4" >
-				@{{ patreon.id }}
-			</span>
-		</div>
-	</li>
-	{% endif %}
-
-	{% for sub in gumroad.subscriptions %}
-	{% if sub.tiers  %}
-	<li>
-		<h4>{{ sub.name }} on Gumroad</h4>
-		<ul>
-		{% for tier in sub.tiers %}
-		<li>
-			
-			<div class="input-group mb-1" style="width: fit-content">
-				<a class="btn btn-primary" href="{{ sub.href }}?option={{ tier.option }}">
-					<i class="bi-calendar-heart"></i>
-					{{ tier.name }}
-				</a>
-				<span class="form-control px-4" >
-					{{ tier.yearly }}/yr or {{ tier.monthly }}/mo
-				</span>
-			</div>
-		
-			<!--
-			<a href="{{ sub.href }}?option={{ tier.option }}">
-			{{ tier.name }}
-			</a>
-			({{ tier.yearly }}/yr or {{ tier.monthly }}/mo)
-			-->
-		</li>
-		{% endfor %}
-		</ul>
-	</li>
-	{% else %}
-	<li>
-		
-		<div class="input-group mb-1" style="width: fit-content">
-			<a class="btn btn-primary" href="{{ sub.href }}">
-				<i class="bi-calendar-heart"></i>
-				{{ sub.name }}
-			</a>
-			<span class="form-control px-4" >
-				{{ sub.yearly }}/yr or {{ sub.monthly }}/mo
-			</span>
-		</div>
-	</li>
-	{% endif %}
-	{% endfor %}
-
-	</ul>
-{% endif %}
-
-
-{% if venmo %}
-<h2>Tips</h2>
-
-<ul>
-
-{% if venmo %}
-<li>
-	<div class="input-group" style="width: fit-content">
-		<a class="btn btn-primary" href="{{ venmo.href }}">
-			<i class="bi-heart"></i>
-			Venmo
-		</a>
-		<span class="form-control" >
-			@{{ venmo.id }}
-		</span>
-	</div>
-</li>
-{% endif %}
-
-</ul>
-{% endif %}
-
-
-{% if twitter or youtube or mastodon or receipt_tracker %}
-	<h2>On Hogumathi</h2>
-
-	<ul>
-
-	{% if twitter %}
-	<li>
-		<div class="input-group mb-1" style="width: fit-content">
-		
-			<a href="{{ twitter.hogu_href }}" class="btn btn-primary">
-				<i class="bi-twitter"></i>
-				<span class="d-none d-md-inline">
-					Twitter
-				</span>
-			</a>
-			<span class="form-control">
-				@{{ twitter.username }}
-				
-			</span>
-		</div>
-		<!--
-		<small class="d-none d-md-inline">
-			(<a class="text-decoration-none" href="{{ twitter.href }}">on Twitter.com</a>)
-		</small>
-		[<a href="{{ twitter.href }}">source</a>]
-		-->
-	</li>
-	{% endif %}
-
-	{% if youtube %}
-	<li>
-		<div class="input-group mb-1" style="width: fit-content">
-		
-			<a href="{{ youtube.hogu_href }}" class="btn btn-primary">
-				<i class="bi-youtube"></i>
-				<span class="d-none d-md-inline">
-					YouTube
-				</span>
-			</a>
-			<span class="form-control">
-				@{{ youtube.username }}
-			</span>
-		</div>
-	
-		<!--
-		<a href="{{ youtube.hogu_href }}">YouTube</a> (@{{ youtube.username }}) [<a href="{{ youtube.href }}">source</a>]
-		-->
-	</li>
-	{% endif %}
-
-	{% if mastodon %}
-	<li>
-		<div class="input-group mb-1" style="width: fit-content">
-		
-			<a href="{{ mastodon.hogu_href }}" class="btn btn-primary">
-				<i class="bi-mastodon"></i>
-				<span class="d-none d-md-inline">
-					Mastodon
-				</span>
-			</a>
-			<span class="form-control">
-				@{{ mastodon.username }}@{{ mastodon.instance }}
-			</span>
-		</div>
-	
-		<!--
-		<a href="{{ mastodon.hogu_href }}">Mastodon</a> (@{{ mastodon.username }}@{{ mastodon.instance }}) [<a href="{{ mastodon.href }}">source</a>]
-		-->
-	</li>
-	{% endif %}
-
-	{% if receipt_tracker %}
-	<li>
-		<div class="input-group mb-1" style="width: fit-content">
-		
-			<a href="{{ receipt_tracker.hogu_href }}" class="btn btn-primary">
-				<i class="bi-rss"></i>
-				<span class="d-none d-md-inline">
-					RSS
-				</span>
-			</a>
-			<span class="form-control">
-				{{ receipt_tracker.href }}
-			</span>
-		</div>
-		<!--
-		<a href="{{ receipt_tracker.hogu_href }}">Receipts</a> [<a href="{{ receipt_tracker.href }}">source</a>]
-		-->
-	</li>
-	{% endif %}
-
-	</ul>
-{% endif %}
-
-{% if mailchimp or discord %}
-	<h2>Community</h2>
-
-	<ul>
-
-	{% if mailchimp %}
-	<li><a href="{{ mailchimp.href }}">Mailchimp</a></li>
-	{% endif %}
-
-	{% if discord %}
-	<li><a href="{{ discord.href }}">Discord</a></li>
-	{% endif %}
-
-	</ul>
-{% endif %}
-
-{% if twitch or instagram %}
-	<h2>Other Socials</h2>
-
-	<ul>
-
-	{% if twitch %}
-	<li>
-		<div class="input-group mb-1" style="width: fit-content">
-		
-			<a href="{{ twitch.hogu_href }}" class="btn btn-primary">
-				<i class="bi-twitch"></i>
-				<span class="d-none d-md-inline">
-					Twitch
-				</span>
-			</a>
-			<span class="form-control">
-				@{{ twitch.id }}
-			</span>
-		</div>
-		<!--
-		<a href="{{ twitch.href }}">Twitch</a> (@{{ twitch.id }})</a>
-		-->
-	</li>
-	{% endif %}
-
-
-	{% if instagram %}
-	<li>
-		<div class="input-group mb-1" style="width: fit-content">
-		
-			<a href="{{ instagram.href }}" class="btn btn-primary">
-				<i class="bi-instagram"></i>
-				<span class="d-none d-md-inline">
-					Instagram
-				</span>
-			</a>
-			<span class="form-control">
-				@{{ instagram.id }}
-			</span>
-		</div>
-		<!--
-		<a href="{{ instagram.href }}">Instagram</a> (@{{ instagram.id }})</a>
-		-->
-	</li>
-	{% endif %}
-	</ul>
-{% endif %}
-
-
-
-{% if contributors %}
-
-	<h2>Contributors</h2>
-	
-	<p>Thanks to our kind contributors for helping support development:</p>
-	
-	<table class="table table-striped">
-		<thead>
-		<tr>
-			<th>Name</th>
-			<th>Amount (Est)</th>
-		</tr>
-		</thead>
-		<tbody>
-		{% for contributor in contributors %}
-			<tr>
-				<td>{{ contributor.name }}</td>
-				<td>{{ contributor.amount }}</td>
-			</tr>
-		{% endfor %}
-		</tbody>
-	</table>
-
-{% endif %}
-
-</div>
-
-</div>
-</div>

hogumathi_app/templates/partial/brand-info.html

@@ -1,153 +0,0 @@
-<div class="w-100" style="background-color: white; border: 1px solid black;">
-
-<h1>{{ brand.display_name }}</h1>
-
-{% if 'logo_href' in brand %}
-
-<img src="{{ brand.logo_href }}" style="max-width: 25%">
-
-{% endif %}
-
-{% if brand.description %}
-{% for para in brand.description.split('\n') %}
-<p>{{ para }}</p>
-{% endfor %}
-{% endif %}
-
-{% if venmo %}
-<h2>Tips</h2>
-
-<ul>
-
-{% if venmo %}
-<li><a href="{{ venmo.href }}">Venmo</a> (@{{ venmo.id }})</li>
-{% endif %}
-
-</ul>
-{% endif %}
-
-
-
-{% if patreon or (gumroad and gumroad.subscriptions and len(gumroad.subscriptions)) %}
-	<h2>Subscriptions</h2>
-
-	<ul>
-
-	{% if patreon %}
-	<li><a href="{{ patreon.href }}">Patreon</a> (@{{ patreon.id }})</li>
-	{% endif %}
-
-	{% for sub in gumroad.subscriptions %}
-	{% if sub.tiers  %}
-	<li>{{ sub.name }}
-		<ul>
-		{% for tier in sub.tiers %}
-		<li><a href="{{ sub.href }}?option={{ tier.option }}">{{ tier.name }}</a> ({{ tier.yearly }}/yr or {{ tier.monthly }}/mo)</li>
-		{% endfor %}
-		</ul>
-	</li>
-	{% else %}
-	<li><a href="{{ sub.href }}">{{ sub.name }} ({{ sub.yearly }}/yr or {{ sub.monthly }}/mo)</a></li>
-	{% endif %}
-	{% endfor %}
-
-	</ul>
-{% endif %}
-
-{% if gumroad and gumroad.products %}
-
-	<h2>Products</h2>
-	
-	<ul>
-	{% for product in gumroad.products %}
-	<li><a href="{{ product.href }}">{{ product.name }}</a> ({{ product.price }})</li>
-	
-	{% endfor %}
-	</ul>
-
-{% endif %}
-
-{% if twitter or youtube or mastodon or receipt_tracker %}
-	<h2>On Hogumathi</h2>
-
-	<ul>
-
-	{% if twitter %}
-	<li><a href="{{ twitter.hogu_href }}">Twitter</a> (@{{ twitter.username }}) [<a href="{{ twitter.href }}">source</a>]</li>
-	{% endif %}
-
-	{% if youtube %}
-	<li><a href="{{ youtube.hogu_href }}">YouTube</a> (@{{ youtube.username }}) [<a href="{{ youtube.href }}">source</a>]</li>
-	{% endif %}
-
-	{% if mastodon %}
-	<li><a href="{{ mastodon.hogu_href }}">Mastodon</a> (@{{ mastodon.username }}@{{ mastodon.instance }}) [<a href="{{ mastodon.href }}">source</a>]</li>
-	{% endif %}
-
-	{% if receipt_tracker %}
-	<li><a href="{{ receipt_tracker.hogu_href }}">Receipts</a> [<a href="{{ receipt_tracker.href }}">source</a>]</li>
-	{% endif %}
-
-	</ul>
-{% endif %}
-
-{% if mailchimp or discord %}
-	<h2>Community</h2>
-
-	<ul>
-
-	{% if mailchimp %}
-	<li><a href="{{ mailchimp.href }}">Mailchimp</a></li>
-	{% endif %}
-
-	{% if discord %}
-	<li><a href="{{ discord.href }}">Discord</a></li>
-	{% endif %}
-
-	</ul>
-{% endif %}
-
-{% if twitch or instagram %}
-	<h2>Other Socials</h2>
-
-	<ul>
-
-	{% if twitch %}
-	<li><a href="{{ twitch.href }}">Twitch</a> (@{{ twitch.id }})</a></li>
-	{% endif %}
-
-
-	{% if instagram %}
-	<li><a href="{{ instagram.href }}">Instagram</a> (@{{ instagram.id }})</a></li>
-	{% endif %}
-	</ul>
-{% endif %}
-
-
-
-{% if contributors %}
-
-	<h2>Contributors</h2>
-	
-	<p>Thanks to our kind contributors for helping support development:</p>
-	
-	<table>
-		<thead>
-		<tr>
-			<th>Name</th>
-			<th>Amount (Est)</th>
-		</tr>
-		</thead>
-		<tbody>
-		{% for contributor in contributors %}
-			<tr>
-				<td>{{ contributor.name }}</td>
-				<td>{{ contributor.amount }}</td>
-			</tr>
-		{% endfor %}
-		</tbody>
-	</table>
-
-{% endif %}
-
-</div>

hogumathi_app/templates/partial/page-nav-bs.html

@@ -1,10 +0,0 @@
-<div class="d-flex flex-row mt-1 flex-wrap">
-	
-		{% if page_nav %}
-		{% for nav_item in page_nav|sort(attribute='order') %}
-		<a class="btn btn-secondary btn-sm mx-1" href="{{ nav_item.href }}">{{ nav_item.label }}</a>
-		{% endfor %}
-		{% endif %}
-		
-	
-</div>

hogumathi_app/templates/partial/timeline-tweet-bs.html

@@ -1,211 +0,0 @@
-
-<div style="width: 60px; max-width: 60px; min-width: 60px">
-	<img loading="lazy"  src="{{ tweet.avi_icon_url }}" alt="Avi">
-</div>
-<div class="d-flex flex-column flex-grow-1">
-	<p class="mt-0 pt-0">
-	<strong><a href="{{ tweet.author_url }}" class="w-100">{{ tweet.display_name }}</a></strong>
-	{% if tweet.author_is_verified %}
-	<small class="verified">[verified]</small>
-	{% endif %}
-	
-	<a href="{{ tweet.author_url }}" class="silver">@{{ tweet.handle }}</a>
-	<a href="{{ tweet.url }}">{{ tweet.created_at }}</a> [<a href="{{ tweet.source_url }}" target="tweet_{{ tweet.id }}">source</a>]
-	</p>
-	
-	<p>
-	
-	{% if tweet.html %}
-		{{ tweet.html | safe }}
-	{% else %}
-		{{ tweet.text | replace('<', '&lt;') | replace('\n', '<br>') | safe }}
-	{% endif %}
-	</p>
-
-	{% if tweet.quoted_tweet %}
-	<div class="d-flex flex-row">
-		{% with tweet = tweet.quoted_tweet %}
-			{% include "partial/timeline-tweet-bs.html" %}
-		{% endwith %}
-	</div>
-	{% endif %}
-	
-	{% if not skip_embed_replies %}
-	{% if tweet.replied_tweet %}
-	<p style="color: silver">
-	Replying to:
-	</p>
-	<div class="reply_to" style="border: 1px solid silver; padding: 6px">
-		<div class="d-flex flex-row reply-box">
-			{% with tweet = tweet.replied_tweet %}
-				{% include "partial/timeline-tweet-bs.html" %}
-			{% endwith %}
-
-		</div>
-	</div>
-	{% elif tweet.replied_tweet_id %}
-	<p style="color: silver">
-	Replying to:
-	</p>
-	{% if tweet.actions.view_replied_tweet %}
-	<p class="reply_to w-100" style="border: 1px solid silver; padding: 6px">
-		<a href="{{ url_for(tweet.actions.view_replied_tweet.route, **tweet.actions.view_replies.route_params) }}">View in Thread</a>.
-	</p>
-	{% endif %}
-	{% endif %}
-	{% endif %}
-	
-	{% if tweet.note %}
-	<p class="note" style="border: 1px solid black; background-color: yellow; padding: 6px">
-		{{ tweet.note.replace('\n', '<br>') | safe }}
-	</p>
-	{% endif %}
-
-	{% if tweet.photos %}
-	
-		<ul>
-		{% for photo in tweet.photos %}
-			<li><img loading="lazy" class="w-100" src="{{ photo.preview_image_url }}" crossorigin="" referrerpolicy="no-referrer" onclick="this.src='{{ photo.url }}'"></li>
-		{% endfor %}
-		</ul>
-
-
-	{% endif %}
-
-	{% if tweet.videos %}
-
-		<ul>
-		{% for video in tweet.videos %}
-			<li><img loading="lazy" class="w-100" 
-			       src="{{ video.preview_image_url }}" referrerpolicy="no-referrer"
-				   
-				   
-				   {% if video.image_url %}
-				   onclick="this.src='{{ video.image_url }}'; this.onclick = undefined" 
-				   {% endif %}
-				   
-				   {% if video.url %}
-				   ondblclick="swapVideoPlayer(this, '{{ video.url }}', '{{ video.content_type }}')"
-				   {% endif %}
-				   >
-				   
-				   <dl>
-				    {% if video.duration_str  %}
-					<dt>Duration</dt>
-					<dd>{{ video.duration_str }}</dt>
-					{% elif video.duration_ms  %}
-					<dt>Duration</dt>
-					<dd>{{ video.duration_ms / 1000 / 60 }} minutes</dt>
-					{% endif %}
-				   </dl>
-				   
-				   {% if video.public_metrics and video.public_metrics.view_count %}
-				   <p class="w-100">
-						view count: {{ video.public_metrics.view_count }}
-				   </p>
-				   {% endif %}
-				   
-				   </li>
-				   
-		{% endfor %}
-		</ul>
-
-	{% endif %}
-	
-		{% if tweet.card %}
-		
-
-		<div class="card-box d-flex">
-			{% if tweet.card.preview_image_url %}
-			<img
-				class="me-1"
-				style=" max-height: 150px; max-width: 150px;"
-				src="{{ tweet.card.preview_image_url }}">
-			{% endif %}
-			<div class="d-flex flex-column">
-				<span><a href="{{ tweet.card.source_url }}">{{ tweet.card.display_url }}</a></span>
-				<span><strong>{{ tweet.card.title }}</strong></span>
-				<p>{{ tweet.card.content }}</p>
-			</div>
-		</div>
-		{% endif %}
-		
-		
-				
-
-		{% if False and tweet.replied_tweet %}
-			<a href="{{ tweet.replied_tweet.url }}">View Parent</a>
-			<a href="{{ url_for('.get_tweet_html', tweet_id=tweet.replied_tweet.conversation_id) }}">View Conversation</a>
-		{% endif %}
-		
-		{% if tweet.public_metrics %}
-		
-
-		<p>
-		{% for k, v in tweet.public_metrics.items() %}
-			{% if v != None %}
-			{{ k.replace('_count', 's').replace('ys', 'ies').replace('_', ' ') }}: {{ v }}, 
-			{% endif %}
-		{% endfor %}
-		
-		</p>
-		{% endif %}
-		
-		
-		
-		{% if tweet.non_public_metrics %}
-		
-
-		
-		<p>
-			{% for k, v in tweet.non_public_metrics.items() %}
-				{% if v != None %}
-				{{ k.replace('_count', 's').replace('ys', 'ies').replace('_', ' ') }}: {{ v }},
-				{% endif %}				
-			{% endfor %}
-			
-		</p>
-		{% endif %}
-		
-		{% if tweet.attachments %}
-		<ul>
-		{% for a in tweet.attachments %}
-			{% if a.content_type == 'application/vnd-hogumathi.livestream-details+json' %}
-			<li class="livestream-details">
-				<dl>
-					
-					{% if a.content.scheduled_start_time %}
-					<dt>Scheduled Start Time</dt>
-					<dd>{{ a.content.scheduled_start_time }}</dd>
-					{% endif %}
-					
-					{% if a.content.start_time %}
-					<dt>Start Time</dt>
-					<dd>{{ a.content.start_time }}</dd>
-					{% endif %}
-					
-					{% if a.content.chat_embed_url %}
-					<dt>Chat</dt>
-					<dd>
-						<iframe class="w-100" height="400" src="{{ a.content.chat_embed_url }}" referrerpolicy="origin"></iframe>
-					</dd>
-					{% endif %}
-					
-					
-				</dl>
-			</li>
-			{% else %}
-			<li><a href="{{ a.url }}">{{ a.name }}</a> {{ a.content_type }} ({{ a.size }})
-			{% endif %}
-			</li>
-		{% endfor %}
-		</ul>
-		{% endif %}
-	
-		{% if show_conversation_id %}
-		<p>
-			Conversation: {{ tweet.conversation_id }}
-			
-		</p>
-		{% endif %}
-</div>

hogumathi_app/templates/partial/timeline-tweet.html

@@ -3,6 +3,13 @@
 	<img loading="lazy"  src="{{ tweet.avi_icon_url }}" alt="Avi">
 </div>
 <div class="dtc w-90 v-top">
+	{% if tweet.title %}
+	<h2 class="w-100 mt0">
+	<a href="{{ tweet.url }}">
+	{{ tweet.title }}
+	</a>
+	</h2>
+	{% endif %}
 	<p class="w-100 mt0 pt0">
 	<strong><a href="{{ tweet.author_url }}" class="w-100">{{ tweet.display_name }}</a></strong>
 	{% if tweet.author_is_verified %}
@@ -69,7 +76,8 @@
 	<p class="w-100">
 		<ul>
 		{% for photo in tweet.photos %}
-			<li><img loading="lazy" class="w-100" src="{{ photo.preview_image_url }}" crossorigin="" referrerpolicy="no-referrer" onclick="this.src='{{ photo.url }}'" style="max-height: {{ photo.height }};""></li>
+			<li><img loading="lazy" class="w-100" src="{{ photo.preview_image_url }}" 
+			referrerpolicy="no-referrer" onclick="this.src='{{ photo.url }}'" style="max-height: {{ photo.height }};""></li>
 		{% endfor %}
 		</ul>
 

hogumathi_app/templates/partial/tweets-timeline-bs.html

@@ -1,360 +0,0 @@
-
-
-<script src="{{ url_for('static', filename='tweets-ui.js') }}"></script>
-
-
-<script>
-
-{% if notes_app_url %}
-var notesAppUrl = {{ notes_app_url | tojson }}
-{% endif %}
-
-	if (!window['dataset']) {
-		{% if visjs_enabled %}
-		window.dataset = new vis.DataSet();
-		{% else %}
-		window.dataset = {
-			items: [],
-			update: function (items) {
-				dataset.items = dataset.items.concat(items);
-			},
-			get: function () {
-				return items;
-			}
-		}
-		{% endif %}
-	}
-</script>
-
-<script>
-function feed_item_to_activity (fi) {
-		
-		var group = 'tweet';
-		var y = 1;
-		if (fi.retweeted_tweet_id) {
-			group = 'retweet';
-			y = 2;
-		} else if (fi.replied_tweet_id) {
-			group = 'reply';
-			y = 3;
-		}
-		
-		return {
-			//'id': fi.id,
-			'x': new Date(fi.created_at),
-			'y': y,
-			'group': group,
-			'feed_item': fi
-		}
-	}
-	
-	function feed_item_to_likes (fi) {
-		
-		if ( !fi['public_metrics'] || !fi.public_metrics['like_count'] ) {
-			return;
-		}
-		
-		var group = 'likes';
-		var y = fi.public_metrics.like_count;
-		
-		
-		return {
-			//'id': fi.id,
-			'x': new Date(fi.created_at),
-			'y': y,
-			'group': group,
-			'feed_item': fi
-		}
-	}
-	
-	function feed_item_to_replies (fi) {
-		
-		if ( !fi['public_metrics'] || !fi.public_metrics['reply_count'] ) {
-			return;
-		}
-		
-		var group = 'replies';
-		var y = fi.public_metrics.reply_count;
-		
-		
-		return {
-			//'id': fi.id,
-			'x': new Date(fi.created_at),
-			'y': y,
-			'group': group,
-			'feed_item': fi
-		}
-	}
-</script>
-
-{% if twitter_live_enabled and visjs_enabled and not skip_plot %}
-
-<div class="w-100" style="position: sticky; top: 20px; background-color: silver; padding: 20px 0; margin: 10px 0;">
-		<div id="visualization"></div>
-	
-</div>
-
-{% endif %}
-
-<ul id="tweets" class="tweets z-0">
-
-{% for tweet in tweets %}
-
-<li class="tweet d-flex flex-column mb-2 {% if tweet.is_marked %}marked{% endif %}">
-<script>
-	
-	
-	var feedItem = {{ tweet | tojson }};
-	var plotItems = [];
-	
-	var likesPoint = feed_item_to_likes(feedItem);
-	if (likesPoint) { plotItems.push(likesPoint); }
-	
-	var repliesPoint = feed_item_to_replies(feedItem);
-	if (repliesPoint) { plotItems.push(repliesPoint); }
-	
-	if (!plotItems.length) {
-		plotItems.push(feed_item_to_activity(feedItem))
-	}
-	
-	
-	
-	dataset.update(plotItems);
-
-</script>
-
-
-	{% if tweet.retweeted_by %}
-	<div class="d-flex flex-row">
-		<p class="text-end pe-3" style="width: 60px; max-width: 60px; min-width: 60px">
-			<i class="bi-repeat"></i>
-		</p>
-		<p class="flex-grow-1"><a class="moon-gray" href="{{ tweet.retweeted_by_url }}">{{ tweet.retweeted_by }} Retweeted</a></p>
-	</div>
-	{% endif %}
-	<div class="d-flex flex-row">
-
-		{% include "partial/timeline-tweet-bs.html" %}
-		
-		
-	</div>
-	<div class="d-flex flex-row">
-		<div class="tweet-actions-box d-flex flex-row justify-content-lg-between border-bottom g-2 p-2 flex-grow-1 flex-wrap" style="margin-left: 60px"">
-		
-		
-		{% if tweet.actions.view_replies %}
-		<a class="btn btn-sm btn-secondary m-1" href="{{ url_for(tweet.actions.view_replies.route, **tweet.actions.view_replies.route_params) }}">
-		
-		<i class="bi-chat"></i> 
-		replies
-		</a>
-
-		{% endif %}
-		
-
-		
-		
-		{% if show_thread_controls and tweet.conversation_id %}
-		{% with tweet=tweets[0] %}
-		{% if tweet.actions.view_thread %}
-		<a class="btn btn-sm btn-secondary m-1" href="{{ url_for(tweet.actions.view_thread.route, **tweet.actions.view_thread.route_params) }}">author thread</a>
-
-		{% endif %}
-		{% if tweet.actions.view_conversation %}
-		<a class="btn btn-sm btn-secondary m-1" href="{{ url_for(tweet.actions.view_conversation.route, **tweet.actions.view_conversation.route_params) }}">full convo</a>
-
-		{% endif %}
-		{% endwith %}
-		{% endif %}
-		
-		{% if tweet.actions.view_activity %}
-		<a class="btn btn-sm btn-secondary m-1" href="{{ url_for(tweet.actions.view_activity.route, **tweet.actions.view_activity.route_params) }}">
-		
-		<i class="bi-graph-up"></i>
-		activity
-		</a>
-
-		{% endif %}
-		
-		{% if tweet.actions.retweet %}
-		<a class="btn btn-sm btn-secondary m-1" hx-post="{{ url_for(tweet.actions.retweet.route, **tweet.actions.retweet.route_params) }}">
-		
-		<i class="bi-repeat"></i> 
-		retweet
-		
-		</a>
-
-		{% endif %}
-		
-		{% if tweet.actions.bookmark %}
-		<a class="btn btn-sm btn-secondary m-1" hx-post="{{ url_for(tweet.actions.bookmark.route, **tweet.actions.bookmark.route_params) }}">
-		
-		<i class="bi-bookmark"></i>
-		bookmark
-		
-		</a>
-		{% if tweet.actions.delete_bookmark %}
-		<a class="btn btn-sm btn-secondary m-1" hx-delete="{{ url_for(tweet.actions.delete_bookmark.route, **tweet.actions.delete_bookmark.route_params) }}">-</a>
-		{% endif %}
-
-		{% endif %}
-		
-		<a class="btn btn-sm btn-secondary m-1" class="tweet-action copy-formatted" href="javascript:copyTweetToClipboard('{{ tweet.id }}')">copy formatted</a>
-		{% if notes_app_url %}
-
-		<a class="btn btn-sm btn-secondary m-1" class="tweet-action swipe-to-note" href="javascript:swipeTweetToNotesApp('{{ tweet.id }}')">swipe to note</a>
-		{% endif %}
-
-		</div>
-	</div>
-	
-
-	</li>
-
-{% endfor %}
-
-
-
-{% if query.next_data_url %}
-
-	<li style="height: 50px; vertical-align: middle"
-		hx-get="{{ query.next_data_url }}"
-		hx-trigger="revealed"
-		hx-swap="outerHTML"
-		hx-select="ul#tweets > li"
-		>
-		<center style="height: 100%">
-
-		<span class="js-only">
-		Loading more tweets...
-		</span>
-
-		</center>
-	</li>
-	
-{% elif query.next_page_url %}
-	<li style="height: 50px; vertical-align: middle"
-		>
-		<center style="height: 100%">
-		<a href="{{ query.next_page_url }}">
-		Go to Next Page
-		</a>
-		
-		
-		</center>
-	
-	</li>
-	
-{% endif %}
-	<li style="display: none">
-		<script>
-			// https://stackoverflow.com/questions/22663353/algorithm-to-remove-extreme-outliers-in-array
-			// we should remove outliers on the X axis. That will mainly be RTs with old dates.
-			// we might also be able to get the date of RT as opposed to OG tweet date.
-			
-			// https://towardsdatascience.com/ways-to-detect-and-remove-the-outliers-404d16608dba
-			var profileDataEl = document.querySelector('#profile-data');
-			
-			if (window['dataset'] && profileDataEl) {
-				profileDataEl.innerHTML = dataset.get().filter(i => 'public_metrics' in i).map(i => i.public_metrics.like_count).join(', ');
-			}
-
-			{% if visjs_enabled %}
-			if (window.profileActivity) {
-				window.profileActivity.fit()
-			}
-			{% endif %}
-		</script>
-	</li>
-</ul>
-
-			{% if twitter_live_enabled and visjs_enabled and not skip_plot %}
-			
-	<script>
-
-
-
-function onClick (e) {
-  // we need to scan the dataset between min/max x/y
-  // 
-  // TODO we want to scale these based on the zoom level / pixel values
-  //
-  // FIXME sometimes we get several points:
-  // We could also go for the closest point within the bound.
-  // Perhaps cycle through upon multiple clicks.
-  // For now we can just zoom in closer.
-  //
-  // range: graph2d.components[3].options.dataAxis.left.range.max
-  // fixing this is lower priority since it is currently static.
-  graph2d = window.profileActivity;
-  
-  var timeWindow = graph2d.getWindow();
-  var windowInSeconds = (timeWindow.end - timeWindow.start) / 1000;
-  
-  var pixelWidth = graph2d.dom.centerContainer.offsetWidth;
-  
-  var secondsPerPixel = windowInSeconds / pixelWidth;
-  
-  console.log('secondsPerPixel = ' + secondsPerPixel);
-  
-  
-  //var MAX_TIME_DIFF  = 1000 * 60 * 60;
-  
-  var MAX_TIME_DIFF = 10 * secondsPerPixel * 1000;
-  var MAX_VALUE_DIFF = 10;
-  
-  console.log(`click. value=${e.value[0]}, time=${e.time}`);
-  console.log(e);
-  
-  var nearbyItems = dataset.get({filter: function (item) {
-	var timeDiff = new Date(item.x).getTime() - e.time.getTime();
-	var valueDiff = item.y - e.value[0];
-	
-	return Math.abs(timeDiff) < MAX_TIME_DIFF 
-		   && Math.abs(valueDiff) < MAX_VALUE_DIFF;
-	
-  }});
-  
-  //console.log([e.time, e.value[0]]);
-  console.log('nearby points:');
-  console.log(nearbyItems);
-}
-
-
-var container = document.getElementById('visualization');
-
-
-
-var options = {
-  sort: false,
-  sampling:false,
-  style:'points',
-  dataAxis: {
-		  width: '88px',
-
-	  //visible: false,
-	  left: {
-		  range: {
-			  min: 0, max: 10
-		  }
-	  }
-  },  
-  drawPoints: {
-	  enabled: true,
-	  size: 6,
-	  style: 'circle' // square, circle
-  },
-  defaultGroup: 'Scatterplot',
-  graphHeight: '50px',
-  width: '100%'
-};
-
-var groups = [{id: 'feed_item'}];
-
-window.profileActivity = new vis.Graph2d(container, window.dataset, groups, options);
-
-window.profileActivity.on('click', onClick);
-
-	</script>
-			
-			{% endif %}

hogumathi_app/templates/partial/user-card-bs.html

@@ -1,96 +0,0 @@
-<div class="user w-100 p-1" style="border: 1px solid black;">
-	
-	<div style="float: right; margin-top: 0; text-align: right">
-		{% if user.profile_image_url %}
-		<img style="float: right; margin-left: 4px;" src="{{ user.profile_image_url }}">
-	{% endif %}
-
-	{% if user.username %}
-	<pre style="display: inline; font-weight: bold; ">@{{ user.username }}</pre>
-	<br>
-	{% endif %}
-	<span><pre style="display: inline;">{{ user.id }}</pre></span>
-	
-	</div>
-	
-	<p style="margin-top: 0">
-		
-		
-		{% if user.actions and user.actions.view_profile %}
-		{% with view_profile=user.actions.view_profile %}
-		<a href="{{ url_for( view_profile.route, **view_profile.route_params ) }}">
-		{% endwith %}
-		{% elif user.url %}
-		<a href="{{ user.url }}">
-		{% else %}
-		<a href="{{ url_for('twitter_v2_facade.get_profile_html', user_id=user.id) }}">
-		{% endif %}
-		<strong>{{ user.name }}</strong></a>
-		
-		{% if user.verified %}
-		<small>[verified]</small>
-		{% endif %}
-		
-		{% if user.protected %}
-		<small>[protected]</small>
-		{% endif %}
-	</p>
-	
-		
-	{% if user.description %}
-	<div>
-	
-		{{ user.description }}
-	</div>
-	{% endif %}
-	
-	
-	{% if user.pinned_tweet_id %}
-	<div>
-		<p>Pinned tweet:</p>
-		<div style="border: 1px solid black">{{ user.pinned_tweet_id }}</div>
-	</div>
-	{% endif %}
-	
-	<dl>
-		{% if user.created_at %}
-		<dt>Member since</dt>
-		<dd>{{ user.created_at }}</dd>
-		{% endif %}
-		
-		{% if user_dc and user_dc.website %}
-		<dt>Website</dt>
-		<dd><a href="{{ user_dc.website }}">{{ user_dc.website }}</a></dd>
-		{% endif %}
-		
-		{% if user.location %}
-		<dt>Location</dt>
-		<dd>{{ user.location }}</dd>
-		{% endif %}
-		
-		{% if user.public_metrics %}
-		
-		{% if user.public_metrics.tweet_count %}
-		<dt>Tweets</dt>
-		<dd>{{ user.public_metrics.tweet_count }}</dd>
-		{% endif %}
-		
-		{% if user.public_metrics.followers_count %}
-		<dt>Followers</dt>
-		<dd>{{ user.public_metrics.followers_count }}</dd>
-		{% endif %}
-		
-		{% if user.public_metrics.following_count %}
-		<dt>Following</dt>
-		<dd>{{ user.public_metrics.following_count }}</dd>
-		{% endif %}
-		
-		{% if user.public_metrics.listed_count %}
-		<dt>Listed</dt>
-		<dd>{{ user.public_metrics.listed_count }}</dd>
-		{% endif %}
-		
-		{% endif %}
-	</dl>
-	
-</div>

hogumathi_app/templates/partial/user-card.html

@@ -44,6 +44,13 @@
 	</div>
 	{% endif %}
 	
+	{% if user.total_count %}
+		{# this is getting into the new Collection territory, which mirrored User #}
+		<p>
+		{{ user.total_count }} items.
+		</p>
+	{% endif %}
+	
 	
 	{% if user.pinned_tweet_id %}
 	<div>

hogumathi_app/templates/partial/user-picker.html

@@ -9,7 +9,7 @@
 			<li><a href="{{ url_for('mastodon_facade.get_timeline_home_html', me=k) }}">{{ k }}</a>
 		{% endif %}
 		{% if youtube_enabled and k.startswith('youtube:') %}
-			<li><a href="{{ url_for('youtube_facade.get_latest_html', me=k) }}">{{ k }}</a>
+			<li><a href="{{ url_for('youtube_v3_facade.get_latest_html', me=k) }}">{{ k }}</a>
 		{% endif %}
 		{% if messages_enabled and k.startswith('messages:') %}
 			<li><a href="{{ url_for('messages_facade.get_latest2_html', me=k) }}">{{ k }}</a>

hogumathi_app/templates/youtube-channel.html

@@ -1,21 +0,0 @@
-{% extends "base.html" %}
-
-{% block head %}
-	<title>Tweet Library: {{ user.id }}</title>
-{% endblock %}
-
-
-{% block content %}
-
-
-
-	{% include "partial/page-nav.html" %}
-	
-	{% include "partial/users-list.html" %}
-
-	{% if brand %}
-	{% include "partial/brand-info.html" %}
-	{% endif %}
-
-	
-{% endblock %}

hogumathi_app/test/unit/hogumathi_app_test/__init__.py

@@ -0,0 +1,4 @@
+import sys
+
+sys.path.append('lib')
+sys.path.append('extensions')

hogumathi_app/test/unit/hogumathi_app_test/test_content_system.py

@@ -0,0 +1,308 @@
+from typing import List, Dict
+
+import pytest
+
+from hogumathi_app.content_system import ContentSystem
+
+def tweet_cs (tweet_id:str) -> object:
+    print(f'tweet_cs: {tweet_id}')
+    if tweet_id == '1':
+        return {'text': 'one', 'id': '1'}
+
+def tweets_cs (content_ids:List[str]) -> Dict[str,object]:
+    print(f'tweets_cs: {content_ids}')
+    if ','.join(content_ids) == 'twitter:tweet:1,twitter:tweet:2':
+        return {
+            'twitter:tweet:1': {'text': 'one', 'id': '1'},
+            'twitter:tweet:2': {'text': 'two', 'id': '2'}
+            }
+    elif ','.join(content_ids) == 'twitter:tweet:2':
+        return {
+            'twitter:tweet:2': {'text': 'two', 'id': '2'}
+        }
+
+
+def video_cs (vid_id:str) -> object:
+    print(f'video_cs: {vid_id}')
+    if vid_id == '3':
+        return {'url': 'three.mp4', 'id': '3'}
+
+def videos_cs (content_ids:List[str]) -> Dict[str, object]:
+    print(f'videos_cs: {content_ids}')
+    if ','.join(content_ids) == 'youtube:video:3,youtube:video:4':
+        return {
+            'youtube:video:3': {'url': 'three.mp4', 'id': '3'},
+            'youtube:video:4': {'url': 'four.mp4', 'id': '4'}
+            }
+
+def photo_cs (post_id:str) -> object:
+    print(f'photo_cs: {post_id}')
+    if post_id == '3':
+        return {'url': 'three.png', 'id': '3'}
+
+
+def test_content_system ():
+    
+    h_cs = ContentSystem()
+    
+    h_cs.register_content_source('twitter:tweet:', tweet_cs)
+    h_cs.register_content_source('twitter:tweets', tweets_cs)
+    
+    t1 = h_cs.get_content('twitter:tweet:1')
+    t2 = h_cs.get_content('twitter:tweet:2')
+    t3 = h_cs.get_content('fake:1')
+    
+    assert(t1 == {'text': 'one', 'id': '1'})
+    assert(t2 == None)
+    assert(t3 == None)
+    
+    t_multi = h_cs.get_content('twitter:tweets', content_ids=['twitter:tweet:1', 'twitter:tweet:2'])
+    
+    assert(t_multi == {
+            'twitter:tweet:1': {'text': 'one', 'id': '1'},
+            'twitter:tweet:2': {'text': 'two', 'id': '2'}
+            })
+    
+def test_bulk_content ():
+    
+    h_cs = ContentSystem()
+    
+    h_cs.register_content_source('twitter:tweet:', tweet_cs)
+    h_cs.register_content_source('twitter:tweets', tweets_cs, id_pattern='')
+    
+    tweets = h_cs.get_all_content(['twitter:tweet:1', 'twitter:tweet:2', 'fake:1'])
+    
+    
+    assert(tweets == {
+        'twitter:tweet:1': {'text': 'one', 'id': '1'},
+        'twitter:tweet:2': {'text': 'two', 'id': '2'},
+        'fake:1': None
+        })
+
+@pytest.mark.xfail(reason="need to rethink get_all_content code for this.")
+def test_bulk_content_partial_miss ():
+    """
+    xfail: note in ContentSystem.get_content
+    """
+    
+    h_cs = ContentSystem()
+    
+    def tweets_cache_cs (content_ids):
+        if ','.join(content_ids) == 'twitter:tweet:1,twitter:tweet:2':
+            return {
+                'twitter:tweet:1': {'text': 'one', 'id': '1'},
+                'twitter:tweet:2': None
+            }
+    
+    h_cs.register_content_source('twitter:tweet:', tweet_cs)
+    h_cs.register_content_source('twitter:tweets', tweets_cs, id_pattern='')
+    h_cs.register_content_source('', tweets_cache_cs, id_pattern='(.+)', weight=99999)
+    
+    tweets = h_cs.get_all_content(['twitter:tweet:1', 'twitter:tweet:2', 'fake:1'])
+    
+    
+    assert(tweets == {
+        'twitter:tweet:1': {'text': 'one', 'id': '1'},
+        'twitter:tweet:2': {'text': 'two', 'id': '2'},
+        'fake:1': None
+        })
+    
+def test_hooks_bulk_content ():
+    
+    h_cs = ContentSystem()
+    
+    h_cs.register_content_source('twitter:tweet:', tweet_cs)
+    h_cs.register_content_source('twitter:tweets', tweets_cs, id_pattern='')
+    
+    hooked_content = []
+    def got_content(content_id, content):
+        hooked_content.append([content_id, content])
+    
+    h_cs.register_hook('got_content', got_content)
+    
+    content = h_cs.get_all_content(['twitter:tweet:1', 'twitter:tweet:2', 'fake:1'])
+    
+    assert(content == {
+        'twitter:tweet:1': {'text': 'one', 'id': '1'},
+        'twitter:tweet:2': {'text': 'two', 'id': '2'},
+        'fake:1': None
+    })
+    
+    print(f'hooked_content: {hooked_content}')
+    assert(hooked_content == [
+        ['twitter:tweets', {
+            'twitter:tweet:1': {'text': 'one', 'id': '1'},
+            'twitter:tweet:2': {'text': 'two', 'id': '2'}
+            }],
+        ['twitter:tweet:1', {'text': 'one', 'id': '1'}],
+        ['twitter:tweet:2', {'text': 'two', 'id': '2'}]
+        ])
+        
+def test_hooks_bulk_content_multi_bulk ():
+    
+    h_cs = ContentSystem()
+    
+    h_cs.register_content_source('twitter:tweet:', tweet_cs)
+    h_cs.register_content_source('twitter:tweets', tweets_cs, id_pattern='')
+    h_cs.register_content_source('youtube:video:', video_cs)
+    h_cs.register_content_source('youtube:videos', videos_cs, id_pattern='')
+    h_cs.register_content_source('instagram:post:', photo_cs)
+    
+    hooked_content = []
+    def got_content(content_id, content):
+        hooked_content.append([content_id, content])
+    
+    h_cs.register_hook('got_content', got_content)
+    
+    content = h_cs.get_all_content(['twitter:tweet:1', 'twitter:tweet:2',
+                                    'youtube:video:3', 'youtube:video:4',
+                                    'instagram:post:3',
+                                    'fake:1'])
+    
+    assert(content == {
+        'twitter:tweet:1': {'text': 'one', 'id': '1'},
+        'twitter:tweet:2': {'text': 'two', 'id': '2'},
+        'youtube:video:3': {'url': 'three.mp4', 'id': '3'},
+        'youtube:video:4': {'url': 'four.mp4', 'id': '4'},
+        'instagram:post:3': {'url': 'three.png', 'id': '3'},
+        'fake:1': None
+    })
+    
+    print(f'hooked_content: {hooked_content}')
+    assert(hooked_content == [
+        ['instagram:post:3', {'url': 'three.png', 'id': '3'}],
+        ['twitter:tweets', {
+            'twitter:tweet:1': {'text': 'one', 'id': '1'},
+            'twitter:tweet:2': {'text': 'two', 'id': '2'}
+            }],
+        ['twitter:tweet:1', {'text': 'one', 'id': '1'}],
+        ['twitter:tweet:2', {'text': 'two', 'id': '2'}],
+        ['youtube:videos', {
+            'youtube:video:3': {'url': 'three.mp4', 'id': '3'},
+            'youtube:video:4': {'url': 'four.mp4', 'id': '4'}
+            }],
+        ['youtube:video:3', {'url': 'three.mp4', 'id': '3'}],
+        ['youtube:video:4', {'url': 'four.mp4', 'id': '4'}]
+        ])
+
+def test_cache ():
+    
+    h_cs = ContentSystem()
+    
+    cache_hits = []
+    
+    cache = {}
+    def cache_cs (content_id = None, content_ids = None):
+        if content_id in cache:
+            content = cache[content_id]
+            cache_hits.append([content_id, content])
+            return content
+        
+    h_cs.register_content_source('twitter:tweet:', tweet_cs)
+    h_cs.register_content_source('twitter:tweets', tweets_cs, id_pattern='')
+    
+    h_cs.register_content_source('', cache_cs, id_pattern='(.+)', weight=99999)
+    
+
+    def cache_hook (content_id, content):
+        # FIXME we might need a skip hook mechanism
+        # this will be invoked even with a cache hit.
+        # perhaps pass source_id into the hook
+        if content_id.endswith('s') or content_id in cache:
+            # FIXME exclusion list/patterns
+            # we should allow bulk fetches to get cached items
+            return
+        cache[content_id] = content
+    
+    h_cs.register_hook('got_content', cache_hook)
+    
+    t1 = h_cs.get_content('twitter:tweet:1')
+    t2 = h_cs.get_content('twitter:tweet:2')
+    t3 = h_cs.get_content('fake:1')
+    
+    assert(t1 == {'text': 'one', 'id': '1'})
+    assert(t2 == None)
+    assert(t3 == None)
+    
+    assert(cache_hits == [])
+    
+    t1_2 = h_cs.get_content('twitter:tweet:1')
+    t2_2 = h_cs.get_content('twitter:tweet:2')
+    t3_2 = h_cs.get_content('fake:1')
+    
+    assert(t1_2 == t1)
+    assert(t2_2 == None)
+    assert(t2_2 == None)
+    
+    print(f'cache_hits = {cache_hits}')
+    
+    assert(cache_hits == [
+        ['twitter:tweet:1', {'text': 'one', 'id': '1'}]
+        ])
+
+
+def test_cache_bulk ():
+    
+    h_cs = ContentSystem()
+    
+    cache_hits = []
+    
+    cache = {}
+    def cache_cs (content_id = None, content_ids = None):
+        if content_id and content_id in cache:
+            content = cache[content_id]
+            cache_hits.append([content_id, content])
+            return content
+        elif content_ids:
+            results = {}
+            for content_id in content_ids:
+                content = cache_cs(content_id)
+                if content:
+                    results[content_id] = content
+            return results
+        
+    h_cs.register_content_source('twitter:tweet:', tweet_cs)
+    h_cs.register_content_source('twitter:tweets', tweets_cs, id_pattern='')
+    h_cs.register_content_source('youtube:video:', video_cs)
+    h_cs.register_content_source('youtube:videos', videos_cs, id_pattern='')
+    h_cs.register_content_source('instagram:post:', photo_cs)
+    
+    h_cs.register_content_source('', cache_cs, id_pattern='(.+)', weight=99999)
+    
+    
+    def cache_hook (content_id, content):
+        # FIXME we might need a skip hook mechanism
+        # this will be invoked even with a cache hit.
+        # perhaps pass source_id into the hook
+        if content_id.endswith('s') or content_id in cache:
+            # FIXME exclusion list/patterns
+            # we should allow bulk fetches to get cached items
+            return
+        cache[content_id] = content
+    
+    h_cs.register_hook('got_content', cache_hook)
+    
+    content = h_cs.get_all_content(['twitter:tweet:1', 'twitter:tweet:2',
+                                    'youtube:video:3', 'youtube:video:4',
+                                    'instagram:post:3',
+                                    'fake:1'])
+    
+    assert(cache_hits == [])
+    
+    content2 = h_cs.get_all_content(['twitter:tweet:1', 'twitter:tweet:2',
+                                    'youtube:video:3', 'youtube:video:4',
+                                    'instagram:post:3',
+                                    'fake:1'])
+    
+    assert(content == content2)
+    
+    print(f'cache_hits = {cache_hits}')
+    
+    assert(cache_hits == [
+        ['instagram:post:3', {'url': 'three.png', 'id': '3'}],
+        ['twitter:tweet:1', {'text': 'one', 'id': '1'}],
+        ['twitter:tweet:2', {'text': 'two', 'id': '2'}],
+        ['youtube:video:3', {'url': 'three.mp4', 'id': '3'}],
+        ['youtube:video:4', {'url': 'four.mp4', 'id': '4'}]
+        ])
+        

hogumathi_app/view_model.py

@@ -5,12 +5,33 @@
 from dataclasses import dataclass, asdict, replace
 from typing import List, Dict, Optional, Tuple
 
+import re
+
+@dataclass
+class ContentId:
+    prefix: str
+    id: str
+    
+    def __str__ (self):
+        return self.prefix + self.id
+    
+    @staticmethod
+    def parse (content_id, prefix, id_pattern = '(.+)'):
+        raw_id = content_id[len(prefix):]
+        id_matches = re.fullmatch(id_pattern, raw_id)
+        if id_matches:
+            raw_id = id_matches[0]
+            return ContentId(prefix=prefix, id=raw_id)
+    
 @dataclass
 class PublicMetrics:
     reply_count: Optional[int] = None
     quote_count: Optional[int] = None
     retweet_count: Optional[int] = None
     like_count: Optional[int] = None
+    impression_count: Optional[int] = None
+    bookmark_count: Optional[int] = None
+    
     
     # may be video only
     view_count: Optional[int] = None
@@ -100,6 +121,7 @@ class FeedItem:
     
     published_by: Optional['FeedServiceUser'] = None
     
+    title: Optional[str] = None
     text: Optional[str] = None
     html: Optional[str] = None
     
@@ -204,16 +226,55 @@ class FeedServiceUser:
         return self.preview_image_url
 
 
+@dataclass
+class Collection:
+    """
+    Works for both collections and lists for now
+    """
+    id:str
+    name: str
+    
+    description: Optional[str] = None
+    preview_image_url: Optional[str] = None
+    
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+    
+    owner_id: Optional[str] = None
+    owner: Optional[FeedServiceUser] = None
+    
+    url: Optional[str] = None
+    
+    total_count: Optional[int] = None
+    
+    current_page: Optional['CollectionPage'] = None
+
+
+
+@dataclass
+class CollectionItem:
+    item: List[FeedServiceUser|FeedItem|ThreadItem|Collection] = None
+    sort_order: Optional[int] = None
+    after_id: Optional[str] = None
+
 @dataclass
 class CollectionPage:
     """
-    Feed is a collection.
+    Works for Collections and Lists for now, as well as Threads and nested Collections.
+    
+    Feed is a Collection.
     """
     
     id: str
-    items: Optional[List[FeedServiceUser|FeedItem|ThreadItem]] = None
+    items: Optional[List[FeedServiceUser|FeedItem|ThreadItem|CollectionItem|Collection]] = None
     next_token: Optional[str] = None
     last_dt: Optional[str] = None
+    total_count: Optional[int] = None
+    
+    includes: Optional[Dict] = None
+
+
+
 
 def cleandict(d):
     if isinstance(d, dict):

hogumathi_app/web.py

@@ -0,0 +1,115 @@
+from hashlib import sha256
+import os
+from pathlib import Path
+
+import json
+import requests
+
+from flask import Flask, g, redirect, url_for, render_template, jsonify, request, send_from_directory
+
+from . import content_system as h_cs
+
+api = Flask(__name__, static_url_path='')
+
+@api.context_processor
+def add_nav_items_to_template_context ():
+    nav_items = []
+    
+    route_nav = g.get('route_nav')
+    
+    if route_nav:
+        nav_items += route_nav
+        
+    module_nav = g.get('module_nav')
+    
+    if module_nav:
+        nav_items += module_nav
+        
+    #nav_items.sort(key = lambda ni: ni['order'])
+    
+    return dict(
+        nav_items = nav_items
+    )
+
+@api.get('/login.html')
+def get_login_html ():
+	opengraph_info = dict(
+		type = 'webpage', # threads might be article
+		url = g.app_url,
+		title = 'Hogumathi',
+		description = 'An app for Twitter, Mastodon, YouTube, etc; Open Source.'
+	)
+	
+	return render_template('login.html', opengraph_info=opengraph_info)
+
+@api.get('/')
+def index ():
+	return redirect(url_for('.get_login_html'))
+
+@api.get('/img')
+def get_image ():
+	print('GET IMG')
+	url = request.args['url']
+	url_hash = sha256(url.encode('utf-8')).hexdigest()
+	path = f'.data/cache/media/{url_hash}'
+	
+	print(f'path = {path}')
+	
+	if not os.path.exists(path):
+		resp = requests.get(url)
+		print(f'status_code = {resp.status_code}')
+		if resp.status_code >= 200 and resp.status_code < 300:
+			with open(path, 'wb') as f:
+				f.write(resp.content)
+			with open(f'{path}.meta', 'w') as f:
+				headers = dict(resp.headers)
+				json.dump(headers, f)
+		else:
+			return 'not found.', 404
+			
+	with open(f'{path}.meta', 'r') as f: 
+		headers = json.load(f)
+	
+	#print(url)
+	#print(url_hash)
+	#print(headers)
+	
+	# not sure why some responses use lower case.
+	mimetype = headers.get('Content-Type') or headers.get('content-type')
+	
+	# Flask goes relative to the module as opposed to the working directory.
+	media_cache_dir = Path(Path.cwd(), '.data/cache/media')
+	
+	return send_from_directory(media_cache_dir, url_hash, mimetype=mimetype)
+
+@api.get('/content/abc123.html')
+def get_abc123_html ():
+
+	return 'abc123'
+
+@api.get('/content/<content_id>.html')
+def get_content_html (content_id, content_kwargs=None):
+	
+	if not content_kwargs:
+		content_kwargs = filter(lambda e: e[0].startswith('content:'), request.args.items())
+		content_kwargs = dict(map(lambda e: [e[0][len('content:'):], e[1]], content_kwargs))
+	
+	content = h_cs.get_content(content_id, **content_kwargs)
+	
+	if type(content) == h_vm.FeedItem:
+		return render_template('tweet-collection.html', tweets=[content], user = {}, query = {})
+	elif type(content) == h_vm.CollectionPage:
+		pagination_token = request.args.get('pagination_token')
+		
+		if content.next_token:
+			print(f'next_token = {content.next_token}')
+		
+
+		return render_template('tweet-collection.html', tweets=content.items, user = {}, query = {})
+	else:
+		return jsonify(content)
+
+@api.get('/content/def456.html')
+def get_def456_html ():
+
+	return get_content_html('brand:ispoogedaily')

lib/mastodon_v2/types.py

@@ -105,9 +105,17 @@ class Status:
     muted: Optional[bool] = None
     bookmarked: Optional[bool] = None
     pinned: Optional[bool] = None
-    filtered: Optional[bool] = None
+    filtered: Optional[List['FilterMatch']] = None
     
 
+Filter = Dict
+
+@dataclass
+class FilterMatch:
+    filter: Filter
+    keyword_matches: Optional[List[str]] = None
+    status_matches: Optional[List[StatusId]] = None
+
 @dataclass
 class Context:
     ancestors: List[Status]

lib/twitter_v2/test/unit/__init__.py

@@ -0,0 +1,3 @@
+import sys
+
+sys.path.append('./lib')

lib/twitter_v2/types.py

@@ -45,6 +45,8 @@ class PublicMetrics:
     reply_count: Optional[int] = None
     like_count: Optional[int] = None
     quote_count: Optional[int] = None
+    impression_count: Optional[int] = None
+    bookmark_count: Optional[int] = None
     
     # on video media (only?)
     view_count: Optional[int] = None

sample-env.txt

@@ -0,0 +1,25 @@
+FLASK_SECRET=
+
+NOTES_APP_URL=
+
+
+MASTODON_CLIENT_NAME=Hogumathi (Local Dev)
+
+# Twitter App Token cache
+BEARER_TOKEN=
+
+TWITTER_CLIENT_ID=
+TWITTER_CLIENT_SECRET=
+
+TWITTER_CONSUMER_KEY=
+TWITTER_CONSUMER_SECRET=
+
+# Path to tweet.js converted into json (barely works, bolting these two things together)
+ARCHIVE_TWEETS_PATH=
+
+# API key for the app to use if the user has not logged in.
+GOOGLE_DEVELOPER_KEY=
+
+# For development on localhost set value to 1
+#OAUTHLIB_INSECURE_TRANSPORT=1
+#PORT=5004