updates to lib

hogumathi_app/__init__.py

@@ -129,6 +129,13 @@ else:
     visjs_enabled = False
 
 
+if find_spec('socialdata_facade'):
+    import socialdata_facade
+    socialdata_enabled = True
+else:
+    print('socialdata module not found.')
+    socialdata_enabled = False
+
 """
 By default we use embed because it's less likely to get banned, because it does not
 use scraping. Once we've used scraping for a while and not been banned we can switch to that.
@@ -161,7 +168,39 @@ def record_content_analytics (content_id, content):
     else:
         print(f'record_content_analytics: unknown type: {content_id}')
 
+def browser_native_messaging ():
+    # stdin = binary stream. ((payload length, payload), ...)
+    
+    print('hey. awaiting msg.')
+    
+    # "Note To write or read binary data from/to the standard streams, 
+    # use the underlying binary buffer object. For example, to write bytes to stdout, use sys.stdout.buffer.write(b'abc')."
+    # https://docs.python.org/3/library/sys.html#sys.stdin
+    
+    stdin = sys.stdin
+    
+    msg_len = read_int(stdin.buffer) # example read_int is in Apple Notes exporter code
+    while msg_len:
+        
+        print(f'new msg_len: {msg_len}')
+        
+        msg_bytes = stdin.buffer.read(msg_len)
+        msg = json.loads(msg_bytes.decode('utf-8'))
+        
+        print(msg)
+        
+        print('awaiting next msg...')
+        msg_len = read_int(stdin.buffer)
+        
+        return 0
+
 if __name__ == '__main__':
+    if len(sys.argv) >= 2 and 	sys.argv[1] == 'browser_native_messaging':
+        print('browser_native_messaging')
+        
+        exit(browser_native_messaging())
+        
+    
     glitch_enabled = os.environ.get('PROJECT_DOMAIN') and True
     
     t = os.environ.get('BEARER_TOKEN')
@@ -335,7 +374,10 @@ if __name__ == '__main__':
         
     if bitchute_enabled:
         bitchute_facade.register_content_sources()
-        
+    
+    if socialdata_enabled:
+        socialdata_facade.register_content_sources()
+    
     #CORS(api)
     
     sched_app = h_sched.ScheduleApplication()

hogumathi_app/content_system.py

@@ -6,6 +6,11 @@ Generally a source returns a CollectionPage or individual items.
 
 At present many sources return a List of Maps because the design is being discovered and solidified as it makes sense rather than big design up front.
 
+Sources are sorted by weight and then checked for prefix and ID match,
+And the first source to return content in that order is used.
+Eg. online sources return no content for some reason so a cache is used.
+
+
 May end up similar to Android's ContentProvider, found later. I was 
 thinking about using content:// URI scheme.
 

hogumathi_app/item_collections.py

@@ -68,7 +68,7 @@
 	</p>
 	{% if tweet.actions.view_replied_tweet %}
 	<p class="reply_to w-100" style="border: 1px solid silver; padding: 6px">
-		<a href="{{ url_for(tweet.actions.view_replied_tweet.route, **tweet.actions.view_replies.route_params) }}">View in Thread</a>.
+		<a href="{{ tweet.actions.view_replied_tweet.url }}">View in Thread</a>.
 	</p>
 	{% endif %}
 	{% endif %}

hogumathi_app/templates/partial/tweet-thread-children.html

@@ -144,7 +144,7 @@ function feed_item_to_activity (fi) {
 		
 		
 		{% if tweet.actions.view_replies %}
-		<a href="{{ url_for(tweet.actions.view_replies.route, **tweet.actions.view_replies.route_params) }}">replies</a>
+		<a href="{{ tweet.actions.view_replies.url }}">replies</a>
 		|
 		{% endif %}
 		
@@ -154,31 +154,31 @@ function feed_item_to_activity (fi) {
 		{% if show_thread_controls and tweet.conversation_id %}
 		{% with tweet=tweets[0] %}
 		{% if tweet.actions.view_thread %}
-		<a href="{{ url_for(tweet.actions.view_thread.route, **tweet.actions.view_thread.route_params) }}">author thread</a>
+		<a href="{{ tweet.actions.view_thread.url }}">author thread</a>
 		|
 		{% endif %}
 		{% if tweet.actions.view_conversation %}
-		<a href="{{ url_for(tweet.actions.view_conversation.route, **tweet.actions.view_conversation.route_params) }}">full convo</a>
+		<a href="{{ tweet.actions.view_conversation.url }}">full convo</a>
 		|
 		{% endif %}
 		{% endwith %}
 		{% endif %}
 		
 		{% if tweet.actions.view_activity %}
-		<a href="{{ url_for(tweet.actions.view_activity.route, **tweet.actions.view_activity.route_params) }}">activity</a>
+		<a href="{{ tweet.actions.view_activity.url }}">activity</a>
 		|
 		{% endif %}
 		
 		{% if tweet.actions.retweet %}
-		<a hx-post="{{ url_for(tweet.actions.retweet.route, **tweet.actions.retweet.route_params) }}">retweet</a>
+		<a hx-post="{{ tweet.actions.retweet.url }}">retweet</a>
 		|
 		{% endif %}
 		
 		{% if tweet.actions.bookmark %}
-		<a hx-post="{{ url_for(tweet.actions.bookmark.route, **tweet.actions.bookmark.route_params) }}">bookmark</a>
+		<a hx-post="{{ tweet.actions.bookmark.url }}">bookmark</a>
 		{% if tweet.actions.delete_bookmark %}
 		[
-		<a hx-delete="{{ url_for(tweet.actions.delete_bookmark.route, **tweet.actions.delete_bookmark.route_params) }}">-</a>
+		<a hx-delete="{{ tweet.actions.delete_bookmark.url }}">-</a>
 		]
 		{% endif %}
 		|

hogumathi_app/templates/partial/user-card.html

@@ -1,3 +1,6 @@
+from dataclasses import dataclass
+from typing import List, Dict, Optional
+
 from hashlib import sha256
 import os
 from pathlib import Path
@@ -49,7 +52,7 @@ def index ():
 
 @api.get('/img')
 def get_image ():
-	print('GET IMG')
+	print('GET IMG: FIXME this should be done with caching proxy')
 	url = request.args['url']
 	url_hash = sha256(url.encode('utf-8')).hexdigest()
 	path = f'.data/cache/media/{url_hash}'
@@ -180,6 +183,107 @@ def get_schedule_create_job_html ():
     
     return render_template_string(template, **view_model)
 
+@dataclass
+class FeedItemMedia:
+    url: str
+    mimetype: str
+
+@dataclass
+class FeedItem:
+    id: str
+    text: str
+    media: Optional[List[FeedItemMedia]] = None
+
+def ingest_feed_item (feed_item: FeedItem) -> FeedItem:
+    with sqlite3.connect('.data/ingest.db') as db:
+        with db.cursor() as cur:
+            #cur.row_factory = sqlite3.Row
+            
+            feed_item_table_exists = False # example in Hogumathi, twitter archive plugin I think
+            if not feed_item_table_exists:
+                cur.execute("""
+                    create table feed_item (
+                        id text,
+                        text text
+                    )
+                    
+                """)
+                cur.execute("""
+                    create table feed_item_media (
+                        feed_item_id integer,
+                        url text,
+                        mimetype text
+                    )
+                    
+                """)
+            sql = 'insert into feed_item (id, text) values (?, ?)'
+            params = [
+                feed_item.id,
+                feed_item.text
+                ]
+            
+            res = cur.execute(sql, params)
+            
+            if not res:
+                print('could not ingest feed_item')
+                return False
+            
+
+            
+            return feed_item
+
+
+def ingest_feed_item_media (feed_item: FeedItem) -> FeedItem:
+    print('ingest_feed_item_media')
+    
+    if not feed_item.media:
+        return feed_item
+    
+    with sqlite3.connect('.data/ingest.db') as db:
+        with db.cursor() as cur:
+            #cur.row_factory = sqlite3.Row
+            
+            for media_item in feed_item.media:
+                
+                # TODO import URL to files app and store that URL.
+                # may want to support several URLs, so that offline LANs work.
+                
+                sql = 'insert into feed_item_media (feed_item_id, url, mimetype) values (?, ?, ?)'
+                params = [
+                    feed_item.id,
+                    media_item.url,
+                    media_item.mimetype
+                    ]
+                res = cur.execute(sql, params)
+                
+                if not res:
+                    print('could not ingest feed_item_media')
+    
+    return feed_item
+    
+
+@api.post('/api/ingest/feed-item')
+def api_ingest_feed_item ():
+    """
+    Eventually other content sources with ingest here, and this will be the main DB.
+    Via inigest_feed_Item and ingest_feed_item_media.
+    
+    They could be worker tasks. Work when submitted directly from browser extension.
+    """
+    print('api_ingest_feed_item')
+    ingest_media = int(request.args.get('ingest_media', 1))
+    feed_item = request.args.get('feed_item') # FIXME might want to use post body/form
+    feed_item = from_dict(data_class=FeedItem, data=feed_item)
+    
+    fi_i_res = ingest_feed_item(feed_item)
+    
+    if ingest_media: # and fi_i_res(blocking=True, timeout=5):
+        fi_i_media_res = ingest_feed_item_media(feed_item)
+        
+        #fi_i_media_res(blocking=True)
+    
+    return 'ok'
+
 @api.get('/health')
 def get_health ():
     return 'ok'

lib/email_source.py

@@ -0,0 +1,123 @@
+from dataclasses import dataclass, asdict
+from typing import Optional, List, Dict, Union
+from dacite import from_dict
+
+from email.utils import getaddresses, parseaddr
+
+import json
+import requests
+
+import messaging_support.types as hm_types
+
+
+class RmConversationContext:
+    
+    def add_routed_message (self, rm):
+        
+        references = eml.get('References')
+        
+        
+        return
+        
+class EmailConversationContext:
+    
+    def add_email (self, eml):
+        
+        # the first reference is the conversation starter
+        # the last is the message this is a reply to
+        # the middle are immediate replies, up to about 10 deep--
+        # ie. some of the chain may be missing; we can get the oldest message
+        #     and recursively rebuild upward
+        references = eml.get('References')
+        
+        
+        return
+
+
+def addr_to_ra (addr):
+    return {'name': addr[0], 'address': addr[1]}
+
+class EmailSource:
+    
+    def __init__ (self, host, user, password):
+        self.host = host
+        self.user = user
+        self.password = password
+    
+    def get_page (self, page=0, limit=10):
+        
+        resp = requests.get(f'http://localhost:5010/data/emails/imap?page={page}')
+        if resp.status_code != 200:
+            raise Exception(f'error: {resp.status_code}')
+        
+        resp_json = json.loads(resp.text)
+        
+        print(resp_json)
+        
+        # check_types=False because the string "None" creates a problem...
+        
+        root_key = resp_json['metaData']['root']
+        fields = resp_json['metaData']['fields']
+        rows = resp_json[ root_key ]
+        
+        data = []
+        for i, row in enumerate(rows):
+            msg_id = 'imap-' + str(i + (limit * page))
+            record = {'msg_id': msg_id, **{k: row[j] for j, k in enumerate(fields)}}
+            
+            record['conversation_id'] = msg_id
+            record['to'] = list(map(addr_to_ra, getaddresses([record['to']])))
+            record['from_'] = list(map(addr_to_ra, getaddresses([record['from']])))[0]
+            del record['from']
+            
+            rm = from_dict(data=record, data_class=hm_types.RoutedMessage)
+            
+            data.append(rm)
+        
+        routed_messages = data
+        
+        return routed_messages
+        
+    
+    def get_page2 (self, page=0, limit=10):
+        token = json.dumps({
+            'host': self.host,
+            'user': self.user,
+            'password': self.password
+        })
+        
+        headers = {
+            'Authorization': token
+        }
+        
+        resp = requests.get(f'http://localhost:5010/data/emails/imap2?page={page}', headers=headers)
+        if resp.status_code != 200:
+            raise Exception(f'error: {resp.status_code}')
+        
+        resp_json = json.loads(resp.text)
+        
+        routed_messages = list(map(lambda rm_dict: from_dict(data=rm_dict, data_class=hm_types.RoutedMessage), resp_json))
+        
+        return routed_messages
+
+    def send_message (self, to, subject, content):
+        
+        token = json.dumps({
+            'host': self.host,
+            'user': self.user,
+            'password': self.password
+        })
+        
+        headers = {
+            'Authorization': token
+        }
+        
+        form_data = {
+            'to': to,
+            'subject': subject,
+            'text': content
+        }
+        
+        resp = requests.post('http://localhost:5010/api/send-message', data=form_data, headers=headers)
+        
+        return resp

lib/flask_bootstrap5/__init__.py

@@ -0,0 +1,7 @@
+from flask import Blueprint
+
+bootstrap5_bp = Blueprint('bootstrap5', __name__,
+    static_folder='static',
+    static_url_path='',
+    template_folder='templates',
+    url_prefix='/lib/bootstrap5')

lib/flask_bootstrap5/templates/bootstrap5/partial/head.html

@@ -0,0 +1,8 @@
+
+{% with bootstrap5_module_path=bootstrap5_module_path or 'bootstrap5' %}
+<link href="{{ url_for(bootstrap5_module_path + '.static', filename='bootstrap.min.css') }}" rel="stylesheet" integrity="sha384-GLhlTQ8iRABdZLl6O3oVMWSktQOp6b7In1Zl3/Jr59b6EGGoI1aFkw7cmDA6j6gD" crossorigin="anonymous">
+<script src="{{ url_for(bootstrap5_module_path + '.static', filename='bootstrap.bundle.min.js') }}" integrity="sha384-w76AqPfDkMBDXo30jS1Sgez6pr3x5MlQ1ZAGC+nuZB+EYdgRZgiwxhTBTkF7CXvN" crossorigin="anonymous"></script>
+
+<link rel="stylesheet" href="{{ url_for(bootstrap5_module_path + '.static', filename='bootstrap-icons.css') }}">
+
+{% endwith %}

lib/mastodon_v2/api.py

@@ -0,0 +1,49 @@
+"""
+ViewModel oriented view of messages. Both DMs and Email should be supported from any provider,
+via the RoutedMessage abstraction.
+"""
+
+from dataclasses import dataclass
+from typing import Optional, List, Dict, Union
+
+Url = str
+Date = str
+
+MessageId = str
+
+@dataclass
+class RoutedAddress:
+    address: str
+    name: Optional[str]
+    avi_icon_url: Optional[Url] = None
+
+@dataclass 
+class MessageAttachment:
+    content_type: str
+    name: Optional[str] = None
+    size: Optional[int] = None
+    preview_url: Optional[Url] = None
+    url: Optional[Url] = None
+
+@dataclass
+class RoutedMessage:
+    to: List[RoutedAddress]
+    from_: RoutedAddress
+    date: Date
+    msg_id: MessageId
+    conversation_id: MessageId
+    in_reply_to_id: Optional[MessageId] = None
+    subject: Optional[str] = None
+    headers: Optional[Dict[str, str]] = None
+    cc: Optional[List[RoutedAddress]] = None
+    text: Optional[str] = None
+    html: Optional[str] = None
+    attachments: Optional[List[MessageAttachment]] = None # content type, name, size (fetch by index)
+    #references: Optional[List[Tuple[str,MessageId]]] = None # type, msg_id. type = forwarded, replied, quoted. Nostr "e" tag, basically.
+    #raw_bytes: Optional[bytes] = None
+
+@dataclass
+class Conversation:
+    id: str
+    participants: List[RoutedAddress]
+    messages: List[RoutedMessage] = None