""" A registry for content sources that work in terms of the View Model (view_model.py). Generally a source returns a CollectionPage or individual items. At present many sources return a List of Maps because the design is being discovered and solidified as it makes sense rather than big design up front. May end up similar to Android's ContentProvider, found later. I was thinking about using content:// URI scheme. https://developer.android.com/reference/android/content/ContentProvider Could also be similar to a Coccoon Generator https://cocoon.apache.org/1363_1_1.html Later processing in Python: https://www.innuy.com/blog/build-data-pipeline-python/ https://www.bonobo-project.org/ """ import re import inspect class ContentSystem: def __init__ (self): self.content_sources = {} self.hooks = {} def register_content_source (self, id_prefix, content_source_fn, id_pattern='(\d+)', source_id=None, weight=None): if not source_id: source_id=f'{inspect.getmodule(content_source_fn).__name__}:{content_source_fn.__name__}' if weight == None: weight = 1000 - len(self.content_sources) print(f'register_content_source: {id_prefix}: {source_id} with ID pattern {id_pattern} (weight={weight})') self.content_sources[ source_id ] = [id_prefix, content_source_fn, id_pattern, source_id, weight] def find_content_id_args (self, id_pattern, content_id): id_args = re.fullmatch(id_pattern, content_id) if not id_args: return [], {} args = [] kwargs = id_args.groupdict() if not kwargs: args = id_args.groups() return args, kwargs def get_content (self, content_id, content_source_id=None, *extra_args, **extra_kwargs): print(f'get_content {content_id}') #source_ids = list(self.content_sources.keys()) #source_ids.sort(key=lambda id_prefix: len(id_prefix), reverse=True) source_ids = list(self.content_sources.keys()) source_ids.sort(key=lambda id_prefix: self.content_sources[id_prefix][4], reverse=True) # 4 = weight #print(source_ids) for source_id in source_ids: if content_source_id and source_id != content_source_id: continue [id_prefix, content_source_fn, id_pattern, source_id, weight] = self.content_sources[ source_id ] if not content_id.startswith(id_prefix): continue source_content_id = content_id[len(id_prefix):] print(f'get_content {content_id} from source {source_id}, resolves to {source_content_id} ( weight={weight})') args, kwargs = self.find_content_id_args(id_pattern, source_content_id) if id_prefix.endswith(':') and not args and not kwargs: continue if extra_args: args += extra_args if extra_kwargs: kwargs = {**extra_kwargs, **kwargs} # if we're calling a bulk source and only get back partial results... # we'd want to remove the found content IDs and merge until # we find them all... # yet we don't want intelligence about the type of content returned. # idea: class BulkResponse(dict): pass content = content_source_fn(*args, **kwargs) if content: self.invoke_hooks('got_content', content_id, content) return content def get_all_content (self, content_ids): """ Get content from all sources, using a grouping call if possible. Returns a map of source_id to to result; the caller needs to have the intelligence to merge and paginate. Native implementation is to juse make one call to get_content per ID, but we need to figure out a way to pass a list of IDs and pagination per source; for exampe a list of 100+ Tweet IDs and 100+ YT videos from a Swipe file. """ return self.get_all_content2(content_ids) def get_all_content2 (self, content_collection_ids, content_args = None, max_results = None): """ Takes a list of collection IDs and content_args is a map of (args, kwargs) keyed by collection ID. We could just use keys from content_args with empty values but that's a little confusing. Interleaving the next page of a source into existing results is a problem. Gracefully degraded could simply get the next page at the end of all pages and then view older content. We also need intelligence about content types, meaning perhaps some lambdas pass in. Eg. CollectionPage. See feeds facade for an example of merging one page. Seems like keeping feed items in a DB is becoming the way to go, serving things in order. Client side content merging might work to insert nodes above, eg. HTMx. Might be jarring to reader, so make optional. Append all new or merge. Cache feed between requests on disk, merge in memory, send merge/append result. """ bulk_prefixes = { #'twitter:tweet:': 'twitter:tweets', #'youtube:video:': 'youtube:videos', } bulk_requests = {} result = {} for content_id in content_collection_ids: is_bulk = False for bulk_prefix in bulk_prefixes: if content_id.startswith(bulk_prefix): bulk_content_id = bulk_prefixes[ bulk_prefix ] if not bulk_content_id in bulk_requests: bulk_requests[ bulk_content_id ] = [] bulk_requests[ bulk_content_id ].append(content_id) # max size for a content source... is_bulk = True if is_bulk: continue if content_args and content_id in content_args: extra_args, extra_kwargs = content_args[content_id] else: extra_args, extra_kwargs = [], {} result[ content_id ] = self.get_content(content_id, *extra_args, **extra_kwargs) for bulk_content_id, content_ids in bulk_requests.items(): print(f'bulk: {bulk_content_id}, content_ids: {content_ids}') bulk_response = self.get_content(bulk_content_id, content_ids=content_ids) # FIXME me=... workaround, provide bulk id in args map print(f'bulk_response: {bulk_response}') # we're not supposed to be smart about get_content response type... # does it return a map by convention? better than iterating something else. if bulk_response: for content_id, content in bulk_response.items(): if content: self.invoke_hooks('got_content', content_id, content) result.update(bulk_response) return result def register_hook (self, hook_type, hook_fn, *extra_args, **extra_kwargs): if not hook_type in self.hooks: self.hooks[hook_type] = [] self.hooks[hook_type].append([hook_fn, extra_args, extra_kwargs]) def invoke_hooks (self, hook_type, *args, **kwargs): if not hook_type in self.hooks: return for hook, extra_args, extra_kwargs in self.hooks[hook_type]: hook_args = args hook_kwargs = kwargs if extra_args: hook_args = args + extra_args if extra_kwargs: hook_kwargs = {**extra_kwargs, **hook_kwargs} hook(*hook_args, **hook_kwargs) #try: # hook(*args, **kwargs) #except TypeError as e: # print ('tried to call a hook with wrong args. no problem') # continue # The app was coded before we turned this into a class... # so we proxy calls with the old interface to this default instance. DEFAULT = ContentSystem() def reset (): print ('compat resetting content system') DEFAULT = ContentSystem() def register_content_source (id_prefix, content_source_fn, id_pattern='(\d+)', source_id=None, weight=None): print('compat register_content_source') return DEFAULT.register_content_source(id_prefix, content_source_fn, id_pattern, source_id) def get_content (content_id, content_source_id=None, *extra_args, **extra_kwargs): print('compat get_content') return DEFAULT.get_content(content_id, content_source_id, *extra_args, **extra_kwargs) def get_all_content (content_ids): print('compat get_all_content') return DEFAULT.get_all_content(content_ids) def register_hook (hook_type, hook_fn, *extra_args, **extra_kwargs): print('compat register_hook') return DEFAULT.register_hook(hook_type, hook_fn, *extra_args, **extra_kwargs)