add excerpt and scraper details.

apiserver/feed.py

@@ -76,14 +76,14 @@ def get_article(url):
         if scraper not in scrapers.keys():
             continue
         try:
-            html = scrapers[scraper].get_html(url)
-            if html:
-                return html
+            details = scrapers[scraper].get_details(url)
+            if details and details.get('content'):
+                return details, scraper
         except KeyboardInterrupt:
             raise
         except:
             pass
-    return ''
+    return None, None
 
 def get_content_type(url):
     try:
@@ -143,7 +143,12 @@ def update_story(story, is_manual=False, urlref=None):
             return False
 
         logging.info('Getting article ' + story['url'])
-        story['text'] = get_article(story['url'])
+        details, scraper = get_article(story['url'])
+        if not details: return False
+        story['text'] = details.get('content', '')
+        story['excerpt'] = details.get('excerpt', '')
+        story['scraper'] = scraper
+        story['scraper_link'] = details.get('scraper_link', '')
         if not story['text']: return False
 
     return True

apiserver/scrapers/outline.py

@@ -12,9 +12,28 @@ def get_html(url):
     details = get_details(url)
     if not details:
         return ''
-    return details['html']
+    return details['content']
 
 def get_details(url):
+    outline = _get_outline(url)
+    if not outline:
+        return None
+    return as_readable(outline)
+
+def as_readable(details):
+    readable = {
+        'title': details['title'],
+        'byline': details['author'],
+        'content': details['html'],
+        'excerpt': _excerpt(details),
+        'siteName': details['site_name'],
+        'url': details['article_url'],
+        'publisher': details['site_name'],
+        'scraper_link': 'https://outline.com/' + details['short_code']
+    }
+    return readable
+
+def _get_outline(url):
     try:
         logging.info(f"Outline Scraper: {url}")
         params = {'source_url': url}
@@ -35,3 +54,10 @@ def get_details(url):
     except BaseException as e:
         logging.error('Problem outlining article: {}'.format(str(e)))
         return None
+
+def _excerpt(details):
+    meta = details.get('meta')
+    if not meta: return ''
+    if meta.get('description'): return meta.get('description', '')
+    if not meta.get('og'): return ''
+    return meta.get('og').get('og:description', '')

webapp/src/components/Article.svelte

@@ -1,9 +1,8 @@
 <script>
   import StoryInfo from "../components/StoryInfo.svelte";
+  import StoryMeta from "../components/StoryMeta.svelte";
 
   export let story;
-
-  let host = new URL(story.url || story.link).hostname.replace(/^www\./, "");
 </script>
 
 <style>
@@ -63,12 +62,13 @@
     <h1 class="article-title">
       {@html story.title}
     </h1>
-    {#if story.url}
-      <div>source: <a class="article-source" href={story.url}>{host}</a></div>
-    {/if}
+
     <section class="article-info">
       <StoryInfo {story} />
     </section>
+    <aside class="article-info">
+      <StoryMeta {story} />
+    </aside>
   </header>
 
   <section class="article-body">

webapp/src/components/StoryList.svelte

@@ -48,9 +48,9 @@
         class="story-source"
         href={story.url || story.link}>{host(story.url || story.link)}</a>
     </header>
-    <section class="story-info">
+    <aside class="story-info">
       <StoryInfo {story} />
-    </section>
+    </aside>
   </article>
 {/each}
 

webapp/src/components/StoryMeta.svelte

@@ -0,0 +1,30 @@
+<script>
+  export let story;
+
+  let host = new URL(story.url || story.link).hostname.replace(/^www\./, "");
+</script>
+
+<style>
+  ul {
+    margin: 0;
+    padding: 0;
+  }
+  li {
+    display: inline-block;
+    list-style-type: circle;
+  }
+  li:not(:first-of-type)::before {
+    content: " | ";
+  }
+</style>
+
+<ul>
+  {#if story.url}
+    <li>source: <a class="article-source" href={story.url}>{host}</a></li>
+  {/if}
+  {#if story.scraper && story.scraper_link}
+    <li>scraper: <a href={story.scraper_link}>{story.scraper}</a></li>
+  {:else if story.scraper}
+    <li>scraper: {story.scraper}</li>
+  {/if}
+</ul>

webapp/src/routes/[id].svelte

@@ -42,6 +42,7 @@
     property="article:published_time"
     content={fromUnixTime(story.date).toISOString()} />
   <meta property="article:author" content={story.author || story.source} />
+  <meta property="og:description" content={story.excerpt || story.title} />
 </svelte:head>
 
 <section class="single">