add excerpt and scraper details.

This commit is contained in:
Jason Schwarzenberger 2020-12-03 16:41:27 +13:00
parent d4260feb72
commit 2a2bf4d671
6 changed files with 76 additions and 14 deletions

View File

@ -76,14 +76,14 @@ def get_article(url):
if scraper not in scrapers.keys():
continue
try:
html = scrapers[scraper].get_html(url)
if html:
return html
details = scrapers[scraper].get_details(url)
if details and details.get('content'):
return details, scraper
except KeyboardInterrupt:
raise
except:
pass
return ''
return None, None
def get_content_type(url):
try:
@ -143,7 +143,12 @@ def update_story(story, is_manual=False, urlref=None):
return False
logging.info('Getting article ' + story['url'])
story['text'] = get_article(story['url'])
details, scraper = get_article(story['url'])
if not details: return False
story['text'] = details.get('content', '')
story['excerpt'] = details.get('excerpt', '')
story['scraper'] = scraper
story['scraper_link'] = details.get('scraper_link', '')
if not story['text']: return False
return True

View File

@ -12,9 +12,28 @@ def get_html(url):
details = get_details(url)
if not details:
return ''
return details['html']
return details['content']
def get_details(url):
outline = _get_outline(url)
if not outline:
return None
return as_readable(outline)
def as_readable(details):
readable = {
'title': details['title'],
'byline': details['author'],
'content': details['html'],
'excerpt': _excerpt(details),
'siteName': details['site_name'],
'url': details['article_url'],
'publisher': details['site_name'],
'scraper_link': 'https://outline.com/' + details['short_code']
}
return readable
def _get_outline(url):
try:
logging.info(f"Outline Scraper: {url}")
params = {'source_url': url}
@ -35,3 +54,10 @@ def get_details(url):
except BaseException as e:
logging.error('Problem outlining article: {}'.format(str(e)))
return None
def _excerpt(details):
meta = details.get('meta')
if not meta: return ''
if meta.get('description'): return meta.get('description', '')
if not meta.get('og'): return ''
return meta.get('og').get('og:description', '')

View File

@ -1,9 +1,8 @@
<script>
import StoryInfo from "../components/StoryInfo.svelte";
import StoryMeta from "../components/StoryMeta.svelte";
export let story;
let host = new URL(story.url || story.link).hostname.replace(/^www\./, "");
</script>
<style>
@ -63,12 +62,13 @@
<h1 class="article-title">
{@html story.title}
</h1>
{#if story.url}
<div>source: <a class="article-source" href={story.url}>{host}</a></div>
{/if}
<section class="article-info">
<StoryInfo {story} />
</section>
<aside class="article-info">
<StoryMeta {story} />
</aside>
</header>
<section class="article-body">

View File

@ -48,9 +48,9 @@
class="story-source"
href={story.url || story.link}>{host(story.url || story.link)}</a>
</header>
<section class="story-info">
<aside class="story-info">
<StoryInfo {story} />
</section>
</aside>
</article>
{/each}

View File

@ -0,0 +1,30 @@
<script>
export let story;
let host = new URL(story.url || story.link).hostname.replace(/^www\./, "");
</script>
<style>
ul {
margin: 0;
padding: 0;
}
li {
display: inline-block;
list-style-type: circle;
}
li:not(:first-of-type)::before {
content: " | ";
}
</style>
<ul>
{#if story.url}
<li>source: <a class="article-source" href={story.url}>{host}</a></li>
{/if}
{#if story.scraper && story.scraper_link}
<li>scraper: <a href={story.scraper_link}>{story.scraper}</a></li>
{:else if story.scraper}
<li>scraper: {story.scraper}</li>
{/if}
</ul>

View File

@ -42,6 +42,7 @@
property="article:published_time"
content={fromUnixTime(story.date).toISOString()} />
<meta property="article:author" content={story.author || story.source} />
<meta property="og:description" content={story.excerpt || story.title} />
</svelte:head>
<section class="single">