import os
import praw
import requests
from redvid import Downloader
import youtube_dl
import re
from datetime import datetime
try:
from logindata import REDDIT_USERNAME, REDDIT_PASSWORD
from logindata import REDDIT_CLIENT_ID, REDDIT_SECRET
except ImportError:
REDDIT_USERNAME = os.getenv("REDDIT_USERNAME")
REDDIT_PASSWORD = os.getenv("REDDIT_PASSWORD")
REDDIT_CLIENT_ID = os.getenv("REDDIT_CLIENT_ID")
REDDIT_SECRET = os.getenv("REDDIT_SECRET")
IMAGE_EXTENSIONS = ["gif", "gifv", "jpg", "jpeg", "png"]
VIDEO_EXTENSIONS = ["mp4"]
PLATFORMS = ["redgifs.com", "gfycat.com", "imgur.com", "youtube.com"]
def make_client():
"""Creates a PRAW client with the details in the secrets.py file."""
return praw.Reddit(
username=REDDIT_USERNAME,
password=REDDIT_PASSWORD,
client_id=REDDIT_CLIENT_ID,
client_secret=REDDIT_SECRET,
user_agent="reddit-save",
)
def get_saved_posts(client):
"""Gets a list of posts that the user has saved."""
return [
saved for saved in client.user.me().saved(limit=None)
if saved.__class__.__name__ == "Submission"
]
def get_upvoted_posts(client):
"""Gets a list of posts that the user has upvoted."""
return [
upvoted for upvoted in client.user.me().upvoted(limit=None)
if upvoted.__class__.__name__ == "Submission"
]
def get_saved_comments(client):
"""Gets a list of comments that the user has saved."""
return [
saved for saved in client.user.me().saved(limit=None)
if saved.__class__.__name__ != "Submission"
]
def get_post_html(post):
"""Takes a post object and creates a HTML for it - but not including the
preview HTML."""
with open(os.path.join("html", "post-div.html"), encoding="utf-8") as f:
html = f.read()
dt = datetime.utcfromtimestamp(post.created_utc)
html = html.replace("", post.title)
html = html.replace("", f"/r/{str(post.subreddit)}")
html = html.replace("", f"/u/{post.author.name}" if post.author else "[deleted]")
html = html.replace("", f"posts/{post.id}.html")
html = html.replace("", f"https://reddit.com{post.permalink}")
html = html.replace("", post.url)
html = html.replace("", post.id)
html = html.replace("", (post.selftext_html or "").replace(
'", str(dt))
html = html.replace("", dt.strftime("%d %B, %Y"))
return html
def save_media(post, location):
"""Takes a post object and tries to download any image/video it might be
associated with. If it can, it will return the filename."""
url = post.url
stripped_url = url.split("?")[0]
if url.endswith(post.permalink): return None
# What is the key information?
extension = stripped_url.split(".")[-1].lower()
domain = ".".join(post.url.split("/")[2].split(".")[-2:])
readable_name = list(filter(bool, post.permalink.split("/")))[-1]
# If it's an imgur gallery, forget it
if domain == "imgur.com" and "gallery" in url: return None
# Can the media be obtained directly?
if extension in IMAGE_EXTENSIONS + VIDEO_EXTENSIONS:
filename = f"{readable_name}_{post.id}.{extension}"
response = requests.get(post.url)
media_type = response.headers.get("Content-Type", "")
if media_type.startswith("image") or media_type.startswith("video"):
with open(os.path.join(location, "media", filename), "wb") as f:
f.write(response.content)
return filename
# Is this a v.redd.it link?
if domain == "redd.it":
downloader = Downloader(max_q=True, log=False)
downloader.url = url
current = os.getcwd()
try:
name = downloader.download()
extension = name.split(".")[-1]
filename = f"{readable_name}_{post.id}.{extension}"
os.rename(name, os.path.join(location, "media", filename))
return filename
except:
os.chdir(current)
return None
# Is it a gfycat link that redirects? Update the URL if possible
if domain == "gfycat.com":
html = requests.get(post.url).content
if len(html) < 50000:
match = re.search(r"http([\dA-Za-z\+\:\/\.]+)\.mp4", html.decode())
if match:
url = match.group()
else:
return None
# Is this an imgur image?
if domain == "imgur.com" and extension != "gifv":
for extension in IMAGE_EXTENSIONS:
direct_url = f'https://i.{url[url.find("//") + 2:]}.{extension}'
direct_url = direct_url.replace("i.imgur.com", "imgur.com")
direct_url = direct_url.replace("m.imgur.com", "imgur.com")
response = requests.get(direct_url)
if response.status_code == 200:
filename = f"{readable_name}_{post.id}.{extension}"
with open(os.path.join(location, "media", filename), "wb") as f:
f.write(response.content)
return filename
# Try to use youtube_dl if it's one of the possible domains
if domain in PLATFORMS:
options = {
"nocheckcertificate": True, "quiet": True, "no_warnings": True,
"ignoreerrors": True,
"outtmpl": os.path.join(
location, "media", f"{readable_name}_{post.id}" + ".%(ext)s"
)
}
with youtube_dl.YoutubeDL(options) as ydl:
try:
ydl.download([url])
except:
pass
for f in os.listdir(os.path.join(location, "media")):
if f.startswith(f"{readable_name}_{post.id}"):
return f
def add_media_preview_to_html(post_html, media):
"""Takes post HTML and returns a modified version with the preview
inserted."""
extension = media.split(".")[-1]
location = "/".join(["media", media])
if extension in IMAGE_EXTENSIONS:
return post_html.replace(
"",
f''
)
if extension in VIDEO_EXTENSIONS:
return post_html.replace(
"",
f''
)
return post_html
def create_post_page_html(post, post_html):
"""Creates the HTML for a post's own page."""
with open(os.path.join("html", "post.html"), encoding="utf-8") as f:
html = f.read()
html = html.replace("", post.title)
html = html.replace("", post_html.replace("h2>", "h1>").replace(
'", f"")
with open(os.path.join("html", "main.js"), encoding="utf-8") as f:
html = html.replace("", f"")
comments_html = []
post.comments.replace_more(limit=0)
for comment in post.comments:
comments_html.append(get_comment_html(
comment, op=post.author.name if post.author else None
))
html = html.replace("", "\n".join(comments_html))
return html
def get_comment_html(comment, children=True, op=None):
"""Takes a post object and creates a HTML for it - it will get its children
too unless you specify otherwise."""
with open(os.path.join("html", "comment-div.html"), encoding="utf-8") as f:
html = f.read()
dt = datetime.utcfromtimestamp(comment.created_utc)
author = "[deleted]"
if comment.author:
if comment.author == op:
author = f'/u/{comment.author.name}'
else:
author = f"/u/{comment.author.name}"
html = html.replace("", author)
html = html.replace("", (comment.body_html or "").replace(
'", str(comment.score))
html = html.replace("", f"https://reddit.com{comment.permalink}")
html = html.replace("", str(dt))
html = html.replace("", comment.id)
html = html.replace("", dt.strftime("%H:%M - %d %B, %Y"))
if children:
children_html = []
for child in comment.replies:
children_html.append(get_comment_html(child, children=False, op=op))
html = html.replace("", "\n".join(children_html))
return html