"""Extractors for https://berriz.in/""" import json from gallery_dl import exception, text from gallery_dl.extractor.common import Extractor, Message # /en, /ko, /zh-Hans, ... are optional, as is the link.berriz.in /web/main prefix BASE_PATTERN = ( r"(?:https?://)?(?:www\.|link\.)?berriz\.in" r"(?:/([a-z]{2}(?:-[A-Za-z]{2,4})?))?" r"(?:/web/main)?" r"/([^/?#]+)" ) UUID = r"[0-9a-fA-F-]{36}" class BerrizExtractor(Extractor): """Base class for berriz extractors""" category = "berriz" root = "https://berriz.in" cookies_domain = ".berriz.in" directory_fmt = ("{category}", "{community_name}", "{user_name}") filename_fmt = "{post_id}_{num:>02}_{filename}.{extension}" archive_fmt = "{post_id}_{media_id}" def __init__(self, match): Extractor.__init__(self, match) self.lang = match.group(1) or "en" self.community_key = match.group(2) def _init(self): self.api = BerrizAPI(self) self._community = None @property def community(self): """Resolve the community slug lazily _init() runs outside the job's exception handling, so any request made there would surface as an uncaught traceback. """ if self._community is None: self._community = self.api.community(self.community_key) return self._community def metadata(self, item): """Build the shared metadata dict for a feed item or post detail""" post = item["post"] writer = item.get("writer") or {} board = item.get("boardInfo") or {} return { "community_id": self.community["communityId"], "community_key": self.community["communityKey"], "community_name": self.community["communityName"], "board_id": board.get("boardId"), "board_name": board.get("name"), "post_id": post["postId"], "title": post.get("title") or "", "body": post.get("plainBody") or post.get("body") or "", "date": self.parse_datetime_iso(post.get("createdAt")), "date_updated": self.parse_datetime_iso(post.get("updatedAt")), "hashtags": post.get("hashtags") or [], "user_id": writer.get("userId") or post.get("userId"), "user_name": writer.get("name") or "", "artist_id": writer.get("communityArtistId"), "is_artist": writer.get("isArtist", False), } def _photos(self, item): return ((item.get("post") or {}).get("media") or {}).get("photo") or [] def items(self): for item in self.posts(): photos = self._photos(item) if not photos: # text-only posts have nothing to download self.log.debug("Skipping text-only post %s", item["post"]["postId"]) continue data = self.metadata(item) data["count"] = len(photos) yield Message.Directory, "", data for data["num"], photo in enumerate(photos, 1): url = photo["imageUrl"] meta = photo.get("imageMetadata") or {} data["media_id"] = photo.get("mediaId") data["width"] = meta.get("width") data["height"] = meta.get("height") text.nameext_from_url(url, data) yield Message.Url, url, data def posts(self): """Return an iterable of feed items""" return () class BerrizPostExtractor(BerrizExtractor): """Extractor for a single berriz post""" subcategory = "post" pattern = BASE_PATTERN + rf"/board/({UUID})/post/({UUID})" example = ("https://berriz.in/en/kiiikiii/board" "/0195c652-e4d8-dc0d-80c1-ae1d5b4df655/post" "/019fe016-c819-bd25-5486-24acbe731982/") def __init__(self, match): BerrizExtractor.__init__(self, match) self.post_id = match.group(4) def posts(self): return (self.api.post(self.community["communityId"], self.post_id),) class BerrizBoardExtractor(BerrizExtractor): """Extractor for all posts on a berriz board""" subcategory = "board" pattern = BASE_PATTERN + rf"/board/({UUID})/?(?:$|[?#])" example = ("https://berriz.in/en/kiiikiii/board" "/0195c652-e4d8-dc0d-80c1-ae1d5b4df655/") def __init__(self, match): BerrizExtractor.__init__(self, match) self.board_id = match.group(3) def posts(self): return self.api.board_feed(self.community["communityId"], self.board_id) class BerrizProfileExtractor(BerrizExtractor): """Extractor for all posts by a single berriz artist""" subcategory = "profile" pattern = BASE_PATTERN + r"/profile/(\d+)" example = "https://berriz.in/en/ive/profile/182041215545370/post/" def __init__(self, match): BerrizExtractor.__init__(self, match) self.user_id = match.group(3) def posts(self): # the archive feed only carries a single preview image per post, # so every post with media has to be fetched in full community_id = self.community["communityId"] for entry in self.api.user_posts(community_id, self.user_id): if not entry.get("imageCount"): continue try: yield self.api.post(community_id, entry["postId"]) except exception.AuthorizationError as exc: # a single gated post must not abort the whole profile self.log.warning("Skipping post %s: %s", entry["postId"], exc) class BerrizImageExtractor(Extractor): """Extractor for a single berriz CDN image""" category = "berriz" subcategory = "image" root = "https://statics.berriz.in" directory_fmt = ("{category}",) filename_fmt = "{filename}.{extension}" archive_fmt = "{filename}" # trailing /dims/... transformations are stripped to get the original pattern = (r"(?:https?://)?statics\.berriz\.in" r"/(cdn/[^?#]+?\.(?:jpe?g|png|gif|webp|avif))(?:/[^?#]*)?") example = ("https://statics.berriz.in/cdn/postmedia/image" "/vi/bo/bo/bf/an/vb/505141903.jpeg") def __init__(self, match): Extractor.__init__(self, match) self.path = match.group(1) def items(self): url = f"{self.root}/{self.path}" data = text.nameext_from_url(url, {}) yield Message.Directory, "", data yield Message.Url, url, data class BerrizAPI: """Interface for the berriz API""" ROOT = "https://svc-api.berriz.in" ACCOUNT_ROOT = "https://account.berriz.in" # public SPA client id, lifted from the site's JS bundle CLIENT_ID = "e8faf56c-575a-42d2-933d-7b2e279ad827" # the API answers with HTTP 200 and signals failure through 'code' SUCCESS = "0000" CODES_LOGIN = ("FS_CU9910", "FS_ER4020") CODES_FANCLUB = ("FS_CU9900",) # deleted/unknown post, unknown community, board and user CODES_NOTFOUND = ("FS_CU2050", "FS_CU9999", "FS_CU9000", "FS_CU9020") # the feed endpoints cap out here; anything larger is rejected outright PAGE_SIZE = 100 def __init__(self, extractor): self.extr = extractor root = extractor.root self.headers = { "Accept": "application/json", "Origin": root, "Referer": f"{root}/", } self.refresh_failed = False def _call(self, endpoint, params=None): params = dict(params or {}) params["languageCode"] = self.extr.lang for attempt in (1, 2): response = self.extr.request( f"{self.ROOT}{endpoint}", params=params, headers=self.headers, fatal=None) if response.status_code == 404: raise exception.NotFoundError(self.extr.subcategory) # an expired token yields HTTP 401 with a plain-text body, # so a non-JSON response is not necessarily fatal try: data = response.json() except ValueError: data = None code = data.get("code") if data else None if code == self.SUCCESS: return data["data"] message = ((data or {}).get("message") or " ".join(response.text.split())[:200] or f"HTTP {response.status_code}") # a stale bz_a breaks even the endpoints that work anonymously if response.status_code == 401 or code in self.CODES_LOGIN: # scoped to this request, so a token that expires partway # through a long crawl is still recovered if attempt == 1 and self._refresh_token(): continue if self.extr.cookies.get("bz_a", domain=self.extr.cookies_domain): hint = ("your berriz cookies are expired - re-export them, " "or omit the 'cookies' option entirely since public " "posts need no account") else: hint = ("log in at berriz.in and provide your cookies " "via the 'cookies' option") raise exception.AuthorizationError(f"{message} - {hint}") if code in self.CODES_FANCLUB: raise exception.AuthorizationError(message) if code in self.CODES_NOTFOUND: raise exception.NotFoundError(self.extr.subcategory) self.extr.log.debug(data or response.text) # AbortExtraction, not StopExtraction: the latter takes a *target* # subcategory rather than a message and is logged as a clean stop, # which would drop the error and exit 0 raise exception.AbortExtraction(f"API request failed: {message}") def _refresh_token(self): """Trade the bz_r refresh cookie for a fresh bz_a Only failures are remembered: a dead bz_r will not come back to life, but a successful refresh may need repeating on a crawl that outlives the one-hour access token. """ if self.refresh_failed: return False cookies = self.extr.cookies if not cookies.get("bz_r", domain=self.extr.cookies_domain): self.refresh_failed = True return False self.extr.log.debug("Refreshing access token") response = self.extr.request( f"{self.ACCOUNT_ROOT}/auth/v1/token:refresh", method="POST", headers={**self.headers, "Content-Type": "application/json"}, data=json.dumps({"clientId": self.CLIENT_ID}), fatal=None) try: # the session cookie jar absorbs the new bz_a from Set-Cookie if response.json().get("code") == self.SUCCESS: return True except ValueError: pass self.refresh_failed = True return False def community(self, community_key): return self._call(f"/service/v1/community/id/{community_key}") def post(self, community_id, post_id): return self._call(f"/service/v1/community/{community_id}/post/{post_id}") def board_feed(self, community_id, board_id): endpoint = f"/service/v1/community/{community_id}/boards/{board_id}/feed" return self._pagination(endpoint) def user_posts(self, community_id, user_id): endpoint = f"/service/v1/community/archive/user/{user_id}/posts" return self._pagination(endpoint, {"communityId": community_id}) def _pagination(self, endpoint, params=None): params = dict(params or {}) params["pageSize"] = self.PAGE_SIZE while True: data = self._call(endpoint, params) contents = data.get("contents") if not contents: return yield from contents if not data.get("hasNext"): return cursor = (data.get("cursor") or {}).get("next") if not cursor: return params["next"] = cursor