Verified Commit 7a9941be authored by Markus Shepherd's avatar Markus Shepherd 🙈
Browse files

fixed typo (again); smarter fields detection

parent ac288176
......@@ -170,7 +170,7 @@ class GameItem(TypedItem):
serializer=JSON_SERIALIZER,
parser=parse_json,
)
image_url_dowload = Field(serializer=JSON_SERIALIZER, parser=parse_json)
image_url_download = Field(serializer=JSON_SERIALIZER, parser=parse_json)
image_file = Field(serializer=JSON_SERIALIZER, parser=parse_json)
video_url = Field(
dtype=list,
......@@ -509,7 +509,7 @@ class UserItem(TypedItem):
serializer=JSON_SERIALIZER,
parser=parse_json,
)
image_url_dowload = Field(serializer=JSON_SERIALIZER, parser=parse_json)
image_url_download = Field(serializer=JSON_SERIALIZER, parser=parse_json)
image_file = Field(serializer=JSON_SERIALIZER, parser=parse_json)
published_at = Field(
......
......@@ -219,17 +219,20 @@ class LimitImagesPipeline:
def process_item(self, item, spider):
"""Copy a limited number of image URLs to be downloaded from source to target."""
values = (
arg_to_iter(item.get(self.source_field))
if self.limit is None or self.limit < 0 # copy through everything
else ()
if not self.limit # limit is zero
else islice(arg_to_iter(item.get(self.source_field)), self.limit)
)
# adding target field would result in error; return item as-is
if hasattr(item, "fields") and self.target_field not in item.fields:
return item
if self.limit is None or self.limit < 0: # copy through everything
item[self.target_field] = list(arg_to_iter(item.get(self.source_field)))
return item
try:
item[self.target_field] = list(values)
except Exception:
pass
if not self.limit: # limit is zero
item[self.target_field] = []
return item
# actual limit
item[self.target_field] = list(
islice(arg_to_iter(item.get(self.source_field)), self.limit)
)
return item
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment