diff --git a/content/scroll_down_content_page.py b/content/scroll_down_content_page.py index b8af7a6..7266cf5 100644 --- a/content/scroll_down_content_page.py +++ b/content/scroll_down_content_page.py @@ -13,7 +13,13 @@ def scroll_down_content_page(driver: webdriver.firefox.webdriver.WebDriver): with open(string_json_path, "r") as file: string_dict = json.load(file) - height = driver.execute_script("return document.body.scrollHeight") + max_height_per_step = driver.execute_script("return document.body.scrollHeight") + + final_y: int = 0 + while final_y == 0: + element = driver.find_elements(By.TAG_NAME, "div")[-1] + # final_x = element.location["x"] + final_y = element.location["y"] found_element_list = driver.find_elements(By.TAG_NAME, "h2") found_element = None @@ -22,10 +28,34 @@ def scroll_down_content_page(driver: webdriver.firefox.webdriver.WebDriver): found_element = element break assert found_element is not None + # start_x = found_element.location["x"] + start_y = found_element.location["y"] + print(start_y) + print(final_y) + print(max_height_per_step) + print("----") + to_scroll = final_y - start_y try: - actions = ActionChains(driver) - actions.scroll_from_origin(ScrollOrigin.from_element(found_element), 0, height) - actions.perform() + if to_scroll > 0: + actions = ActionChains(driver) + if to_scroll > max_height_per_step: + actions.scroll_from_origin( + ScrollOrigin.from_element(found_element), 0, max_height_per_step + ) + to_scroll -= max_height_per_step + + while to_scroll > 0: + if to_scroll > max_height_per_step: + actions.scroll_by_amount(0, max_height_per_step) + to_scroll -= max_height_per_step + else: + actions.scroll_by_amount(0, to_scroll) + to_scroll -= to_scroll + else: + actions.scroll_from_origin( + ScrollOrigin.from_element(found_element), 0, to_scroll + ) + actions.perform() except selenium.common.exceptions.MoveTargetOutOfBoundsException: pass