Source code for cognite.extractorutils.throttle

#  Copyright 2021 Cognite AS
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  See the License for the specific language governing permissions and
#  limitations under the License.

Module containing tools for throttling extractors to avoid overloading a source system

import logging
from threading import Event
from time import time
from typing import Generator

[docs]def throttled_loop(target_time: int, cancelation_token: Event) -> Generator[None, None, None]: """ A loop generator that automatically sleeps until each iteration has taken the desired amount of time. Useful for when you want to avoid overloading a source system with requests. Example: This example will throttle printing to only print every 10th second: .. code-block:: python for _ in throttled_loop(10, stop_event): print("Hello every 10 seconds!") Args: target_time: How long (in seconds) an iteration should take om total cancelation_token: An Event object that will act as the stop event. When set, the loop will stop. Returns: A generator that will only yield when the target iteration time is met """ logger = logging.getLogger(__name__) while not cancelation_token.is_set(): start_time = time() yield iteration_time = time() - start_time if iteration_time > target_time: logger.warning("Iteration time longer than target time, will not sleep") else: logger.debug(f"Iteration took {iteration_time:.1f} s, sleeping {target_time - iteration_time:.1f} s") cancelation_token.wait(target_time - iteration_time)