Python源码示例:asyncio.Semaphore()

示例1
def interval_task(self):
        try:
            to_backup = self.bot.db.intervals.find({"next": {
                "$lt": datetime.utcnow()
            }})
            semaphore = Semaphore(10)
            async for interval in to_backup:
                async def run_interval():
                    try:
                        next = datetime.utcnow() + timedelta(minutes=interval["interval"])
                        await self.bot.db.intervals.update_one({"_id": interval["_id"]}, {"$set": {"next": next}})
                        await self.run_backup(interval["_id"])
                    finally:
                        semaphore.release()

                await semaphore.acquire()
                self.bot.loop.create_task(run_interval())
                await sleep(0)

        except Exception:
            pass 
示例2
def main(global_delay, local_delay, concurrency):
    global global_sleep, local_sleep, semaphore, index
    global_sleep = global_delay
    local_sleep = local_delay
    semaphore = asyncio.Semaphore(concurrency)
    print('Global delay =', global_delay)
    print('Local delay =', local_delay)
    print('Max. concurrency =', concurrency)
    print('Building inverted index...')
    index = build_index()

    app = web.Application()
    app.router.add_get('/', usage)
    app.router.add_get('/index/{word}', index_for)
    app.router.add_get('/name/{char}', char_name)

    print('Listening on port', PORT)
    web.run_app(app, port=PORT) 
示例3
def downloader_coro(loop, cc_list, base_url, verbose, concur_req):
    counter = collections.Counter()
    semaphore = asyncio.Semaphore(concur_req)
    async with aiohttp.ClientSession(loop=loop) as client:

        to_do = [download_one(client, cc, base_url, semaphore, verbose)
                 for cc in sorted(cc_list)]

        to_do_iter = asyncio.as_completed(to_do)
        if not verbose:
            to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))
        for future in to_do_iter:
            try:
                res = await future
            except FetchError as exc:
                country_code = exc.country_code
                try:
                    error_msg = exc.__cause__.args[0]
                except IndexError:
                    error_msg = exc.__cause__.__class__.__name__
                if verbose and error_msg:
                    msg = '*** Error for {}: {}'
                    print(msg.format(country_code, error_msg))
                status = HTTPStatus.error
            else:
                status = res.status

            counter[status] += 1

    return counter 
示例4
def __init__(self, in_state, in_forward, search_threads):
        self.noise_eps = 0.25
        self.dirichlet_alpha = 0.3    #0.03
        self.p_ = (1 - self.noise_eps) * 1 + self.noise_eps * np.random.dirichlet([self.dirichlet_alpha])
        self.root = leaf_node(None, self.p_, in_state)
        self.c_puct = 5    #1.5
        # self.policy_network = in_policy_network
        self.forward = in_forward
        self.node_lock = defaultdict(Lock)

        self.virtual_loss = 3
        self.now_expanding = set()
        self.expanded = set()
        self.cut_off_depth = 30
        # self.QueueItem = namedtuple("QueueItem", "feature future")
        self.sem = asyncio.Semaphore(search_threads)
        self.queue = Queue(search_threads)
        self.loop = asyncio.get_event_loop()
        self.running_simulation_num = 0 
示例5
def __init__(self, in_state, in_forward, search_threads):
        self.noise_eps = 0.25
        self.dirichlet_alpha = 0.3    #0.03
        self.p_ = (1 - self.noise_eps) * 1 + self.noise_eps * np.random.dirichlet([self.dirichlet_alpha])
        self.root = leaf_node(None, self.p_, in_state)
        self.c_puct = 5    #1.5
        # self.policy_network = in_policy_network
        self.forward = in_forward
        self.node_lock = defaultdict(Lock)

        self.virtual_loss = 3
        self.now_expanding = set()
        self.expanded = set()
        self.cut_off_depth = 30
        # self.QueueItem = namedtuple("QueueItem", "feature future")
        self.sem = asyncio.Semaphore(search_threads)
        self.queue = Queue(search_threads)
        self.loop = asyncio.get_event_loop()
        self.running_simulation_num = 0 
示例6
def _check_uris_async(
    links_to_check: Iterable[str], max_threads: int = 10, delay: float = 0
) -> Iterable[UrlResult]:
    tasks = []
    # create instance of Semaphore
    sem = asyncio.Semaphore(max_threads)

    # Create client session that will ensure we dont open new connection
    # per each request.
    async with ClientSession() as session:
        for uri in links_to_check:
            if delay:
                asyncio.sleep(delay)
            # pass Semaphore and session to every GET request
            task = asyncio.ensure_future(_check_uri_with_sem_async(sem, uri, session))
            tasks.append(task)

        results = await asyncio.gather(*tasks)
        return results 
示例7
def __init__(self, threads: int = None) -> None:
        try:  # get or create loop (threads don't have one)
            #: our asyncio loop
            self.loop = asyncio.get_event_loop()
        except RuntimeError:
            self.loop = asyncio.new_event_loop()
            asyncio.set_event_loop(self.loop)
        #: number of threads to use
        self.threads = threads or threads_to_use()
        #: semaphore to limit io parallelism
        self.io_sem: asyncio.Semaphore = asyncio.Semaphore(1)
        #: must never run more than one conda at the same time
        #: (used by PyPi when running skeleton)
        self.conda_sem: asyncio.Semaphore = asyncio.Semaphore(1)
        #: the filters successively applied to each item
        self.filters: List[AsyncFilter] = []
        #: executor running things in separate python processes
        self.proc_pool_executor = ProcessPoolExecutor(self.threads)

        self._shutting_down = False 
示例8
def __init__(
        self,
        context: InjectionContext,
        receive_inbound: Coroutine,
        return_inbound: Callable = None,
    ):
        """Initialize an `InboundTransportManager` instance."""
        self.context = context
        self.max_message_size = 0
        self.receive_inbound = receive_inbound
        self.return_inbound = return_inbound
        self.registered_transports = {}
        self.running_transports = {}
        self.sessions = OrderedDict()
        self.session_limit: asyncio.Semaphore = None
        self.task_queue = TaskQueue()
        self.undelivered_queue: DeliveryQueue = None 
示例9
def setup(self):
        """Perform setup operations."""
        # Load config settings
        if self.context.settings.get("transport.max_message_size"):
            self.max_message_size = self.context.settings["transport.max_message_size"]

        inbound_transports = (
            self.context.settings.get("transport.inbound_configs") or []
        )
        for transport in inbound_transports:
            module, host, port = transport
            self.register(
                InboundTransportConfiguration(module=module, host=host, port=port)
            )

        # Setup queue for undelivered messages
        if self.context.settings.get("transport.enable_undelivered_queue"):
            self.undelivered_queue = DeliveryQueue()

        # self.session_limit = asyncio.Semaphore(50) 
示例10
def __init__(self, daemon, coin, blocks_event):
        self.logger = class_logger(__name__, self.__class__.__name__)
        self.daemon = daemon
        self.coin = coin
        self.blocks_event = blocks_event
        self.blocks = []
        self.caught_up = False
        # Access to fetched_height should be protected by the semaphore
        self.fetched_height = None
        self.semaphore = asyncio.Semaphore()
        self.refill_event = asyncio.Event()
        # The prefetched block cache size.  The min cache size has
        # little effect on sync time.
        self.cache_size = 0
        self.min_cache_size = 10 * 1024 * 1024
        # This makes the first fetch be 10 blocks
        self.ave_size = self.min_cache_size // 10
        self.polling_delay = 5 
示例11
def scan_host(address, start_port, end_port, max_conns=400):
        """

        :param address: IPv4 address to scan
        :param start_port: First port value to scan
        :param end_port: Last port value to scan
        :param max_conns: Maximum simultaneous number of connections
        :return:
        """
        sem = asyncio.Semaphore(max_conns)
        ports = range(start_port, end_port)
        tasks = [asyncio.ensure_future(PortScanner.check_port_sem(sem, address, port)) for port in ports]
        responses = await asyncio.gather(*tasks)
        open_ports = list(filter(lambda x: x is not None, responses))

        return open_ports 
示例12
def find_node_ports(address, ports):
    """
    Find all the open ports for a host.

    :param address: IP address of the host.
    :param ports: Port to check.
    :return: A list of the found open ports.
    """
    sem = asyncio.Semaphore(400)  # Change this value for concurrency limitation
    tasks = [asyncio.ensure_future(check_port_sem(sem, address, p)) for p in ports]
    found_ports = []
    for response in await asyncio.gather(*tasks):
        if response:
            found_ports.append(response)

    return found_ports 
示例13
def check_routers(self, addresses: List[str], ports: List[int]) -> List[BaseIndustrialRouter]:
        """
        Check for routers in a range of addressess and ports.

        :param addresses: List of addressess to be checked.
        :param ports: List of ports to be checked for each address.
        :return: A list of found routers.
        """
        async def check_routers_aio(addresses, ports):
            semaphore = Semaphore(50)
            futures = []
            routers = []
            for address in addresses:
                for port in ports:
                    futures.append(asyncio.ensure_future(self.check_is_router(address, port, semaphore=semaphore)))
            done, pending = await asyncio.wait(futures)
            for future in done:
                if future:
                    routers.append(future)
            return routers

        return asyncio.run(check_routers_aio(addresses, ports), debug=True) 
示例14
def check_router_credentials(self, routers: List[BaseIndustrialRouter]):
        """
        Check default credentials for a list of routers.

        :param routers: List of routers to be checked.
        """
        async def check_router_credentials_aio(routers):
            semaphore = Semaphore(100)
            futures = []
            for router in routers:
                if isinstance(router, self.__class__.router_cls):
                    futures.append(asyncio.ensure_future(self.check_default_password(router, semaphore=semaphore)))

            await asyncio.wait(futures, return_when=ALL_COMPLETED, )

        asyncio.run(check_router_credentials_aio(routers), debug=True) 
示例15
def test_context_manager_async_with(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            async with lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 
示例16
def test_context_manager_with_await(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            with await lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 
示例17
def test_semaphore(self):
        sem = asyncio.Semaphore(loop=self.loop)
        self.assertEqual(1, sem._value)

        @asyncio.coroutine
        def acquire_lock():
            return (yield from sem)

        res = self.loop.run_until_complete(acquire_lock())

        self.assertTrue(res)
        self.assertTrue(sem.locked())
        self.assertEqual(0, sem._value)

        sem.release()
        self.assertFalse(sem.locked())
        self.assertEqual(1, sem._value) 
示例18
def test_acquire_cancel_before_awoken(self):
        sem = asyncio.Semaphore(value=0, loop=self.loop)

        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
        t3 = asyncio.Task(sem.acquire(), loop=self.loop)
        t4 = asyncio.Task(sem.acquire(), loop=self.loop)

        test_utils.run_briefly(self.loop)

        sem.release()
        t1.cancel()
        t2.cancel()

        test_utils.run_briefly(self.loop)
        num_done = sum(t.done() for t in [t3, t4])
        self.assertEqual(num_done, 1)

        t3.cancel()
        t4.cancel()
        test_utils.run_briefly(self.loop) 
示例19
def __init__(self, config: Config, model, play_config=None):

        self.config = config
        self.model = model
        self.play_config = play_config or self.config.play
        self.api = Connect4ModelAPI(self.config, self.model)

        self.labels_n = config.n_labels
        self.var_n = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_w = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_q = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_u = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_p = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.expanded = set()
        self.now_expanding = set()
        self.prediction_queue = Queue(self.play_config.prediction_queue_size)
        self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)

        self.moves = []
        self.loop = asyncio.get_event_loop()
        self.running_simulation_num = 0

        self.thinking_history = {}  # for fun 
示例20
def test_repr(self):
        sem = asyncio.Semaphore(loop=self.loop)
        self.assertTrue(repr(sem).endswith('[unlocked,value:1]>'))
        self.assertTrue(RGX_REPR.match(repr(sem)))

        self.loop.run_until_complete(sem.acquire())
        self.assertTrue(repr(sem).endswith('[locked]>'))
        self.assertTrue('waiters' not in repr(sem))
        self.assertTrue(RGX_REPR.match(repr(sem)))

        sem._waiters.append(mock.Mock())
        self.assertTrue('waiters:1' in repr(sem))
        self.assertTrue(RGX_REPR.match(repr(sem)))

        sem._waiters.append(mock.Mock())
        self.assertTrue('waiters:2' in repr(sem))
        self.assertTrue(RGX_REPR.match(repr(sem))) 
示例21
def test_semaphore(self):
        sem = asyncio.Semaphore(loop=self.loop)
        self.assertEqual(1, sem._value)

        @asyncio.coroutine
        def acquire_lock():
            return (yield from sem)

        res = self.loop.run_until_complete(acquire_lock())

        self.assertTrue(res)
        self.assertTrue(sem.locked())
        self.assertEqual(0, sem._value)

        sem.release()
        self.assertFalse(sem.locked())
        self.assertEqual(1, sem._value) 
示例22
def test_acquire_cancel_before_awoken(self):
        sem = asyncio.Semaphore(value=0, loop=self.loop)

        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
        t3 = asyncio.Task(sem.acquire(), loop=self.loop)
        t4 = asyncio.Task(sem.acquire(), loop=self.loop)

        test_utils.run_briefly(self.loop)

        sem.release()
        t1.cancel()
        t2.cancel()

        test_utils.run_briefly(self.loop)
        num_done = sum(t.done() for t in [t3, t4])
        self.assertEqual(num_done, 1)

        t3.cancel()
        t4.cancel()
        test_utils.run_briefly(self.loop) 
示例23
def print_tests():
    st = '''Available functions:
print_tests()  Print this list.
ack_test()  Test event acknowledge and Message class.
message_test() Test Message class.
event_test()  Test Event and Lock objects.
barrier_test()  Test the Barrier class.
semaphore_test(bounded=False)  Test Semaphore or BoundedSemaphore.
condition_test()  Test the Condition class.
queue_test()  Test the  Queue class

Recommended to issue ctrl-D after running each test.
'''
    print('\x1b[32m')
    print(st)
    print('\x1b[39m') 
示例24
def _fill_body(body: Body, semaphore: asyncio.Semaphore, limit: int) -> None:
    for number in range(limit):
        body.append(b"%d" % number)
        await semaphore.acquire()
    body.set_complete() 
示例25
def test_full_body() -> None:
    body = Body(None, None)
    limit = 3
    semaphore = asyncio.Semaphore(limit)
    asyncio.ensure_future(_fill_body(body, semaphore, limit))
    assert b"012" == await body  # type: ignore 
示例26
def test_body_streaming() -> None:
    body = Body(None, None)
    limit = 3
    semaphore = asyncio.Semaphore(0)
    asyncio.ensure_future(_fill_body(body, semaphore, limit))
    index = 0
    async for data in body:  # type: ignore
        semaphore.release()
        assert data == b"%d" % index
        index += 1
    assert b"" == await body  # type: ignore 
示例27
def test_body_streaming_no_data() -> None:
    body = Body(None, None)
    semaphore = asyncio.Semaphore(0)
    asyncio.ensure_future(_fill_body(body, semaphore, 0))
    async for _ in body:  # type: ignore # noqa: F841
        assert False  # Should not reach this
    assert b"" == await body  # type: ignore 
示例28
def init(sanic, loop):
    global sem
    concurrency_per_worker = 4
    sem = asyncio.Semaphore(concurrency_per_worker, loop=loop) 
示例29
def __init__(self, cb):
        super().__init__()
        self.cb = cb
        self.cnt= 0
        self.log = logging.getLogger() #.getChild('TcpConnection')
        self.log.info('TcpConnection: creating TcpCOnnection')
        self.queue = asyncio.Queue(maxsize=100)
        self.hipri_queue = asyncio.Queue()
        self._ready = asyncio.Event()
        self._msg_ready = asyncio.Semaphore(value=0)
        self.tsk= asyncio.async(self._send_messages())  # Or asyncio.ensure_future if using 3.4.3+
        self.flush= False 
示例30
def run(loop, r):
    url = config.get("server_url") + "{}"
    tasks = []
    # create instance of Semaphore
    batch_size = int(config.get("batch_size"))
    sem = asyncio.Semaphore(batch_size)
    for i in range(r):
        # pass Semaphore to every GET request
        task = asyncio.ensure_future(bound_fetch(sem, url.format(i)))
        tasks.append(task)

    responses = await asyncio.gather(*tasks)
    #await responses
    print("Got {} responses".format(len(responses)))