From b61e850db43b2410ca9c6c0b099a8220c29e0321 Mon Sep 17 00:00:00 2001 From: isabella Date: Thu, 24 Mar 2022 16:07:44 +0000 Subject: [PATCH 1/8] fixed mp speed up --- bittensor/utils/__init__.py | 125 ++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 54 deletions(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 98a1110abe..07af9d4e13 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -97,7 +97,7 @@ def solve_for_difficulty( block_hash, difficulty ): break return nonce, seal -def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = 5, update_interval: int = 100000 ) -> Tuple[int, int, Any, int, Any]: +def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = 10, update_interval: int = 500000 ) -> Tuple[int, int, Any, int, Any]: """ Solves the POW for registration using multiprocessing. Args: @@ -122,75 +122,92 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = 5, update block_hash = subtensor.substrate.get_block_hash( block_number ) block_bytes = block_hash.encode('utf-8')[2:] - nonce = -1 + nonce = 0 limit = int(math.pow(2,256) - 1) start_time = time.time() console = bittensor.__console__ - with console.status("Solving") as status: - found_solution = multiprocessing.Value('q', -1, lock=False) # int - best = multiprocessing.Value(ctypes.c_char_p, lock=True) # byte array to get around int size of ctypes - best.raw = struct.pack("d", float('inf')) - best_seal = multiprocessing.Array('h', 32, lock=True) # short array should hold bytes (0, 256) - - with multiprocessing.Pool(processes=num_processes, initializer=initProcess_, initargs=(solve_, found_solution, best, best_seal)) as pool: - # while no solution found and wallet has not been registered - while found_solution.value == -1 and not wallet.is_registered(subtensor): - result = pool.starmap(solve_, iterable=[(nonce, block_bytes, difficulty, block_hash, block_number, limit) for nonce in range(update_interval)]) - nonce += update_interval - itrs_per_sec = update_interval / (time.time() - start_time) - start_time = time.time() - difficulty = subtensor.difficulty - block_number = subtensor.get_current_block() - block_hash = subtensor.substrate.get_block_hash( block_number) - while block_hash == None: - block_hash = subtensor.substrate.get_block_hash( block_number) - block_bytes = block_hash.encode('utf-8')[2:] - with best_seal.get_lock(): - status.update(f""" - Solving - Nonce: [bold white]{nonce}[/bold white] - Difficulty: [bold white]{difficulty}[/bold white] - Iters: [bold white]{int(itrs_per_sec)}/s[/bold white] - Block: [bold white]{block_number}[/bold white] - Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white] - Best: [bold white]{binascii.hexlify(bytes(best_seal) or bytes(0))}[/bold white]""") - - # exited while, found_solution contains the nonce or wallet is registered - if found_solution.value == -1: # didn't find solution - return None, None, None, None, None + status = console.status("Solving") - nonce, block_number, block_hash, difficulty, seal = result[found_solution.value] - return nonce, block_number, block_hash, difficulty, seal + found_solution = multiprocessing.Value('q', -1, lock=False) # int + best = multiprocessing.Value(ctypes.c_char_p, lock=True) # byte array to get around int size of ctypes + best.raw = struct.pack("d", float('inf')) + best_seal = multiprocessing.Array('h', 32, lock=True) # short array should hold bytes (0, 256) + + with multiprocessing.Pool(processes=num_processes, initializer=initProcess_, initargs=(solve_, found_solution, best, best_seal)) as pool: + status.start() + while found_solution.value == -1 and not wallet.is_registered(subtensor): + iterable = [( nonce_start, + nonce_start + update_interval , + block_bytes, + difficulty, + block_hash, + block_number, + limit) for nonce_start in list(range(nonce, nonce + update_interval*num_processes, update_interval))] + result = pool.starmap(solve_, iterable=iterable) + old_nonce = nonce + nonce += update_interval*num_processes + itrs_per_sec = update_interval*num_processes / (time.time() - start_time) + start_time = time.time() + difficulty = subtensor.difficulty + block_number = subtensor.get_current_block() + block_hash = subtensor.substrate.get_block_hash( block_number) + while block_hash == None: + block_hash = subtensor.substrate.get_block_hash( block_number) + block_bytes = block_hash.encode('utf-8')[2:] + with best_seal.get_lock(): + message = f"""Solving + Nonce: [bold white]{nonce}[/bold white] + Difficulty: [bold white]{difficulty}[/bold white] + Iters: [bold white]{int(itrs_per_sec)}/s[/bold white] + Block: [bold white]{block_number}[/bold white] + Block_hash: [bold white]{block_hash.encode('utf-8')}[/bold white] + Best: [bold white]{binascii.hexlify(bytes(best_seal) or bytes(0))}[/bold white]""" + status.update(message.replace(" ", "")) + + # exited while, found_solution contains the nonce or wallet is registered + if found_solution.value == -1: # didn't find solution + return None, None, None, None, None + + nonce, block_number, block_hash, difficulty, seal = result[ math.floor( (found_solution.value-old_nonce) / update_interval) ] + status.stop() + return nonce, block_number, block_hash, difficulty, seal def initProcess_(f, found_solution, best, best_seal): f.found = found_solution - f.best = best + f.best = best f.best_seal = best_seal -def solve_(nonce, block_bytes, difficulty, block_hash, block_number, limit): - # Create seal. - nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) - pre_seal = nonce_bytes + block_bytes - seal = hashlib.sha256( bytearray(hex_bytes_to_u8_list(pre_seal)) ).digest() - - seal_number = int.from_bytes(seal, "big") - product = seal_number * difficulty +def solve_(nonce_start, nonce_end, block_bytes, difficulty, block_hash, block_number, limit): + best_local = float('inf') + best_seal_local = [0]*32 + start = time.time() + for nonce in range(nonce_start, nonce_end): + # Create seal. + nonce_bytes = binascii.hexlify(nonce.to_bytes(8, 'little')) + pre_seal = nonce_bytes + block_bytes + seal = hashlib.sha256( bytearray(hex_bytes_to_u8_list(pre_seal)) ).digest() + + seal_number = int.from_bytes(seal, "big") + product = seal_number * difficulty - if product < limit: - solve_.found.value = nonce - return (nonce, block_number, block_hash, difficulty, seal) + if product < limit: + solve_.found.value = nonce + return (nonce, block_number, block_hash, difficulty, seal) + + if (product - limit) < best_local: + best_local = product - limit + best_seal_local = seal with solve_.best.get_lock(): - if not hasattr(solve_.best, 'raw'): - solve_.best.raw = struct.pack("d", float('inf')) best_value_as_d = struct.unpack('d', solve_.best.raw)[0] - if (product - limit) < best_value_as_d: - + + if best_local < best_value_as_d: with solve_.best_seal.get_lock(): - solve_.best.raw = struct.pack('d', product - limit) + solve_.best.raw = struct.pack('d', best_local) for i in range(32): - solve_.best_seal[i] = seal[i] + solve_.best_seal[i] = best_seal_local[i] + return None def create_pow( subtensor, wallet ): From c0a9b1f58c7fc2e040ce5c805b919e961549c340 Mon Sep 17 00:00:00 2001 From: isabella Date: Thu, 24 Mar 2022 20:33:03 +0000 Subject: [PATCH 2/8] fix test --- tests/unit_tests/bittensor_tests/utils/test_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 59b1c7f781..2dc6965061 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -168,7 +168,7 @@ def test_solve_for_difficulty_fast_registered_already(): block_hash = '0xba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279' subtensor = MagicMock() subtensor.get_current_block = MagicMock( return_value=1 ) - subtensor.difficulty = 2560000000 # set high to make solving take a long time + subtensor.difficulty = 100000000000000000000000000 # set high to make solving take a long time subtensor.substrate = MagicMock() subtensor.substrate.get_block_hash = MagicMock( return_value=block_hash ) wallet = MagicMock() @@ -183,3 +183,7 @@ def test_solve_for_difficulty_fast_registered_already(): assert e is None # called every time until True assert wallet.is_registered.call_count == workblocks_before_is_registered + 1 + + +if __name__ == "__main__": + test_solve_for_difficulty_fast_registered_already() \ No newline at end of file From 39a91fb370853754db25bf7757b2e0eb009c8b62 Mon Sep 17 00:00:00 2001 From: isabella Date: Thu, 24 Mar 2022 21:46:08 +0000 Subject: [PATCH 3/8] test fix --- bittensor/utils/__init__.py | 1 + tests/unit_tests/bittensor_tests/utils/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 07af9d4e13..59fb887dca 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -167,6 +167,7 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = 10, updat # exited while, found_solution contains the nonce or wallet is registered if found_solution.value == -1: # didn't find solution + status.stop() return None, None, None, None, None nonce, block_number, block_hash, difficulty, seal = result[ math.floor( (found_solution.value-old_nonce) / update_interval) ] diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index 2dc6965061..b4b985cec1 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -168,7 +168,7 @@ def test_solve_for_difficulty_fast_registered_already(): block_hash = '0xba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279' subtensor = MagicMock() subtensor.get_current_block = MagicMock( return_value=1 ) - subtensor.difficulty = 100000000000000000000000000 # set high to make solving take a long time + subtensor.difficulty = 1000000000# set high to make solving take a long time subtensor.substrate = MagicMock() subtensor.substrate.get_block_hash = MagicMock( return_value=block_hash ) wallet = MagicMock() From 1fdee59f7125f2b52b39d1ceced7d4c4d2b7e80a Mon Sep 17 00:00:00 2001 From: isabella Date: Fri, 25 Mar 2022 14:44:03 +0000 Subject: [PATCH 4/8] test fix --- tests/unit_tests/bittensor_tests/utils/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/bittensor_tests/utils/test_utils.py b/tests/unit_tests/bittensor_tests/utils/test_utils.py index b4b985cec1..ffb30e1d14 100644 --- a/tests/unit_tests/bittensor_tests/utils/test_utils.py +++ b/tests/unit_tests/bittensor_tests/utils/test_utils.py @@ -168,14 +168,14 @@ def test_solve_for_difficulty_fast_registered_already(): block_hash = '0xba7ea4eb0b16dee271dbef5911838c3f359fcf598c74da65a54b919b68b67279' subtensor = MagicMock() subtensor.get_current_block = MagicMock( return_value=1 ) - subtensor.difficulty = 1000000000# set high to make solving take a long time + subtensor.difficulty = 100000000000# set high to make solving take a long time subtensor.substrate = MagicMock() subtensor.substrate.get_block_hash = MagicMock( return_value=block_hash ) wallet = MagicMock() wallet.is_registered = MagicMock( side_effect=is_registered_return_values ) # all arugments should return None to indicate an early return - a, b, c, d, e = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet ) + a, b, c, d, e = bittensor.utils.solve_for_difficulty_fast( subtensor, wallet, num_processes = 1, update_interval = 1000) assert a is None assert b is None assert c is None From 2b37f40e830407d33c89cccec3175fcde359754d Mon Sep 17 00:00:00 2001 From: isabella Date: Fri, 25 Mar 2022 15:26:05 +0000 Subject: [PATCH 5/8] num_processes fix --- bittensor/utils/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index 59fb887dca..be8e620a43 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -97,7 +97,7 @@ def solve_for_difficulty( block_hash, difficulty ): break return nonce, seal -def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = 10, update_interval: int = 500000 ) -> Tuple[int, int, Any, int, Any]: +def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = None, update_interval: int = 500000 ) -> Tuple[int, int, Any, int, Any]: """ Solves the POW for registration using multiprocessing. Args: @@ -115,6 +115,9 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = 10, updat while still updating the block information after a different number of nonces, to increase the transparency of the process while still keeping the speed. """ + if num_processes == None: + num_processes = round(multiprocessing.cpu_count() / 2 ) + block_number = subtensor.get_current_block() difficulty = subtensor.difficulty block_hash = subtensor.substrate.get_block_hash( block_number ) From c9ceb94377004f681813f266eff37c144f335747 Mon Sep 17 00:00:00 2001 From: isabella Date: Fri, 25 Mar 2022 15:29:28 +0000 Subject: [PATCH 6/8] undo use only half of the processes --- bittensor/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index be8e620a43..c26c5dedf4 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -116,7 +116,7 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = None, upd to increase the transparency of the process while still keeping the speed. """ if num_processes == None: - num_processes = round(multiprocessing.cpu_count() / 2 ) + num_processes = multiprocessing.cpu_count() block_number = subtensor.get_current_block() difficulty = subtensor.difficulty From 1483d1c840971a36f2ece32015edea0637191ca4 Mon Sep 17 00:00:00 2001 From: isabella Date: Fri, 25 Mar 2022 15:44:21 +0000 Subject: [PATCH 7/8] print time --- bittensor/utils/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bittensor/utils/__init__.py b/bittensor/utils/__init__.py index c26c5dedf4..7906bedece 100644 --- a/bittensor/utils/__init__.py +++ b/bittensor/utils/__init__.py @@ -160,6 +160,7 @@ def solve_for_difficulty_fast( subtensor, wallet, num_processes: int = None, upd block_bytes = block_hash.encode('utf-8')[2:] with best_seal.get_lock(): message = f"""Solving + time spent: {time.time() - start_time} Nonce: [bold white]{nonce}[/bold white] Difficulty: [bold white]{difficulty}[/bold white] Iters: [bold white]{int(itrs_per_sec)}/s[/bold white] From b766951b4387c29c0f3b67085428077c5deb865d Mon Sep 17 00:00:00 2001 From: Eugene Date: Mon, 22 Aug 2022 01:40:38 -0400 Subject: [PATCH 8/8] Validator hotfix --- bittensor/_neuron/text/core_validator/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bittensor/_neuron/text/core_validator/__init__.py b/bittensor/_neuron/text/core_validator/__init__.py index 73becd68fa..4329f9265b 100644 --- a/bittensor/_neuron/text/core_validator/__init__.py +++ b/bittensor/_neuron/text/core_validator/__init__.py @@ -603,9 +603,10 @@ def calculate_weights(self, responsive_uids: Set, queried_uids: Set): preferred_uids = preferred_uids[neuron_weights > 0] # filter to non-zero weights neuron_weights = neuron_weights[neuron_weights > 0] # filter to non-zero weights - # === Slice min_allowed_weights UIDs === - sample_uids = preferred_uids[:min_allowed_weights] # slice to min_allowed_weights - sample_weights = neuron_weights[:min_allowed_weights] # slice to min_allowed_weights + # === Slice weights_to_set UIDs === + weights_to_set = max([min_allowed_weights, len(responsive_uids)]) + sample_uids = preferred_uids[:weights_to_set] # slice to weights_to_set + sample_weights = neuron_weights[:weights_to_set] # slice to weights_to_set # === Normalize and apply max_allowed_ratio === sample_weights = bittensor.utils.weight_utils.normalize_max_multiple(x=sample_weights,