diff --git a/.github/workflows/build-linux-installer.yml b/.github/workflows/build-linux-installer.yml new file mode 100644 index 0000000..b8eae14 --- /dev/null +++ b/.github/workflows/build-linux-installer.yml @@ -0,0 +1,96 @@ +name: Linux installer on Python 3.8 + +on: + push: + branches: + - main + tags: + - '**' + pull_request: + branches: + - '**' + +jobs: + build: + name: Linux installer on Python 3.8 + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + python-version: [3.8] + os: [ubuntu-18.04] + + steps: + - name: Cancel previous runs on the same branch + if: ${{ github.ref != 'refs/heads/main' }} + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + + - name: Checkout Code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache npm + uses: actions/cache@v2.1.5 + env: + cache-name: cache-node-modules + with: + # npm cache files are stored in `~/.npm` on Linux/macOS + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip + uses: actions/cache@v2.1.5 + with: + # Note that new runners may break this https://github.com/actions/cache/issues/292 + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Run install script + env: + INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} + BUILD_VDF_CLIENT: "N" + run: | + sh install.sh + + - name: Setup Node 12.x + uses: actions/setup-node@v2.1.5 + with: + node-version: '12.x' + + - name: Build .deb and .rpm packages + run: | + . ./activate + sudo apt-get -y install rpm + ldd --version + cd ./flora-blockchain-gui + git status + cd ../build_scripts + sh build_linux.sh amd64 + + - name: Upload Linux artifacts + uses: actions/upload-artifact@v2 + with: + name: Linux-Installers + path: ${{ github.workspace }}/build_scripts/final_installer/ diff --git a/.github/workflows/build-macos-installer.yml b/.github/workflows/build-macos-installer.yml new file mode 100644 index 0000000..2ba6bb0 --- /dev/null +++ b/.github/workflows/build-macos-installer.yml @@ -0,0 +1,115 @@ +name: MacOS installer on Catalina and Python 3.8 + +on: + push: + branches: + - main + tags: + - '**' + pull_request: + branches: + - '**' + +jobs: + build: + name: MacOS installer on Catalina and Python 3.8 + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + strategy: + fail-fast: false + max-parallel: 4 + matrix: + python-version: [3.9] + os: [macOS-latest] + + steps: + - name: Cancel previous runs on the same branch + if: ${{ github.ref != 'refs/heads/main' }} + uses: styfle/cancel-workflow-action@0.9.0 + with: + access_token: ${{ github.token }} + + - name: Checkout Code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: recursive + + - name: Setup Python environment + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache npm + uses: actions/cache@v2.1.5 + env: + cache-name: cache-node-modules + with: + # npm cache files are stored in `~/.npm` on Linux/macOS + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip + uses: actions/cache@v2.1.5 + with: + # Note that new runners may break this https://github.com/actions/cache/issues/292 + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Test for secrets access + id: check_secrets + shell: bash + run: | + unset HAS_SECRET + if [ -n "$SECRET" ]; then HAS_SECRET='true' ; fi + echo ::set-output name=HAS_SECRET::${HAS_SECRET} + env: + SECRET: "${{ secrets.APPLE_DEV_ID_APP }}" + + - name: Import Apple app signing certificate + if: steps.check_secrets.outputs.HAS_SECRET + uses: Apple-Actions/import-codesign-certs@v1 + with: + p12-file-base64: ${{ secrets.APPLE_DEV_ID_APP }} + p12-password: ${{ secrets.APPLE_DEV_ID_APP_PASS }} + + - name: Run install script + env: + INSTALL_PYTHON_VERSION: ${{ matrix.python-version }} + BUILD_VDF_CLIENT: "N" + run: | + sh install.sh + + - name: Setup Node 12.x + uses: actions/setup-node@v2.1.5 + with: + node-version: '12.x' + + - name: Build MacOS DMG in Catalina + env: + NOTARIZE: ${{ steps.check_secrets.outputs.HAS_SECRET }} + APPLE_NOTARIZE_USERNAME: "${{ secrets.APPLE_NOTARIZE_USERNAME }}" + APPLE_NOTARIZE_PASSWORD: "${{ secrets.APPLE_NOTARIZE_PASSWORD }}" + run: | + . ./activate + cd ./flora-blockchain-gui + git status + cd ../build_scripts + sh build_macos.sh + + - name: Upload MacOS artifacts + uses: actions/upload-artifact@v2 + with: + name: Flora-Installer-on-MacOS-10.15-dmg + path: ${{ github.workspace }}/build_scripts/final_installer/ diff --git a/build_scripts/build_linux.sh b/build_scripts/build_linux.sh index b0fdb24..ae9ef4b 100644 --- a/build_scripts/build_linux.sh +++ b/build_scripts/build_linux.sh @@ -6,23 +6,23 @@ if [ ! "$1" ]; then elif [ "$1" = "amd64" ]; then PLATFORM="$1" REDHAT_PLATFORM="x86_64" - DIR_NAME="chia-blockchain-linux-x64" + DIR_NAME="flora-blockchain-linux-x64" else PLATFORM="$1" - DIR_NAME="chia-blockchain-linux-arm64" + DIR_NAME="flora-blockchain-linux-arm64" fi pip install setuptools_scm -# The environment variable CHIA_INSTALLER_VERSION needs to be defined +# The environment variable FLORA_INSTALLER_VERSION needs to be defined # If the env variable NOTARIZE and the username and password variables are # set, this will attempt to Notarize the signed DMG -CHIA_INSTALLER_VERSION=$(python installer-version.py) +FLORA_INSTALLER_VERSION=$(python installer-version.py) -if [ ! "$CHIA_INSTALLER_VERSION" ]; then - echo "WARNING: No environment variable CHIA_INSTALLER_VERSION set. Using 0.0.0." - CHIA_INSTALLER_VERSION="0.0.0" +if [ ! "$FLORA_INSTALLER_VERSION" ]; then + echo "WARNING: No environment variable FLORA_INSTALLER_VERSION set. Using 0.0.0." + FLORA_INSTALLER_VERSION="0.0.0" fi -echo "Chia Installer Version is: $CHIA_INSTALLER_VERSION" +echo "Flora Installer Version is: $FLORA_INSTALLER_VERSION" echo "Installing npm and electron packagers" npm install electron-packager -g @@ -35,7 +35,7 @@ mkdir dist echo "Create executables with pyinstaller" pip install pyinstaller==4.2 -SPEC_FILE=$(python -c 'import chia; print(chia.PYINSTALLER_SPEC_PATH)') +SPEC_FILE=$(python -c 'import flora; print(flora.PYINSTALLER_SPEC_PATH)') pyinstaller --log-level=INFO "$SPEC_FILE" LAST_EXIT_CODE=$? if [ "$LAST_EXIT_CODE" -ne 0 ]; then @@ -43,9 +43,9 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then exit $LAST_EXIT_CODE fi -cp -r dist/daemon ../chia-blockchain-gui +cp -r dist/daemon ../flora-blockchain-gui cd .. || exit -cd chia-blockchain-gui || exit +cd flora-blockchain-gui || exit echo "npm build" npm install @@ -57,9 +57,9 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then exit $LAST_EXIT_CODE fi -electron-packager . chia-blockchain --asar.unpack="**/daemon/**" --platform=linux \ ---icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \ ---appVersion=$CHIA_INSTALLER_VERSION +electron-packager . flora-blockchain --asar.unpack="**/daemon/**" --platform=linux \ +--icon=src/assets/img/Flora.icns --overwrite --app-bundle-id=net.flora.blockchain \ +--appVersion=$FLORA_INSTALLER_VERSION LAST_EXIT_CODE=$? if [ "$LAST_EXIT_CODE" -ne 0 ]; then echo >&2 "electron-packager failed!" @@ -69,11 +69,11 @@ fi mv $DIR_NAME ../build_scripts/dist/ cd ../build_scripts || exit -echo "Create chia-$CHIA_INSTALLER_VERSION.deb" +echo "Create flora-$FLORA_INSTALLER_VERSION.deb" rm -rf final_installer mkdir final_installer electron-installer-debian --src dist/$DIR_NAME/ --dest final_installer/ \ ---arch "$PLATFORM" --options.version $CHIA_INSTALLER_VERSION +--arch "$PLATFORM" --options.version $FLORA_INSTALLER_VERSION LAST_EXIT_CODE=$? if [ "$LAST_EXIT_CODE" -ne 0 ]; then echo >&2 "electron-installer-debian failed!" @@ -81,9 +81,9 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then fi if [ "$REDHAT_PLATFORM" = "x86_64" ]; then - echo "Create chia-blockchain-$CHIA_INSTALLER_VERSION.rpm" + echo "Create flora-blockchain-$FLORA_INSTALLER_VERSION.rpm" electron-installer-redhat --src dist/$DIR_NAME/ --dest final_installer/ \ - --arch "$REDHAT_PLATFORM" --options.version $CHIA_INSTALLER_VERSION \ + --arch "$REDHAT_PLATFORM" --options.version $FLORA_INSTALLER_VERSION \ --license ../LICENSE LAST_EXIT_CODE=$? if [ "$LAST_EXIT_CODE" -ne 0 ]; then diff --git a/build_scripts/build_macos.sh b/build_scripts/build_macos.sh index 4df16ec..8fc0761 100644 --- a/build_scripts/build_macos.sh +++ b/build_scripts/build_macos.sh @@ -1,15 +1,15 @@ #!/bin/bash pip install setuptools_scm -# The environment variable CHIA_INSTALLER_VERSION needs to be defined. +# The environment variable FLORA_INSTALLER_VERSION needs to be defined. # If the env variable NOTARIZE and the username and password variables are # set, this will attempt to Notarize the signed DMG. -CHIA_INSTALLER_VERSION=$(python installer-version.py) +FLORA_INSTALLER_VERSION=$(python installer-version.py) -if [ ! "$CHIA_INSTALLER_VERSION" ]; then - echo "WARNING: No environment variable CHIA_INSTALLER_VERSION set. Using 0.0.0." - CHIA_INSTALLER_VERSION="0.0.0" +if [ ! "$FLORA_INSTALLER_VERSION" ]; then + echo "WARNING: No environment variable FLORA_INSTALLER_VERSION set. Using 0.0.0." + FLORA_INSTALLER_VERSION="0.0.0" fi -echo "Chia Installer Version is: $CHIA_INSTALLER_VERSION" +echo "Flora Installer Version is: $FLORA_INSTALLER_VERSION" echo "Installing npm and electron packagers" npm install electron-installer-dmg -g @@ -23,16 +23,16 @@ mkdir dist echo "Create executables with pyinstaller" pip install pyinstaller==4.2 -SPEC_FILE=$(python -c 'import chia; print(chia.PYINSTALLER_SPEC_PATH)') +SPEC_FILE=$(python -c 'import flora; print(flora.PYINSTALLER_SPEC_PATH)') pyinstaller --log-level=INFO "$SPEC_FILE" LAST_EXIT_CODE=$? if [ "$LAST_EXIT_CODE" -ne 0 ]; then echo >&2 "pyinstaller failed!" exit $LAST_EXIT_CODE fi -cp -r dist/daemon ../chia-blockchain-gui +cp -r dist/daemon ../flora-blockchain-gui cd .. || exit -cd chia-blockchain-gui || exit +cd flora-blockchain-gui || exit echo "npm build" npm install @@ -44,9 +44,9 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then exit $LAST_EXIT_CODE fi -electron-packager . Chia --asar.unpack="**/daemon/**" --platform=darwin \ ---icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \ ---appVersion=$CHIA_INSTALLER_VERSION +electron-packager . Flora --asar.unpack="**/daemon/**" --platform=darwin \ +--icon=src/assets/img/Flora.icns --overwrite --app-bundle-id=net.flora.blockchain \ +--appVersion=$FLORA_INSTALLER_VERSION LAST_EXIT_CODE=$? if [ "$LAST_EXIT_CODE" -ne 0 ]; then echo >&2 "electron-packager failed!" @@ -54,8 +54,8 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then fi if [ "$NOTARIZE" ]; then - electron-osx-sign Chia-darwin-x64/Chia.app --platform=darwin \ - --hardened-runtime=true --provisioning-profile=chiablockchain.provisionprofile \ + electron-osx-sign Flora-darwin-x64/Flora.app --platform=darwin \ + --hardened-runtime=true --provisioning-profile=florablockchain.provisionprofile \ --entitlements=entitlements.mac.plist --entitlements-inherit=entitlements.mac.plist \ --no-gatekeeper-assess fi @@ -65,13 +65,13 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then exit $LAST_EXIT_CODE fi -mv Chia-darwin-x64 ../build_scripts/dist/ +mv Flora-darwin-x64 ../build_scripts/dist/ cd ../build_scripts || exit -DMG_NAME="Chia-$CHIA_INSTALLER_VERSION.dmg" +DMG_NAME="Flora-$FLORA_INSTALLER_VERSION.dmg" echo "Create $DMG_NAME" mkdir final_installer -electron-installer-dmg dist/Chia-darwin-x64/Chia.app Chia-$CHIA_INSTALLER_VERSION \ +electron-installer-dmg dist/Flora-darwin-x64/Flora.app Flora-$FLORA_INSTALLER_VERSION \ --overwrite --out final_installer LAST_EXIT_CODE=$? if [ "$LAST_EXIT_CODE" -ne 0 ]; then @@ -82,7 +82,7 @@ fi if [ "$NOTARIZE" ]; then echo "Notarize $DMG_NAME on ci" cd final_installer || exit - notarize-cli --file=$DMG_NAME --bundle-id net.chia.blockchain \ + notarize-cli --file=$DMG_NAME --bundle-id net.flora.blockchain \ --username "$APPLE_NOTARIZE_USERNAME" --password "$APPLE_NOTARIZE_PASSWORD" echo "Notarization step complete" else @@ -93,7 +93,7 @@ fi # # Ask for username and password. password should be an app specific password. # Generate app specific password https://support.apple.com/en-us/HT204397 -# xcrun altool --notarize-app -f Chia-0.1.X.dmg --primary-bundle-id net.chia.blockchain -u username -p password +# xcrun altool --notarize-app -f Flora-0.1.X.dmg --primary-bundle-id net.flora.blockchain -u username -p password # xcrun altool --notarize-app; -should return REQUEST-ID, use it in next command # # Wait until following command return a success message". @@ -101,7 +101,7 @@ fi # It can take a while, run it every few minutes. # # Once that is successful, execute the following command": -# xcrun stapler staple Chia-0.1.X.dmg +# xcrun stapler staple Flora-0.1.X.dmg # # Validate DMG: -# xcrun stapler validate Chia-0.1.X.dmg +# xcrun stapler validate Flora-0.1.X.dmg diff --git a/build_scripts/build_windows.ps1 b/build_scripts/build_windows.ps1 index 37d66bf..8ef59be 100644 --- a/build_scripts/build_windows.ps1 +++ b/build_scripts/build_windows.ps1 @@ -107,7 +107,7 @@ Write-Output "packageName is $packageName" Write-Output " ---" Write-Output "electron-packager" -electron-packager . Flora --asar.unpack="**\daemon\**" --overwrite --icon=.\src\assets\img\chia.ico --app-version=$packageVersion +electron-packager . Flora --asar.unpack="**\daemon\**" --overwrite --icon=.\src\assets\img\flora.ico --app-version=$packageVersion Write-Output " ---" Write-Output " ---" diff --git a/flora-blockchain-gui b/flora-blockchain-gui index 145b0f3..29ba3de 160000 --- a/flora-blockchain-gui +++ b/flora-blockchain-gui @@ -1 +1 @@ -Subproject commit 145b0f390c2111706b94b584b3840fe494c5cc23 +Subproject commit 29ba3de54189b374fee38af96ef76fc3a70a96fa diff --git a/flora/farmer/farmer.py b/flora/farmer/farmer.py index d30d78e..4e96629 100644 --- a/flora/farmer/farmer.py +++ b/flora/farmer/farmer.py @@ -56,6 +56,20 @@ HARVESTER PROTOCOL (FARMER <-> HARVESTER) """ +class HarvesterCacheEntry: + def __init__(self): + self.data: Optional[dict] = None + self.last_update: float = 0 + + def set_data(self, data): + self.data = data + self.last_update = time.time() + + def needs_update(self): + return time.time() - self.last_update > UPDATE_HARVESTER_CACHE_INTERVAL + + def expired(self): + return time.time() - self.last_update > UPDATE_HARVESTER_CACHE_INTERVAL * 10 class Farmer: def __init__( @@ -132,7 +146,7 @@ def __init__( # Last time we updated pool_state based on the config file self.last_config_access_time: uint64 = uint64(0) - self.harvester_cache: Dict[str, Dict[str, Tuple[Dict, float]]] = {} + self.harvester_cache: Dict[str, Dict[str, HarvesterCacheEntry]] = {} async def _start(self): self.update_pool_state_task = asyncio.create_task(self._periodically_update_pool_state_task()) @@ -532,17 +546,18 @@ async def generate_login_link(self, launcher_id: bytes32) -> Optional[str]: async def update_cached_harvesters(self): # First remove outdated cache entries + self.log.debug(f"update_cached_harvesters cache entries: {len(self.harvester_cache)}") remove_hosts = [] for host, host_cache in self.harvester_cache.items(): remove_peers = [] for peer_id, peer_cache in host_cache.items(): - _, last_update = peer_cache - # If the peer cache hasn't been updated for 10x interval, drop it since the harvester doesn't respond - if time.time() - last_update > UPDATE_HARVESTER_CACHE_INTERVAL * 10: + # If the peer cache is expired it means the harvester didn't respond for too long + if peer_cache.expired(): remove_peers.append(peer_id) for key in remove_peers: del host_cache[key] if len(host_cache) == 0: + self.log.debug(f"update_cached_harvesters remove host: {host}") remove_hosts.append(host) for key in remove_hosts: del self.harvester_cache[key] @@ -551,17 +566,15 @@ async def update_cached_harvesters(self): if connection.connection_type != NodeType.HARVESTER: continue cache_entry = await self.get_cached_harvesters(connection) - if cache_entry is None or time.time() - cache_entry[1] > UPDATE_HARVESTER_CACHE_INTERVAL: - response = await connection.request_plots(harvester_protocol.RequestPlots(), timeout=5) + if cache_entry.needs_update(): + self.log.debug(f"update_cached_harvesters update harvester: {connection.peer_node_id}") + response = await connection.request_plots( + harvester_protocol.RequestPlots(), timeout=UPDATE_HARVESTER_CACHE_INTERVAL + ) if response is not None: if isinstance(response, harvester_protocol.RespondPlots): - if connection.peer_host not in self.harvester_cache: - self.harvester_cache[connection.peer_host] = {} - - self.harvester_cache[connection.peer_host][connection.peer_node_id.hex()] = ( - response.to_json_dict(), - time.time(), - ) + cache_entry.set_data(response.to_json_dict()) + self.log.debug(f"update_cached_harvesters cache updated: {connection.peer_node_id}") else: self.log.error( f"Invalid response from harvester:" @@ -572,11 +585,16 @@ async def update_cached_harvesters(self): "Harvester did not respond. You might need to update harvester to the latest version" ) - async def get_cached_harvesters(self, connection: WSFloraConnection) -> Optional[Tuple[Dict, float]]: + async def get_cached_harvesters(self, connection: WSFloraConnection) -> HarvesterCacheEntry: host_cache = self.harvester_cache.get(connection.peer_host) if host_cache is None: - return None - return host_cache.get(connection.peer_node_id.hex()) + host_cache = {} + self.harvester_cache[connection.peer_host] = host_cache + node_cache = host_cache.get(connection.peer_node_id.hex()) + if node_cache is None: + node_cache = HarvesterCacheEntry() + host_cache[connection.peer_node_id.hex()] = node_cache + return node_cache async def get_harvesters(self) -> Dict: harvesters: List = [] @@ -584,15 +602,18 @@ async def get_harvesters(self) -> Dict: if connection.connection_type != NodeType.HARVESTER: continue + self.log.debug(f"get_harvesters host: {connection.peer_host}, node_id: {connection.peer_node_id}") cache_entry = await self.get_cached_harvesters(connection) - if cache_entry is not None: - harvester_object: dict = dict(cache_entry[0]) + if cache_entry.data is not None: + harvester_object: dict = dict(cache_entry.data) harvester_object["connection"] = { "node_id": connection.peer_node_id.hex(), "host": connection.peer_host, "port": connection.peer_port, } harvesters.append(harvester_object) + else: + self.log.debug(f"get_harvesters no cache: {connection.peer_host}, node_id: {connection.peer_node_id}") return {"harvesters": harvesters} diff --git a/flora/harvester/harvester.py b/flora/harvester/harvester.py index d95bc90..03165e7 100644 --- a/flora/harvester/harvester.py +++ b/flora/harvester/harvester.py @@ -77,6 +77,7 @@ def on_disconnect(self, connection: ws.WSFloraConnection): self._state_changed("close_connection") def get_plots(self) -> Tuple[List[Dict], List[str], List[str]]: + self.log.debug(f"get_plots prover items: {len(self.provers)}") response_plots: List[Dict] = [] for path, plot_info in self.provers.items(): prover = plot_info.prover @@ -93,7 +94,11 @@ def get_plots(self) -> Tuple[List[Dict], List[str], List[str]]: "time_modified": plot_info.time_modified, } ) - + self.log.debug( + f"get_plots response: plots: {len(response_plots)}, " + f"failed_to_open_filenames: {len(self.failed_to_open_filenames)}, " + f"no_key_filenames: {len(self.no_key_filenames)}" + ) return ( response_plots, [str(s) for s, _ in self.failed_to_open_filenames.items()], diff --git a/flora/server/rate_limits.py b/flora/server/rate_limits.py index 1d95919..268bf18 100644 --- a/flora/server/rate_limits.py +++ b/flora/server/rate_limits.py @@ -96,7 +96,7 @@ class RLSettings: ProtocolMessageTypes.respond_peers_introducer: RLSettings(100, 1024 * 1024), ProtocolMessageTypes.farm_new_block: RLSettings(200, 200), ProtocolMessageTypes.request_plots: RLSettings(10, 10 * 1024 * 1024), - ProtocolMessageTypes.respond_plots: RLSettings(10, 10 * 1024 * 1024), + ProtocolMessageTypes.respond_plots: RLSettings(10, 100 * 1024 * 1024), }