diff --git a/TSRM/TSRM.c b/TSRM/TSRM.c index c8b95d034d..62be0a4214 100644 --- a/TSRM/TSRM.c +++ b/TSRM/TSRM.c @@ -161,6 +161,23 @@ TSRM_API int tsrm_startup(int expected_threads, int expected_resources, int debu return 1; }/*}}}*/ +static void ts_free_resources(tsrm_tls_entry *thread_resources) +{ + /* Need to destroy in reverse order to respect dependencies. */ + for (int i = thread_resources->count - 1; i >= 0; i--) { + if (!resource_types_table[i].done) { + if (resource_types_table[i].dtor) { + resource_types_table[i].dtor(thread_resources->storage[i]); + } + + if (!resource_types_table[i].fast_offset) { + free(thread_resources->storage[i]); + } + } + } + + free(thread_resources->storage); +} /* Shutdown TSRM (call once for the entire process) */ TSRM_API void tsrm_shutdown(void) @@ -183,20 +200,13 @@ TSRM_API void tsrm_shutdown(void) tsrm_tls_entry *p = tsrm_tls_table[i], *next_p; while (p) { - int j; - next_p = p->next; - for (j=0; jcount; j++) { - if (p->storage[j]) { - if (resource_types_table && !resource_types_table[j].done && resource_types_table[j].dtor) { - resource_types_table[j].dtor(p->storage[j]); - } - if (!resource_types_table[j].fast_offset) { - free(p->storage[j]); - } - } + if (resource_types_table) { + /* This call will already free p->storage for us */ + ts_free_resources(p); + } else { + free(p->storage); } - free(p->storage); free(p); p = next_p; } @@ -283,9 +293,9 @@ TSRM_API ts_rsrc_id ts_allocate_id(ts_rsrc_id *rsrc_id, size_t size, ts_allocate tsrm_resource_type *_tmp; _tmp = (tsrm_resource_type *) realloc(resource_types_table, sizeof(tsrm_resource_type)*id_count); if (!_tmp) { - tsrm_mutex_unlock(tsmm_mutex); TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate storage for resource")); *rsrc_id = 0; + tsrm_mutex_unlock(tsmm_mutex); return 0; } resource_types_table = _tmp; @@ -326,10 +336,10 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz size = TSRM_ALIGNED_SIZE(size); if (tsrm_reserved_size - tsrm_reserved_pos < size) { - tsrm_mutex_unlock(tsmm_mutex); TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate space for fast resource")); *rsrc_id = 0; *offset = 0; + tsrm_mutex_unlock(tsmm_mutex); return 0; } @@ -341,9 +351,9 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz tsrm_resource_type *_tmp; _tmp = (tsrm_resource_type *) realloc(resource_types_table, sizeof(tsrm_resource_type)*id_count); if (!_tmp) { - tsrm_mutex_unlock(tsmm_mutex); TSRM_ERROR((TSRM_ERROR_LEVEL_ERROR, "Unable to allocate storage for resource")); *rsrc_id = 0; + tsrm_mutex_unlock(tsmm_mutex); return 0; } resource_types_table = _tmp; @@ -362,7 +372,13 @@ TSRM_API ts_rsrc_id ts_allocate_fast_id(ts_rsrc_id *rsrc_id, size_t *offset, siz return *rsrc_id; }/*}}}*/ +static void set_thread_local_storage_resource_to(tsrm_tls_entry *thread_resource) +{ + tsrm_tls_set(thread_resource); + TSRMLS_CACHE = thread_resource; +} +/* Must be called with tsmm_mutex held */ static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_T thread_id) {/*{{{*/ int i; @@ -378,8 +394,7 @@ static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_ (*thread_resources_ptr)->next = NULL; /* Set thread local storage to this new thread resources structure */ - tsrm_tls_set(*thread_resources_ptr); - TSRMLS_CACHE = *thread_resources_ptr; + set_thread_local_storage_resource_to(*thread_resources_ptr); if (tsrm_new_thread_begin_handler) { tsrm_new_thread_begin_handler(thread_id); @@ -402,17 +417,14 @@ static void allocate_new_resource(tsrm_tls_entry **thread_resources_ptr, THREAD_ if (tsrm_new_thread_end_handler) { tsrm_new_thread_end_handler(thread_id); } - - tsrm_mutex_unlock(tsmm_mutex); }/*}}}*/ - /* fetches the requested resource for the current thread */ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id) {/*{{{*/ THREAD_T thread_id; int hash_value; - tsrm_tls_entry *thread_resources; + tsrm_tls_entry *thread_resources, **last_thread_resources; if (!th_id) { /* Fast path for looking up the resources for the current @@ -443,25 +455,55 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id) if (!thread_resources) { allocate_new_resource(&tsrm_tls_table[hash_value], thread_id); + tsrm_mutex_unlock(tsmm_mutex); return ts_resource_ex(id, &thread_id); } else { - do { - if (thread_resources->thread_id == thread_id) { - break; - } + last_thread_resources = &tsrm_tls_table[hash_value]; + while (thread_resources->thread_id != thread_id) { + last_thread_resources = &thread_resources->next; if (thread_resources->next) { thread_resources = thread_resources->next; } else { allocate_new_resource(&thread_resources->next, thread_id); + tsrm_mutex_unlock(tsmm_mutex); return ts_resource_ex(id, &thread_id); - /* - * thread_resources = thread_resources->next; - * break; - */ } - } while (thread_resources); + } + } + + /* It's possible that the current thread resources are requested, and that we get here. + * This means that the TSRM key pointer and cached pointer are NULL, but there is still + * a thread resource associated with this ID in the hashtable. This can occur if a thread + * goes away, but its resources are never cleaned up, and then that thread ID is reused. + * Since we don't always have a way to know when a thread goes away, we can't clean up + * the thread's resources before the new thread spawns. + * To solve this issue, we'll free up the old thread resources gracefully (gracefully + * because there might still be resources open like database connection which need to + * be shut down cleanly). After freeing up, we'll create the new resources for this thread + * as if the stale resources never existed in the first place. From that point forward, + * it is as if that situation never occurred. + * The fact that this situation happens isn't that bad because a child process containing + * threads will eventually be respawned anyway by the SAPI, so the stale threads won't last + * forever. */ + TSRM_ASSERT(thread_resources->thread_id == thread_id); + if (thread_id == tsrm_thread_id() && !tsrm_tls_get()) { + tsrm_tls_entry *next = thread_resources->next; + /* In case that extensions don't use the pointer passed from the dtor, but incorrectly + * use the global pointer, we need to setup the global pointer temporarily here. */ + set_thread_local_storage_resource_to(thread_resources); + /* Free up the old resource from the old thread instance */ + ts_free_resources(thread_resources); + free(thread_resources); + /* Allocate a new resource at the same point in the linked list, and relink the next pointer */ + allocate_new_resource(last_thread_resources, thread_id); + thread_resources = *last_thread_resources; + thread_resources->next = next; + /* We don't have to tail-call ts_resource_ex, we can take the fast path to the return + * because we already have the correct pointer. */ } + tsrm_mutex_unlock(tsmm_mutex); + /* Read a specific resource from the thread's resources. * This is called outside of a mutex, so have to be aware about external * changes to the structure as we read it. @@ -474,7 +516,6 @@ TSRM_API void *ts_resource_ex(ts_rsrc_id id, THREAD_T *th_id) void ts_free_thread(void) {/*{{{*/ tsrm_tls_entry *thread_resources; - int i; THREAD_T thread_id = tsrm_thread_id(); int hash_value; tsrm_tls_entry *last=NULL; @@ -487,17 +528,7 @@ void ts_free_thread(void) while (thread_resources) { if (thread_resources->thread_id == thread_id) { - for (i=0; icount; i++) { - if (resource_types_table[i].dtor) { - resource_types_table[i].dtor(thread_resources->storage[i]); - } - } - for (i=0; icount; i++) { - if (!resource_types_table[i].fast_offset) { - free(thread_resources->storage[i]); - } - } - free(thread_resources->storage); + ts_free_resources(thread_resources); if (last) { last->next = thread_resources->next; } else { @@ -531,11 +562,13 @@ void ts_free_id(ts_rsrc_id id) while (p) { if (p->count > j && p->storage[j]) { - if (resource_types_table && resource_types_table[j].dtor) { - resource_types_table[j].dtor(p->storage[j]); - } - if (!resource_types_table[j].fast_offset) { - free(p->storage[j]); + if (resource_types_table) { + if (resource_types_table[j].dtor) { + resource_types_table[j].dtor(p->storage[j]); + } + if (!resource_types_table[j].fast_offset) { + free(p->storage[j]); + } } p->storage[j] = NULL; } @@ -729,13 +762,15 @@ TSRM_API size_t tsrm_get_ls_cache_tcb_offset(void) #if defined(__APPLE__) && defined(__x86_64__) // TODO: Implement support for fast JIT ZTS code ??? return 0; -#elif defined(__x86_64__) && defined(__GNUC__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) +#elif defined(__x86_64__) && defined(__GNUC__) && !defined(__FreeBSD__) && \ + !defined(__OpenBSD__) && !defined(__MUSL__) && !defined(__HAIKU__) size_t ret; asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0" : "=r" (ret)); return ret; -#elif defined(__i386__) && defined(__GNUC__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) +#elif defined(__i386__) && defined(__GNUC__) && !defined(__FreeBSD__) && \ + !defined(__OpenBSD__) && !defined(__MUSL__) && !defined(__HAIKU__) size_t ret; asm ("leal _tsrm_ls_cache@ntpoff,%0" diff --git a/TSRM/TSRM.h b/TSRM/TSRM.h index 941d502226..63b42dc186 100644 --- a/TSRM/TSRM.h +++ b/TSRM/TSRM.h @@ -147,7 +147,7 @@ TSRM_API const char *tsrm_api_name(void); # define __has_attribute(x) 0 #endif -#if !__has_attribute(tls_model) || defined(__FreeBSD__) || defined(__MUSL__) +#if !__has_attribute(tls_model) || defined(__FreeBSD__) || defined(__MUSL__) || defined(__HAIKU__) # define TSRM_TLS_MODEL_ATTR #elif __PIC__ # define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("initial-exec"))) diff --git a/TSRM/tsrm_win32.c b/TSRM/tsrm_win32.c index cfe344e377..0cb4d36d4d 100644 --- a/TSRM/tsrm_win32.c +++ b/TSRM/tsrm_win32.c @@ -30,6 +30,7 @@ #include "tsrm_win32.h" #include "zend_virtual_cwd.h" #include "win32/ioutil.h" +#include "win32/winutil.h" #ifdef ZTS static ts_rsrc_id win32_globals_id; @@ -90,7 +91,7 @@ static void tsrm_win32_dtor(tsrm_win32_globals *globals) TSRM_API void tsrm_win32_startup(void) {/*{{{*/ #ifdef ZTS - ts_allocate_id(&win32_globals_id, sizeof(tsrm_win32_globals), (ts_allocate_ctor)tsrm_win32_ctor, (ts_allocate_ctor)tsrm_win32_dtor); + ts_allocate_id(&win32_globals_id, sizeof(tsrm_win32_globals), (ts_allocate_ctor)tsrm_win32_ctor, (ts_allocate_dtor)tsrm_win32_dtor); #else tsrm_win32_ctor(&win32_globals); #endif @@ -613,6 +614,22 @@ TSRM_API int pclose(FILE *stream) #define DESCRIPTOR_PREFIX "TSRM_SHM_DESCRIPTOR:" #define INT_MIN_AS_STRING "-2147483648" + +#define TSRM_BASE_SHM_KEY_ADDRESS 0x20000000 +/* Returns a number between 0x2000_0000 and 0x3fff_ffff. On Windows, key_t is int. */ +static key_t tsrm_choose_random_shm_key(key_t prev_key) { + unsigned char buf[4]; + if (php_win32_get_random_bytes(buf, 4) != SUCCESS) { + return prev_key + 2; + } + uint32_t n = + ((uint32_t)(buf[0]) << 24) | + (((uint32_t)buf[1]) << 16) | + (((uint32_t)buf[2]) << 8) | + (((uint32_t)buf[3])); + return (n & 0x1fffffff) + TSRM_BASE_SHM_KEY_ADDRESS; +} + TSRM_API int shmget(key_t key, size_t size, int flags) {/*{{{*/ shm_pair *shm; @@ -626,6 +643,9 @@ TSRM_API int shmget(key_t key, size_t size, int flags) shm_handle = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, shm_segment); info_handle = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, shm_info); + } else { + /* IPC_PRIVATE always creates a new segment even if IPC_CREAT flag isn't passed. */ + flags |= IPC_CREAT; } if (!shm_handle && !info_handle) { @@ -662,6 +682,19 @@ TSRM_API int shmget(key_t key, size_t size, int flags) } } + if (key == IPC_PRIVATE) { + /* This should call shm_get with a brand new key id that isn't used yet. See https://man7.org/linux/man-pages/man2/shmget.2.html + * Because extensions such as shmop/sysvshm can be used in userland to attach to shared memory segments, use unpredictable high positive numbers to avoid accidentally conflicting with userland. */ + key = tsrm_choose_random_shm_key(TSRM_BASE_SHM_KEY_ADDRESS); + for (shm_pair *ptr = TWG(shm); ptr < (TWG(shm) + TWG(shm_size)); ptr++) { + if (ptr->descriptor && ptr->descriptor->shm_perm.key == key) { + key = tsrm_choose_random_shm_key(key); + ptr = TWG(shm); + continue; + } + } + } + shm = shm_get(key, NULL); if (!shm) { CloseHandle(shm_handle); @@ -702,7 +735,7 @@ TSRM_API void *shmat(int key, const void *shmaddr, int flags) {/*{{{*/ shm_pair *shm = shm_get(key, NULL); - if (!shm->segment) { + if (!shm || !shm->segment) { return (void*)-1; } @@ -726,7 +759,7 @@ TSRM_API int shmdt(const void *shmaddr) shm_pair *shm = shm_get(0, (void*)shmaddr); int ret; - if (!shm->segment) { + if (!shm || !shm->segment) { return -1; } @@ -746,7 +779,7 @@ TSRM_API int shmctl(int key, int cmd, struct shmid_ds *buf) {/*{{{*/ shm_pair *shm = shm_get(key, NULL); - if (!shm->segment) { + if (!shm || !shm->segment) { return -1; } diff --git a/configure.ac b/configure.ac index 4c51bb85f4..d88ead381b 100644 --- a/configure.ac +++ b/configure.ac @@ -17,7 +17,7 @@ dnl Basic autoconf initialization, generation of config.nice. dnl ---------------------------------------------------------------------------- AC_PREREQ([2.68]) -AC_INIT([PHP],[8.1.27],[https://github.com/php/php-src/issues],[php],[https://www.php.net]) +AC_INIT([PHP],[8.1.29],[https://github.com/php/php-src/issues],[php],[https://www.php.net]) AC_CONFIG_SRCDIR([main/php_version.h]) AC_CONFIG_AUX_DIR([build]) AC_PRESERVE_HELP_ORDER diff --git a/docs/upgrade.md b/docs/upgrade.md index 12a359c29f..c7fa95d188 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -13,3 +13,8 @@ php sync-source-code.php --action run ``` + +## 目录说明 + + pool :持久化目录，存放扩展、 PHP 、依赖库等文件，此目录下的文件不会被主动删除 + var :运行时目录，临时存在一些文件，在完成配置或构建后将被主动清空删除 diff --git a/ext/filter/tests/ghsa-w8qr-v226-r27w.phpt b/ext/filter/tests/ghsa-w8qr-v226-r27w.phpt index 638e5dddbe..0092408ee5 100644 --- a/ext/filter/tests/ghsa-w8qr-v226-r27w.phpt +++ b/ext/filter/tests/ghsa-w8qr-v226-r27w.phpt @@ -38,4 +38,4 @@ bool(false) --- These ones should work --- string(21) "http://test@127.0.0.1" string(50) "http://test@[2001:db8:3333:4444:5555:6666:1.2.3.4]" -string(17) "http://test@[::1]" \ No newline at end of file +string(17) "http://test@[::1]" diff --git a/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_001.phpt b/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_001.phpt index 3a1b4db83a..2873210608 100644 --- a/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_001.phpt +++ b/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_001.phpt @@ -53,4 +53,4 @@ Warning: proc_open(): CreateProcess failed, error code: 2 in %s on line %d --CLEAN-- \ No newline at end of file +?> diff --git a/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_002.phpt b/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_002.phpt index ef340132c6..714836557a 100644 --- a/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_002.phpt +++ b/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_002.phpt @@ -63,4 +63,4 @@ Warning: proc_open(): CreateProcess failed, error code: 2 in %s on line %d --CLEAN-- \ No newline at end of file +?> diff --git a/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_003.phpt b/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_003.phpt index 8e7f9b7f6c..a632965eb9 100644 --- a/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_003.phpt +++ b/ext/standard/tests/general_functions/ghsa-9fcc-425m-g385_003.phpt @@ -547,4 +547,4 @@ bool(false) --CLEAN-- \ No newline at end of file +?> diff --git a/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_1.phpt b/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_1.phpt index 20ffbe1fac..8d0939cdf1 100644 --- a/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_1.phpt +++ b/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_1.phpt @@ -26,4 +26,4 @@ proc_close($proc); --CLEAN-- \ No newline at end of file +?> diff --git a/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_2.phpt b/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_2.phpt index 4767b8206b..a1e39d7ef9 100644 --- a/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_2.phpt +++ b/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_2.phpt @@ -26,4 +26,4 @@ proc_close($proc); --CLEAN-- \ No newline at end of file +?> diff --git a/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_3.phpt b/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_3.phpt index 8a61e82f29..69f12d7b35 100644 --- a/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_3.phpt +++ b/ext/standard/tests/general_functions/ghsa-pc52-254m-w9w7_3.phpt @@ -26,4 +26,4 @@ proc_close($proc); --CLEAN-- \ No newline at end of file +?> diff --git a/ext/standard/tests/password/password_bcrypt_errors.phpt b/ext/standard/tests/password/password_bcrypt_errors.phpt index 54edb01caf..5d823cba02 100644 --- a/ext/standard/tests/password/password_bcrypt_errors.phpt +++ b/ext/standard/tests/password/password_bcrypt_errors.phpt @@ -24,4 +24,4 @@ try { --EXPECT-- Invalid bcrypt cost parameter specified: 3 Invalid bcrypt cost parameter specified: 32 -Bcrypt password must not contain null character \ No newline at end of file +Bcrypt password must not contain null character diff --git a/main/TSRM.h b/main/TSRM.h index 941d502226..63b42dc186 100644 --- a/main/TSRM.h +++ b/main/TSRM.h @@ -147,7 +147,7 @@ TSRM_API const char *tsrm_api_name(void); # define __has_attribute(x) 0 #endif -#if !__has_attribute(tls_model) || defined(__FreeBSD__) || defined(__MUSL__) +#if !__has_attribute(tls_model) || defined(__FreeBSD__) || defined(__MUSL__) || defined(__HAIKU__) # define TSRM_TLS_MODEL_ATTR #elif __PIC__ # define TSRM_TLS_MODEL_ATTR __attribute__((tls_model("initial-exec"))) diff --git a/run-tests.php b/run-tests.php index b718246654..39621a39aa 100755 --- a/run-tests.php +++ b/run-tests.php @@ -103,12 +103,12 @@ function show_usage(): void Do not delete 'all' files, 'php' test file, 'skip' or 'clean' file. - --set-timeout [n] - Set timeout for individual tests, where [n] is the number of + --set-timeout + Set timeout for individual tests, where is the number of seconds. The default value is 60 seconds, or 300 seconds when testing for memory leaks. - --context [n] + --context Sets the number of lines of surrounding context to print for diffs. The default value is 3. @@ -119,8 +119,8 @@ function show_usage(): void 'mem'. The result types get written independent of the log format, however 'diff' only exists when a test fails. - --show-slow [n] - Show all tests that took longer than [n] milliseconds to run. + --show-slow + Show all tests that took longer than milliseconds to run. --no-clean Do not execute clean section if any. @@ -530,7 +530,11 @@ function main(): void $just_save_results = true; break; case '--set-timeout': - $environment['TEST_TIMEOUT'] = $argv[++$i]; + $timeout = $argv[++$i] ?? ''; + if (!preg_match('/^\d+$/', $timeout)) { + error("'$timeout' is not a valid number of seconds, try e.g. --set-timeout 60 for 1 minute"); + } + $environment['TEST_TIMEOUT'] = intval($timeout, 10); break; case '--context': $context_line_count = $argv[++$i] ?? ''; @@ -545,7 +549,11 @@ function main(): void } break; case '--show-slow': - $slow_min_ms = $argv[++$i]; + $slow_min_ms = $argv[++$i] ?? ''; + if (!preg_match('/^\d+$/', $slow_min_ms)) { + error("'$slow_min_ms' is not a valid number of milliseconds, try e.g. --show-slow 1000 for 1 second"); + } + $slow_min_ms = intval($slow_min_ms, 10); break; case '--temp-source': $temp_source = $argv[++$i]; @@ -574,9 +582,10 @@ function main(): void $environment['SKIP_PERF_SENSITIVE'] = 1; if ($switch === '--msan') { $environment['SKIP_MSAN'] = 1; + $environment['MSAN_OPTIONS'] = 'intercept_tls_get_addr=0'; } - $lsanSuppressions = __DIR__ . '/azure/lsan-suppressions.txt'; + $lsanSuppressions = __DIR__ . '/.github/lsan-suppressions.txt'; if (file_exists($lsanSuppressions)) { $environment['LSAN_OPTIONS'] = 'suppressions=' . $lsanSuppressions . ':print_suppressions=0'; @@ -1275,6 +1284,10 @@ function system_with_timeout( } $timeout = $valgrind ? 300 : ($env['TEST_TIMEOUT'] ?? 60); + /* ASAN can cause a ~2-3x slowdown. */ + if (isset($env['SKIP_ASAN'])) { + $timeout *= 3; + } while (true) { /* hide errors from interrupted syscalls */ @@ -1575,6 +1588,10 @@ function run_all_tests_parallel(array $test_files, array $env, $redir_tested): v kill_children($workerProcs); error("Could not find worker stdout in array of worker stdouts, THIS SHOULD NOT HAPPEN."); } + if (feof($workerSock)) { + kill_children($workerProcs); + error("Worker $i died unexpectedly"); + } while (false !== ($rawMessage = fgets($workerSock))) { // work around fgets truncating things if (($rawMessageBuffers[$i] ?? '') !== '') { @@ -1873,6 +1890,9 @@ function run_test(string $php, $file, array $env): string $skipCache = new SkipCache($enableSkipCache, $cfg['keep']['skip']); } + $retried = false; +retry: + $temp_filenames = null; $org_file = $file; $orig_php = $php; @@ -1917,8 +1937,10 @@ function run_test(string $php, $file, array $env): string $tested = $test->getName(); - if ($num_repeats > 1 && $test->hasSection('FILE_EXTERNAL')) { - return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable'); + if ($test->hasSection('FILE_EXTERNAL')) { + if ($num_repeats > 1) { + return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable'); + } } if ($test->hasSection('CAPTURE_STDIO')) { @@ -1966,15 +1988,11 @@ function run_test(string $php, $file, array $env): string } } - if ($num_repeats > 1) { - if ($test->hasSection('CLEAN')) { - return skip_test($tested, $tested_file, $shortname, 'Test with CLEAN might not be repeatable'); - } - if ($test->hasSection('STDIN')) { - return skip_test($tested, $tested_file, $shortname, 'Test with STDIN might not be repeatable'); - } - if ($test->hasSection('CAPTURE_STDIO')) { - return skip_test($tested, $tested_file, $shortname, 'Test with CAPTURE_STDIO might not be repeatable'); + foreach (['CLEAN', 'STDIN', 'CAPTURE_STDIO'] as $section) { + if ($test->hasSection($section)) { + if ($num_repeats > 1) { + return skip_test($tested, $tested_file, $shortname, "Test with $section might not be repeatable"); + } } } @@ -2085,11 +2103,19 @@ function run_test(string $php, $file, array $env): string $ini_settings = $workerID ? ['opcache.cache_id' => "worker$workerID"] : []; // Additional required extensions + $extensions = []; if ($test->hasSection('EXTENSIONS')) { + $extensions = preg_split("/[\n\r]+/", trim($test->getSection('EXTENSIONS'))); + } + if (is_array($IN_REDIRECT) && $IN_REDIRECT['EXTENSIONS'] != []) { + $extensions = array_merge($extensions, $IN_REDIRECT['EXTENSIONS']); + } + + /* Load required extensions */ + if ($extensions != []) { $ext_params = []; settings2array($ini_overwrites, $ext_params); $ext_params = settings2params($ext_params); - $extensions = preg_split("/[\n\r]+/", trim($test->getSection('EXTENSIONS'))); [$ext_dir, $loaded] = $skipCache->getExtensions("$orig_php $pass_options $extra_options $ext_params $no_file_cache"); $ext_prefix = IS_WINDOWS ? "php_" : ""; $missing = []; @@ -2145,8 +2171,10 @@ function run_test(string $php, $file, array $env): string $ini = preg_replace('/{MAIL:(\S+)}/', $replacement, $ini); settings2array(preg_split("/[\n\r]+/", $ini), $ini_settings); - if ($num_repeats > 1 && isset($ini_settings['opcache.opt_debug_level'])) { - return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable'); + if (isset($ini_settings['opcache.opt_debug_level'])) { + if ($num_repeats > 1) { + return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable'); + } } } @@ -2241,6 +2269,7 @@ function run_test(string $php, $file, array $env): string $IN_REDIRECT['via'] = "via [$shortname]\n\t"; $IN_REDIRECT['dir'] = realpath(dirname($file)); $IN_REDIRECT['prefix'] = $tested; + $IN_REDIRECT['EXTENSIONS'] = $extensions; if (!empty($IN_REDIRECT['TESTS'])) { if (is_array($org_file)) { @@ -2671,6 +2700,9 @@ function run_test(string $php, $file, array $env): string } elseif ($test->hasSection('XLEAK')) { $warn = true; $info = " (warn: XLEAK section but test passes)"; + } elseif ($retried) { + $warn = true; + $info = " (warn: Test passed on retry attempt)"; } else { show_result("PASS", $tested, $tested_file, '', $temp_filenames); $junit->markTestAs('PASS', $shortname, $tested); @@ -2700,6 +2732,9 @@ function run_test(string $php, $file, array $env): string } elseif ($test->hasSection('XLEAK')) { $warn = true; $info = " (warn: XLEAK section but test passes)"; + } elseif ($retried) { + $warn = true; + $info = " (warn: Test passed on retry attempt)"; } else { show_result("PASS", $tested, $tested_file, '', $temp_filenames); $junit->markTestAs('PASS', $shortname, $tested); @@ -2710,6 +2745,10 @@ function run_test(string $php, $file, array $env): string $wanted_re = null; } + if (!$passed && !$retried && error_may_be_retried($test, $output)) { + $retried = true; + goto retry; + } // Test failed so we need to report details. if ($failed_headers) { @@ -2834,6 +2873,46 @@ function run_test(string $php, $file, array $env): string return $restype[0] . 'ED'; } +function is_flaky(TestFile $test): bool +{ + if ($test->hasSection('FLAKY')) { + return true; + } + if (!$test->hasSection('FILE')) { + return false; + } + $file = $test->getSection('FILE'); + $flaky_functions = [ + 'disk_free_space', + 'hrtime', + 'microtime', + 'sleep', + 'usleep', + ]; + $regex = '(\b(' . implode('|', $flaky_functions) . ')\()i'; + return preg_match($regex, $file) === 1; +} + +function is_flaky_output(string $output): bool +{ + $messages = [ + '404: page not found', + 'address already in use', + 'connection refused', + 'deadlock', + 'mailbox already exists', + 'timed out', + ]; + $regex = '(\b(' . implode('|', $messages) . ')\b)i'; + return preg_match($regex, $output) === 1; +} + +function error_may_be_retried(TestFile $test, string $output): bool +{ + return is_flaky_output($output) + || is_flaky($test); +} + /** * @return bool|int */ @@ -3395,6 +3474,9 @@ class JUnit 'execution_time' => 0, ]; + /** + * @throws Exception + */ public function __construct(array $env, int $workerID) { // Check whether a junit log is wanted. @@ -3612,6 +3694,9 @@ public function initSuite(string $suite_name): void $this->suites[$suite_name] = self::EMPTY_SUITE + ['name' => $suite_name]; } + /** + * @throws Exception + */ public function stopTimer(string $file_name): void { if (!$this->enabled) { @@ -3809,8 +3894,12 @@ class TestFile 'INI', 'ENV', 'EXTENSIONS', 'SKIPIF', 'XFAIL', 'XLEAK', 'CLEAN', 'CREDITS', 'DESCRIPTION', 'CONFLICTS', 'WHITESPACE_SENSITIVE', + 'FLAKY', ]; + /** + * @throws BorkageException + */ public function __construct(string $fileName, bool $inRedirect) { $this->fileName = $fileName; @@ -3851,6 +3940,9 @@ public function sectionNotEmpty(string $name): bool return !empty($this->sections[$name]); } + /** + * @throws Exception + */ public function getSection(string $name): string { if (!isset($this->sections[$name])) { @@ -3866,7 +3958,7 @@ public function getName(): string public function isCGI(): bool { - return $this->sectionNotEmpty('CGI') + return $this->hasSection('CGI') || $this->sectionNotEmpty('GET') || $this->sectionNotEmpty('POST') || $this->sectionNotEmpty('GZIP_POST') @@ -3887,6 +3979,7 @@ public function setSection(string $name, string $value): void /** * Load the sections of the test file + * @throws BorkageException */ private function readFile(): void { @@ -3949,6 +4042,9 @@ private function readFile(): void fclose($fp); } + /** + * @throws BorkageException + */ private function validateAndProcess(bool $inRedirect): void { // the redirect section allows a set of tests to be reused outside of diff --git a/scripts/dev/bless_tests.php b/scripts/dev/bless_tests.php index 674bbeda8e..fa49647fcf 100644 --- a/scripts/dev/bless_tests.php +++ b/scripts/dev/bless_tests.php @@ -72,6 +72,7 @@ function normalizeOutput(string $out): string { 'Resource ID#%d used as offset, casting to integer (%d)', $out); $out = preg_replace('/string\(\d+\) "([^"]*%d)/', 'string(%d) "$1', $out); + $out = str_replace("\0", '%0', $out); return $out; } @@ -86,6 +87,7 @@ function formatToRegex(string $format): string { $result = str_replace('%x', '[0-9a-fA-F]+', $result); $result = str_replace('%f', '[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?', $result); $result = str_replace('%c', '.', $result); + $result = str_replace('%0', '\0', $result); return "/^$result$/s"; } @@ -113,7 +115,7 @@ function generateMinimallyDifferingOutput(string $out, string $oldExpect) { function insertOutput(string $phpt, string $out): string { return preg_replace_callback('/--EXPECTF?--.*?(--CLEAN--|$)/sD', function($matches) use($out) { - $hasWildcard = preg_match('/%[resSaAwidxfc]/', $out); + $hasWildcard = preg_match('/%[resSaAwidxfc0]/', $out); $F = $hasWildcard ? 'F' : ''; return "--EXPECT$F--\n" . $out . "\n" . $matches[1]; }, $phpt); diff --git a/scripts/dev/check_parameters.php b/scripts/dev/check_parameters.php index 52424def1e..f8e905cc02 100644 --- a/scripts/dev/check_parameters.php +++ b/scripts/dev/check_parameters.php @@ -7,7 +7,7 @@ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | | available through the world-wide-web at the following url: | - | http://www.php.net/license/3_01.txt | + | https://www.php.net/license/3_01.txt | | If you did not receive a copy of the PHP license and are unable to | | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | @@ -31,7 +31,7 @@ $API_params = array( 'a' => array('zval**'), // array 'A' => array('zval**'), // array or object - 'b' => array('zend_bool*'), // boolean + 'b' => array('bool*'), // boolean 'd' => array('double*'), // double 'f' => array('zend_fcall_info*', 'zend_fcall_info_cache*'), // function 'h' => array('HashTable**'), // array as an HashTable* @@ -238,7 +238,7 @@ function check_function($name, $txt, $offset) // nullable arguments case '!': if (in_array($last_char, array('l', 'L', 'd', 'b'))) { - check_param($params, ++$j, 'zend_bool*', $optional); + check_param($params, ++$j, 'bool*', $optional); } break; diff --git a/scripts/dev/makedist b/scripts/dev/makedist index ad5c3c01b2..38553d63d3 100644 --- a/scripts/dev/makedist +++ b/scripts/dev/makedist @@ -5,6 +5,15 @@ # Written by Stig Bakken 1997-05-28. # Adapted to Git by Stanislav Malyshev . +# Check whether gtar is present (GNU tar) +tar="$(which gtar)" +tar="${tar:-$(which tar)}" + +if [[ $($tar --version) == *"bsdtar"* ]]; then + echo "Found bsdtar at $tar, but this script needs GNU tar." + exit 1 +fi + # Go to project root directory. cd $(CDPATH= cd -- "$(dirname -- "$0")/../../" && pwd -P) @@ -124,7 +133,7 @@ fi # Export PHP. echo "makedist: Exporting $treeish from $git" -git archive --format=tar $remote_option --prefix=$prefix/ $treeish | tar xvf - || exit 4 +git archive --format=tar $remote_option --prefix=$prefix/ $treeish | "$tar" xvf - || exit 4 cd $prefix || exit 5 @@ -166,7 +175,7 @@ cd .. echo "" echo "makedist: Creating $prefix.tar archive." -tar cf "$prefix".tar "$prefix" +"$tar" cf "$prefix".tar "$prefix" rm -rf "$prefix" "$prefix".tar.* echo "makedist: Creating $prefix.tar.gz archive." diff --git a/scripts/dev/search_underscores.php b/scripts/dev/search_underscores.php index bd74ea2af7..4f2f8e8eeb 100644 --- a/scripts/dev/search_underscores.php +++ b/scripts/dev/search_underscores.php @@ -8,7 +8,7 @@ | This source file is subject to version 3.01 of the PHP license, | | that is bundled with this package in the file LICENSE, and is | | available through the world-wide-web at the following url: | - | http://www.php.net/license/3_01.txt | + | https://www.php.net/license/3_01.txt | | If you did not receive a copy of the PHP license and are unable to | | obtain it through the world-wide-web, please send a note to | | license@php.net so we can mail you a copy immediately. | diff --git a/scripts/man1/php-config.1.in b/scripts/man1/php-config.1.in index dc1317b503..20f09d611f 100644 --- a/scripts/man1/php-config.1.in +++ b/scripts/man1/php-config.1.in @@ -72,7 +72,7 @@ that is bundled with this package in the file LICENSE, and is available through the world-wide-web at the following url: .PD 0 .P -.B http://www.php.net/license/3_01.txt +.B https://www.php.net/license/3_01.txt .PD 1 .P If you did not receive a copy of the PHP license and are unable to diff --git a/scripts/man1/phpize.1.in b/scripts/man1/phpize.1.in index 62613d347c..308b51e03d 100644 --- a/scripts/man1/phpize.1.in +++ b/scripts/man1/phpize.1.in @@ -39,7 +39,7 @@ that is bundled with this package in the file LICENSE, and is available through the world-wide-web at the following url: .PD 0 .P -.B http://www.php.net/license/3_01.txt +.B https://www.php.net/license/3_01.txt .PD 1 .P If you did not receive a copy of the PHP license and are unable to diff --git a/scripts/phpize.m4 b/scripts/phpize.m4 index cdd0e12989..1f0cbd70eb 100644 --- a/scripts/phpize.m4 +++ b/scripts/phpize.m4 @@ -170,6 +170,14 @@ CXXFLAGS_CLEAN='$(CXXFLAGS)' test "$prefix" = "NONE" && prefix="/usr/local" test "$exec_prefix" = "NONE" && exec_prefix='$(prefix)' +if test "$cross_compiling" = yes ; then + AC_MSG_CHECKING(for native build C compiler) + AC_CHECK_PROGS(BUILD_CC, [gcc clang c99 c89 cc cl],none) + AC_MSG_RESULT($BUILD_CC) +else + BUILD_CC=$CC +fi + PHP_SUBST(PHP_MODULES) PHP_SUBST(PHP_ZEND_EX) @@ -202,6 +210,7 @@ PHP_SUBST(SHARED_LIBTOOL) PHP_SUBST(LIBTOOL) PHP_SUBST(SHELL) PHP_SUBST(INSTALL_HEADERS) +PHP_SUBST(BUILD_CC) PHP_GEN_BUILD_DIRS PHP_GEN_GLOBAL_MAKEFILE diff --git a/setup-php-runtime.sh b/setup-php-runtime.sh index 3cb838eeae..5f0f8f929d 100644 --- a/setup-php-runtime.sh +++ b/setup-php-runtime.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -exu __DIR__=$( @@ -48,7 +48,8 @@ case $ARCH in ;; esac -SWOOLE_VERSION='v5.1.3' +APP_VERSION='v5.1.3' +APP_NAME='swoole-cli' VERSION='v5.1.3.0' mkdir -p bin/runtime @@ -56,12 +57,12 @@ mkdir -p var/runtime cd ${__PROJECT__}/var/runtime -SWOOLE_CLI_DOWNLOAD_URL="https://github.com/swoole/swoole-cli/releases/download/${VERSION}/swoole-cli-${SWOOLE_VERSION}-${OS}-${ARCH}.tar.xz" +APP_DOWNLOAD_URL="https://github.com/swoole/swoole-cli/releases/download/${VERSION}/${APP_NAME}-${APP_VERSION}-${OS}-${ARCH}.tar.xz" COMPOSER_DOWNLOAD_URL="https://getcomposer.org/download/latest-stable/composer.phar" CACERT_DOWNLOAD_URL="https://curl.se/ca/cacert.pem" if [ $OS = 'windows' ]; then - SWOOLE_CLI_DOWNLOAD_URL="https://github.com/swoole/swoole-cli/releases/download/${VERSION}/swoole-cli-${SWOOLE_VERSION}-cygwin-${ARCH}.zip" + APP_DOWNLOAD_URL="https://github.com/swoole/swoole-cli/releases/download/${VERSION}/${APP_NAME}-${APP_VERSION}-cygwin-${ARCH}.zip" fi MIRROR='' @@ -89,10 +90,10 @@ done case "$MIRROR" in china) - SWOOLE_CLI_DOWNLOAD_URL="https://wenda-1252906962.file.myqcloud.com/dist/swoole-cli-${SWOOLE_VERSION}-${OS}-${ARCH}.tar.xz" + APP_DOWNLOAD_URL="https://wenda-1252906962.file.myqcloud.com/dist/${APP_NAME}-${APP_VERSION}-${OS}-${ARCH}.tar.xz" COMPOSER_DOWNLOAD_URL="https://mirrors.tencent.com/composer/composer.phar" if [ $OS = 'windows' ]; then - SWOOLE_CLI_DOWNLOAD_URL="https://wenda-1252906962.file.myqcloud.com/dist/swoole-cli-${SWOOLE_VERSION}-cygwin-${ARCH}.zip" + APP_DOWNLOAD_URL="https://wenda-1252906962.file.myqcloud.com/dist/${APP_NAME}-${APP_VERSION}-cygwin-${ARCH}.zip" fi ;; @@ -103,25 +104,20 @@ chmod a+x composer.phar test -f cacert.pem || curl -LSo cacert.pem ${CACERT_DOWNLOAD_URL} -SWOOLE_CLI_RUNTIME="swoole-cli-${SWOOLE_VERSION}-${OS}-${ARCH}" +APP_RUNTIME="${APP_NAME}-${APP_VERSION}-${OS}-${ARCH}" if [ $OS = 'windows' ]; then { - SWOOLE_CLI_RUNTIME="swoole-cli-${SWOOLE_VERSION}-cygwin-${ARCH}" - test -f ${SWOOLE_CLI_RUNTIME}.zip || curl -LSo ${SWOOLE_CLI_RUNTIME}.zip ${SWOOLE_CLI_DOWNLOAD_URL} - test -d ${SWOOLE_CLI_RUNTIME} && rm -rf ${SWOOLE_CLI_RUNTIME} - unzip "${SWOOLE_CLI_RUNTIME}.zip" - test -d ${__PROJECT__}/${SWOOLE_CLI_RUNTIME} && rm -rf ${__PROJECT__}/${SWOOLE_CLI_RUNTIME} - cp -f composer.phar ${SWOOLE_CLI_RUNTIME}/bin/ - #cp -f ${SWOOLE_CLI_RUNTIME}/bin/swoole-cli.exe ${SWOOLE_CLI_RUNTIME}/bin/php.exe - mv ${SWOOLE_CLI_RUNTIME} ${__PROJECT__} - echo + APP_RUNTIME="${APP_NAME}-${APP_VERSION}-cygwin-${ARCH}" + test -f ${APP_RUNTIME}.zip || curl -LSo ${APP_RUNTIME}.zip ${APP_DOWNLOAD_URL} + test -d ${APP_RUNTIME} && rm -rf ${APP_RUNTIME} + unzip "${APP_RUNTIME}.zip" exit 0 } else - test -f ${SWOOLE_CLI_RUNTIME}.tar.xz || curl -LSo ${SWOOLE_CLI_RUNTIME}.tar.xz ${SWOOLE_CLI_DOWNLOAD_URL} - test -f ${SWOOLE_CLI_RUNTIME}.tar || xz -d -k ${SWOOLE_CLI_RUNTIME}.tar.xz - test -f swoole-cli || tar -xvf ${SWOOLE_CLI_RUNTIME}.tar + test -f ${APP_RUNTIME}.tar.xz || curl -LSo ${APP_RUNTIME}.tar.xz ${APP_DOWNLOAD_URL} + test -f ${APP_RUNTIME}.tar || xz -d -k ${APP_RUNTIME}.tar.xz + test -f swoole-cli || tar -xvf ${APP_RUNTIME}.tar chmod a+x swoole-cli cp -f ${__PROJECT__}/var/runtime/swoole-cli ${__PROJECT__}/bin/runtime/php fi @@ -135,6 +131,20 @@ cat >${__PROJECT__}/bin/runtime/php.ini <