diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index 6b1c0092ce4816..468b7e8639db2b 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -416,7 +416,13 @@ _multiprocessing_SemLock__get_value_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock__get_value(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__get_value_impl((SemLockObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock__get_value_impl((SemLockObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -438,7 +444,13 @@ _multiprocessing_SemLock__is_zero_impl(SemLockObject *self); static PyObject * _multiprocessing_SemLock__is_zero(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _multiprocessing_SemLock__is_zero_impl((SemLockObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _multiprocessing_SemLock__is_zero_impl((SemLockObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_MP_SEMAPHORE) */ @@ -582,4 +594,4 @@ _multiprocessing_SemLock___exit__(PyObject *self, PyObject *const *args, Py_ssiz #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=d1e349d4ee3d4bbf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73eacaba84db92de input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/dump_shm_macosx/dump_shared_mem.c b/Modules/_multiprocessing/dump_shm_macosx/dump_shared_mem.c deleted file mode 100644 index aecc8d9a9b000b..00000000000000 --- a/Modules/_multiprocessing/dump_shm_macosx/dump_shared_mem.c +++ /dev/null @@ -1,102 +0,0 @@ -#include -#include // puts, printf, scanf -#include // ctime, time -#include // memcpy, memcmp - -#include // sem_t -typedef sem_t *SEM_HANDLE; - -#include "../semaphore_macosx.h" -#include "./shared_mem.h" - -// Static datas for each process. -CountersWorkaround shm_semlock_counters = { - .state_this = THIS_NOT_OPEN, - .name_shm = SHAREDMEM_NAME, - .handle_shm = (MEMORY_HANDLE)0, - .name_shm_lock = GLOCK_NAME, - .handle_shm_lock = (SEM_HANDLE)0, - .header = (HeaderObject *)NULL, - .counters = (CounterObject *)NULL, -}; - -HeaderObject *header = NULL; -CounterObject *counter = NULL; - -#define MAX_SEMAPHORES_SHOW 32 - -static char *show_counter(char *p, CounterObject *counter) { - sprintf(p, "p:%p, n:%s, v:%d, t:%s", counter, - counter->sem_name, - counter->internal_value, - ctime(&counter->ctimestamp)); - return p; -} - -static void dump_shm_semlock_counters(void) { -puts(__func__); - - char buf[256]; - int i = 0, j = 0; - - if (shm_semlock_counters.state_this == THIS_AVAILABLE) { - CounterObject *counter = shm_semlock_counters.counters; - HeaderObject *header = shm_semlock_counters.header; - dump_shm_semlock_header_counters(); - dump_shm_semlock_header(); - int show_max = header->n_semlocks > MAX_SEMAPHORES_SHOW ? MAX_SEMAPHORES_SHOW : header->n_semlocks; - for(; i < header->n_slots && j < show_max; i++, counter++ ) { - if (counter->sem_name[0] != 0) { - printf("%s", show_counter(buf, counter)); - ++j; - } - } - if (show_max < header->n_semlocks) { - printf("......\n--------- More %d Semphores ---------\n", header->n_semlocks-show_max); - } - } -} - -int main(int argc, char *argv[]) { - int repeat = 0; - long udelay = 5000; - HeaderObject save = {0}; - int unlink = 0; - int force_open = 1; - int releases_lock = 1; - - puts("--------"); - printf("PID:%d, PPID:%d\n", getpid(), getppid()); - connect_shm_semlock_counters(unlink, force_open, releases_lock); - puts("+++++++++"); - if (argc > 1) { - sscanf(argv[1], "%d", &repeat); - if (argc > 2) { - puts(argv[2]); - sscanf(argv[2], "%lu", &udelay); - } - } else { - puts("dump_shared_mem where:\n repeat (-1 " - "is infinite) and a delay (us) between two dumps \n"); - return 1; - } - - printf("Repeat:%d, udelay:%lu\n", repeat, udelay); - - if (shm_semlock_counters.state_this == THIS_AVAILABLE) { - memset(&save, '\0', sizeof(save)); - do { - ACQUIRE_SHM_LOCK; - if (memcmp(&save, shm_semlock_counters.header, sizeof(HeaderObject)) ) { - time_t timestamp = time(NULL); - puts(ctime(×tamp)); - dump_shm_semlock_counters(); - memcpy(&save, shm_semlock_counters.header, sizeof(HeaderObject)); - puts("=========="); - } - RELEASE_SHM_LOCK; - usleep(udelay); - } while(repeat--); - } - return 1; -} diff --git a/Modules/_multiprocessing/dump_shm_macosx/make_all.sh b/Modules/_multiprocessing/dump_shm_macosx/make_all.sh deleted file mode 100755 index 2a1d6ded19a3d7..00000000000000 --- a/Modules/_multiprocessing/dump_shm_macosx/make_all.sh +++ /dev/null @@ -1,2 +0,0 @@ -gcc -o ./dump_shm ./dump_shared_mem.c ./shared_mem.c -gcc -o ./reset_shm ./reset_shared_mem.c ./shared_mem.c diff --git a/Modules/_multiprocessing/dump_shm_macosx/readme.md b/Modules/_multiprocessing/dump_shm_macosx/readme.md deleted file mode 100644 index 826579e11bc433..00000000000000 --- a/Modules/_multiprocessing/dump_shm_macosx/readme.md +++ /dev/null @@ -1,41 +0,0 @@ -** For MacOSX only ** ---- - -This directory contains 2 programs : - -* dump_shared_mem: view content of shared memory. -* reset_shared_mem: erase all stored datas of shared memory. - -the `make_all.sh` batch builds these 2 programs. - -# dump_shm. - -`dump_shm` tries to connect to the shared memory only if its exists. -This program doesn't use synchronization primitive to read the shared memory. -To quit this program, press `Ctrl+C`. - -```zsh -dump_shm -1 300 -``` -Executes this program forever, and check all 300 *us* if shared memory changes. - -When there are changes in the shared memory (only about sempahore count), program prints the new content of shared memory as below: - -```zsh -========== -Tue Feb 25 17:04:05 2025 - -dump_shm_semlock_counters -header:0x1022b4000 - counter array:0x1022b4010 -n sems:2 - n sem_slots:87551, n procs:1, size_shm:2801664 -p:0x1022b4010, n:/mp-fwl20ahw, v:6, r:0, t:Tue Feb 25 17:04:05 2025 -p:0x1022b4030, n:/mp-z3635cdr, v:6, r:0, t:Tue Feb 25 17:04:04 2025 - -``` - -# reset_shm. - -`reset_shm` tries to connect to the shared memory only if its exists. -This program uses synchronization primitive to read the shared memory. - -When exits, this program calls `shm_unlink`. \ No newline at end of file diff --git a/Modules/_multiprocessing/dump_shm_macosx/reset_shared_mem.c b/Modules/_multiprocessing/dump_shm_macosx/reset_shared_mem.c deleted file mode 100644 index b884e312880094..00000000000000 --- a/Modules/_multiprocessing/dump_shm_macosx/reset_shared_mem.c +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include // puts, printf, scanf -#include // isupper -#include // memset - -#include // sem_t -typedef sem_t *SEM_HANDLE; - -#include "../semaphore_macosx.h" -#include "./shared_mem.h" - -// Static datas for each process. -CountersWorkaround shm_semlock_counters = { - .state_this = THIS_NOT_OPEN, - .name_shm = SHAREDMEM_NAME, - .handle_shm = (MEMORY_HANDLE)0, - .name_shm_lock = GLOCK_NAME, - .handle_shm_lock = (SEM_HANDLE)0, - .header = (HeaderObject *)NULL, - .counters = (CounterObject *)NULL, -}; - -HeaderObject *header = NULL; -CounterObject *counter = NULL; - -static void reset_shm_semlock_counters(int size, int nb_slots) { -puts(__func__); - - if (shm_semlock_counters.state_this == THIS_AVAILABLE) { - ACQUIRE_SHM_LOCK; - CounterObject *counter = shm_semlock_counters.counters; - HeaderObject *header = shm_semlock_counters.header; - dump_shm_semlock_header_counters(); - dump_shm_semlock_header(); - long size_to_reset = header->size_shm-sizeof(HeaderObject); - printf("1 - size to reset:%lu\n", size_to_reset); - if (size && size <= size_to_reset) { - memset(counter, 0, size); - - } else { - memset(counter, 0, size_to_reset); - } - puts("2 - Reset all header parameters"); - if (nb_slots) { - header->n_slots = nb_slots; - } - header->n_semlocks = 0; - header->n_slots = CALC_NB_SLOTS(header->size_shm); - header->n_procs = 0; - dump_shm_semlock_header(); - RELEASE_SHM_LOCK; - } -} - -int main(int argc, char *argv[]) { - char c; - int size = CALC_SIZE_SHM; - int nb_slots = CALC_NB_SLOTS(size); - int unlink = 1; - int force_open = 1; - int release_lock = 1; - - if (argc >= 2) { - sscanf(argv[2], "%d", &size); - nb_slots = CALC_NB_SLOTS(size); - } - puts("--------"); - printf("size:%d, sem slots:%d\n", size, nb_slots); - connect_shm_semlock_counters(unlink, force_open, release_lock); - puts("+++++++++"); - dump_shm_semlock_header_counters(); - dump_shm_semlock_header(); - if (shm_semlock_counters.state_this == THIS_AVAILABLE) { - puts("confirm (Y/N):"); - c = getchar(); - if ( isupper(c) == 'Y') { - reset_shm_semlock_counters(size, nb_slots); - } - } - return 1; -} diff --git a/Modules/_multiprocessing/dump_shm_macosx/shared_mem.c b/Modules/_multiprocessing/dump_shm_macosx/shared_mem.c deleted file mode 100644 index 9d33646e7eb43b..00000000000000 --- a/Modules/_multiprocessing/dump_shm_macosx/shared_mem.c +++ /dev/null @@ -1,166 +0,0 @@ -#include /* O_CREAT and O_EXCL */ -#include // signal -#include // printf, puts -#include // atexit -#include // errno -#include // sysconf - -#include // sem_open -typedef sem_t *SEM_HANDLE; - -#include "../semaphore_macosx.h" -#include "shared_mem.h" - -void sigterm(int code) { - exit(EXIT_SUCCESS); -} - -int acquire_lock(SEM_HANDLE sem) { - sem_wait(sem); - return 1; -} - -int release_lock(SEM_HANDLE sem) { - sem_post(sem); - return 1; -} - -int exist_lock(SEM_HANDLE sem) { - int res = -1 ; - - errno = 0; - res = sem_trywait(sem); - if (res < 0 ) { - if (errno == EBADF) { - puts("global lock does not exist"); - shm_semlock_counters.state_this = THIS_NOT_OPEN; - return 0; - } - return 0; - } - if (sem_post(sem) < 0) { - return 0; - } - return 1; -} - -void connect_shm_semlock_counters(int unlink, int force_open, int call_release_lock) { -puts(__func__); - - int oflag = O_RDWR; - int shm = -1; - int res = -1; - SEM_HANDLE sem = SEM_FAILED; - long size_shm_init = CALC_SIZE_SHM; - long size_shm = ALIGN_SHM_PAGE(size_shm_init); - - // printf("size1: %lu vs size2:%lu\n", size_shm_init, size_shm); - - // Install signals. - signal(SIGTERM, &sigterm); - signal(SIGINT, &sigterm); - - errno = 0; - if (sem == SEM_FAILED) { - errno = 0; - // Semaphore exists, just opens it. - printf("Try to open glock '%s'\n", shm_semlock_counters.name_shm_lock); - sem = sem_open(shm_semlock_counters.name_shm_lock, 0); - // Not exists, creates it. - if (force_open && sem == SEM_FAILED) { - sem = sem_open(shm_semlock_counters.name_shm_lock, O_CREAT, 0600, 1); - } - } - printf("sem:%p\n", sem); - shm_semlock_counters.handle_shm_lock = sem; - - if (call_release_lock) { - RELEASE_SHM_LOCK; - } - - // Locks to semaphore. - ACQUIRE_SHM_LOCK; - printf("Shm Lock ok on %p\n", sem); - // connect to Shared mem - shm = shm_open(shm_semlock_counters.name_shm, oflag, 0); - if (shm != -1) { - shm_semlock_counters.handle_shm = shm; - printf("Shared Mem ok on '%d'\n", shm); - char *ptr = (char *)mmap(NULL, - size_shm, - (PROT_WRITE | PROT_READ), - MAP_SHARED, - shm_semlock_counters.handle_shm, - 0L); - shm_semlock_counters.header = (HeaderObject *)ptr; - shm_semlock_counters.counters = (CounterObject *)(ptr+sizeof(HeaderObject)); - printf("Shared memory size is %lu vs %d\n", size_shm, - shm_semlock_counters.header->size_shm); - // Initialization is successful. - shm_semlock_counters.state_this = THIS_AVAILABLE; - header = shm_semlock_counters.header; - counter = shm_semlock_counters.counters; - if (unlink) { - atexit(delete_shm_semlock_counters); - - } else { - atexit(delete_shm_semlock_counters_without_unlink); - } - puts("Ok...."); - } else { - printf("The shared memory '%s' does not exist\n", shm_semlock_counters.name_shm); - } - RELEASE_SHM_LOCK; - printf("Shm Unlock ok on %p\n", sem); - -} - -void _delete_shm_semlock_counters(int unlink) { - - puts("clean up..."); - if (shm_semlock_counters.state_this == THIS_AVAILABLE) { - if (shm_semlock_counters.counters) { - ACQUIRE_SHM_LOCK; - // unmmap - munmap(shm_semlock_counters.counters, - shm_semlock_counters.header->size_shm); - if (unlink) { - shm_unlink(shm_semlock_counters.name_shm); - } - shm_semlock_counters.state_this = THIS_CLOSED; - RELEASE_SHM_LOCK; - } - } - // close lock - sem_close(shm_semlock_counters.handle_shm_lock); - if (unlink) { - sem_unlink(shm_semlock_counters.name_shm_lock); - } -} - - - -void delete_shm_semlock_counters_without_unlink(void) { -puts(__func__); - _delete_shm_semlock_counters(0); -} - -void delete_shm_semlock_counters(void) { -puts(__func__); - _delete_shm_semlock_counters(1); -} - -void dump_shm_semlock_header(void) { - if (shm_semlock_counters.state_this == THIS_AVAILABLE) { - printf("n sems:%d - n sem_slots:%d, n procs:%d, size_shm:%d\n", header->n_semlocks, - header->n_slots, - header->n_procs, - header->size_shm); - } -} - -void dump_shm_semlock_header_counters(void) { - if (shm_semlock_counters.state_this == THIS_AVAILABLE) { - printf("header:%p - counter array:%p\n", header, counter); - } -} diff --git a/Modules/_multiprocessing/dump_shm_macosx/shared_mem.h b/Modules/_multiprocessing/dump_shm_macosx/shared_mem.h deleted file mode 100644 index e24a5265dad4a9..00000000000000 --- a/Modules/_multiprocessing/dump_shm_macosx/shared_mem.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef SHARED_MEM_H -#define SHARED_MEM_H - -extern CountersWorkaround shm_semlock_counters; -extern HeaderObject *header; -extern CounterObject *counter; - -extern int acquire_lock(SEM_HANDLE sem); -extern int release_lock(SEM_HANDLE sem); -extern int exist_lock(SEM_HANDLE sem); - -void connect_shm_semlock_counters(int unlink, int force_connect, int release_lock); -void delete_shm_semlock_counters_without_unlink(void); -void delete_shm_semlock_counters(void); - -void dump_shm_semlock_header(void); -void dump_shm_semlock_header_counters(void); - -#endif /* SHARED_MEM_H */ diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index e989b28fb30e98..62686ed5a9e279 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -375,6 +375,8 @@ CountersWorkaround shm_semlock_counters = { .counters = (CounterObject *)NULL, }; +static PyMutex shm_counters_mutex = {0}; + /* SemLockObject with aditionnal members: + a mutex to handle safely the associated CounterObject. @@ -423,6 +425,11 @@ exists_lock(SEM_HANDLE handle) { int res = -1; int err = 0; + if (handle == NULL || handle == SEM_FAILED) { + shm_semlock_counters.state_this = THIS_NOT_OPEN; + return 0; + } + errno = 0; do { res = sem_trywait(handle); @@ -558,6 +565,7 @@ connect_shm_lock_and_lock(const char *sem_name) { static int delete_shm_lock(void) { + // Note: caller must hold shm_counters_mutex int res = -1; if (shm_semlock_counters.handle_shm_lock != SEM_FAILED) { @@ -1041,13 +1049,15 @@ _multiprocessing_SemLock_impl(PyTypeObject *type, int kind, int value, #ifdef HAVE_BROKEN_SEM_GETVALUE semlock = _SemLockObject_CAST(result); if (ISSEMAPHORE(semlock)) { + PyMutex_Lock(&shm_counters_mutex); if (!exists_lock(shm_semlock_counters.handle_shm_lock) || shm_semlock_counters.state_this != THIS_AVAILABLE) { if (create_shm_semlock_counters(name) < 0) { - + PyMutex_Unlock(&shm_counters_mutex); goto failure; } } + PyMutex_Unlock(&shm_counters_mutex); // error is set in ACQUIRE/RELEASE_* macros. if (!ACQUIRE_SHM_LOCK) // error set in acquire_lock function @@ -1163,13 +1173,16 @@ _multiprocessing_SemLock__rebuild_impl(PyTypeObject *type, SEM_HANDLE handle, #ifdef HAVE_BROKEN_SEM_GETVALUE semlock = _SemLockObject_CAST(result); if (ISSEMAPHORE(semlock)) { + PyMutex_Lock(&shm_counters_mutex); if (!exists_lock(shm_semlock_counters.handle_shm_lock) || shm_semlock_counters.state_this != THIS_AVAILABLE) { if (create_shm_semlock_counters(name) < 0) { + PyMutex_Unlock(&shm_counters_mutex); strcpy(fail, "create_shm_semlock_counters failed"); goto failure; } } + PyMutex_Unlock(&shm_counters_mutex); DEBUG_PID_FUNC(name_copy, 0, counter, "init"); if(!ACQUIRE_SHM_LOCK) { @@ -1246,6 +1259,7 @@ semlock_dealloc(SemLockObject* self) SEM_CLOSE(self->handle_mutex); } /* Case of fork with MacOSX */ + PyMutex_Lock(&shm_counters_mutex); ACQUIRE_SHM_LOCK; DEBUG_PID_FUNC(self->name, self->handle_mutex, self->counter, ""); if (self->created && self->counter) { @@ -1256,6 +1270,7 @@ semlock_dealloc(SemLockObject* self) if (!res ) { delete_shm_lock(); } + PyMutex_Unlock(&shm_counters_mutex); } #endif /* HAVE_BROKEN_SEM_GETVALUE */ @@ -1294,9 +1309,8 @@ _multiprocessing_SemLock__is_mine_impl(SemLockObject *self) return PyBool_FromLong(ISMINE(self)); } -PyObject * _multiprocessing_SemLock__is_zero_impl(SemLockObject *self); - /*[clinic input] +@critical_section _multiprocessing.SemLock._get_value Get the value of the semaphore. @@ -1304,13 +1318,19 @@ Get the value of the semaphore. static PyObject * _multiprocessing_SemLock__get_value_impl(SemLockObject *self) -/*[clinic end generated code: output=64bc1b89bda05e36 input=cb10f9a769836203]*/ +/*[clinic end generated code: output=64bc1b89bda05e36 input=b900f9766c864d35]*/ { int sval = -1; #ifdef HAVE_BROKEN_SEM_GETVALUE if (self->maxvalue <= 1) { - return PyLong_FromLong((long)Py_IsFalse(_multiprocessing_SemLock__is_zero_impl(self))); + PyObject *is_zero = _multiprocessing_SemLock__is_zero_impl(self); + if (is_zero == NULL) { + return NULL; + } + long val = (long)Py_IsFalse(is_zero); + Py_DECREF(is_zero); + return PyLong_FromLong(val); } // error is set in ACQUIRE/RELEASE_* macros. @@ -1340,6 +1360,7 @@ _multiprocessing_SemLock__get_value_impl(SemLockObject *self) } /*[clinic input] +@critical_section _multiprocessing.SemLock._is_zero Return whether semaphore has value zero. @@ -1347,7 +1368,7 @@ Return whether semaphore has value zero. static PyObject * _multiprocessing_SemLock__is_zero_impl(SemLockObject *self) -/*[clinic end generated code: output=815d4c878c806ed7 input=294a446418d31347]*/ +/*[clinic end generated code: output=815d4c878c806ed7 input=7401329b1f0f059c]*/ { #ifdef HAVE_BROKEN_SEM_GETVALUE if (sem_trywait(self->handle) < 0) {