diff --git a/mysql-test/main/spatial_utility_function_simplify.result b/mysql-test/main/spatial_utility_function_simplify.result index 435447025d5ec..6fb373fc45310 100644 --- a/mysql-test/main/spatial_utility_function_simplify.result +++ b/mysql-test/main/spatial_utility_function_simplify.result @@ -557,5 +557,25 @@ SELECT ST_ASTEXT(ST_SIMPLIFY(ST_GEOMFROMTEXT('POLYGON((0 0,0 10,10 10,10 0,0 0,0 ERROR HY000: Incorrect arguments to st_simplify SELECT ST_ASTEXT(ST_SIMPLIFY(ST_GEOMFROMTEXT('POLYGON((0 0,0 10,10 10,10 0,0 0,0 0,0 0,0 0))'), a)); ERROR 42S22: Unknown column 'a' in 'SELECT' +# +# MDEV-39911 Crash in ST_SIMPLIFY of a collection geometry +# +# ST_SIMPLIFY of a collection reserved less space than the WKB header it +# writes, so the element count overran a small buffer. CONCAT_WS +# builds its separator in a ten byte stack buffer and COLUMN_GET builds its +# column name in an eleven byte stack buffer. Each query below crashed +# before the fix. +SELECT CONCAT_WS(ST_SIMPLIFY(ST_GEOMFROMTEXT('MULTILINESTRING((0 0,5 5,0 10),(0 0,-5 5,0 10))'), 5), 7, 5) IS NOT NULL AS ok; +ok +1 +SELECT CONCAT_WS(ST_SIMPLIFY(ST_GEOMFROMTEXT('POLYGON((0 0,10 0,15 5,10 10,0 10,-5 5,0 0))'), 0.0001), 7, 5) IS NOT NULL AS ok; +ok +1 +SELECT CONCAT_WS(ST_SIMPLIFY(ST_GEOMFROMTEXT('MULTIPOLYGON(((0 0,10 0,15 5,10 10,0 10,-5 5,0 0)))'), 0.0001), 7, 5) IS NOT NULL AS ok; +ok +1 +SELECT COLUMN_GET(COLUMN_CREATE('a', 1), ST_SIMPLIFY(ST_GEOMFROMTEXT('GEOMETRYCOLLECTION(MULTILINESTRING((0 0,5 5,0 10),(0 0,-5 5,0 10)))'), 5) AS CHAR) IS NULL AS ok; +ok +1 # Clean up DROP TABLE gis_geometrycollection; diff --git a/mysql-test/main/spatial_utility_function_simplify.test b/mysql-test/main/spatial_utility_function_simplify.test index 8ef99c7e2cc6c..6e14401a13cba 100644 --- a/mysql-test/main/spatial_utility_function_simplify.test +++ b/mysql-test/main/spatial_utility_function_simplify.test @@ -497,5 +497,19 @@ SELECT ST_ASTEXT(ST_SIMPLIFY(ST_GEOMFROMTEXT('POLYGON((0 0,0 10,10 10,10 0,0 0,0 --error ER_BAD_FIELD_ERROR SELECT ST_ASTEXT(ST_SIMPLIFY(ST_GEOMFROMTEXT('POLYGON((0 0,0 10,10 10,10 0,0 0,0 0,0 0,0 0))'), a)); +--echo # +--echo # MDEV-39911 Crash in ST_SIMPLIFY of a collection geometry +--echo # + +--echo # ST_SIMPLIFY of a collection reserved less space than the WKB header it +--echo # writes, so the element count overran a small buffer. CONCAT_WS +--echo # builds its separator in a ten byte stack buffer and COLUMN_GET builds its +--echo # column name in an eleven byte stack buffer. Each query below crashed +--echo # before the fix. +SELECT CONCAT_WS(ST_SIMPLIFY(ST_GEOMFROMTEXT('MULTILINESTRING((0 0,5 5,0 10),(0 0,-5 5,0 10))'), 5), 7, 5) IS NOT NULL AS ok; +SELECT CONCAT_WS(ST_SIMPLIFY(ST_GEOMFROMTEXT('POLYGON((0 0,10 0,15 5,10 10,0 10,-5 5,0 0))'), 0.0001), 7, 5) IS NOT NULL AS ok; +SELECT CONCAT_WS(ST_SIMPLIFY(ST_GEOMFROMTEXT('MULTIPOLYGON(((0 0,10 0,15 5,10 10,0 10,-5 5,0 0)))'), 0.0001), 7, 5) IS NOT NULL AS ok; +SELECT COLUMN_GET(COLUMN_CREATE('a', 1), ST_SIMPLIFY(ST_GEOMFROMTEXT('GEOMETRYCOLLECTION(MULTILINESTRING((0 0,5 5,0 10),(0 0,-5 5,0 10)))'), 5) AS CHAR) IS NULL AS ok; + --echo # Clean up DROP TABLE gis_geometrycollection; diff --git a/sql/spatial.cc b/sql/spatial.cc index 1d602b8578c76..bf3f62a871ab1 100644 --- a/sql/spatial.cc +++ b/sql/spatial.cc @@ -2387,7 +2387,9 @@ int Gis_polygon::simplify(String *result, double max_distance) const return 1; result->length(0); - result->reserve(SRID_SIZE + WKB_HEADER_SIZE); + // Reserve room for the SRID, byte order, type, and ring count. + if (result->reserve(SRID_SIZE + WKB_HEADER_SIZE + sizeof(uint32))) + return 1; result->q_append(SRID_PLACEHOLDER); result->q_append((char) wkb_ndr); result->q_append((uint32) wkb_polygon); @@ -3492,7 +3494,9 @@ int Gis_multi_line_string::simplify(String *result, double max_distance) const return 1; result->length(0); - result->reserve(SRID_SIZE + WKB_HEADER_SIZE); + // Reserve room for the SRID, byte order, type, and line count. + if (result->reserve(SRID_SIZE + WKB_HEADER_SIZE + sizeof(uint32))) + return 1; result->q_append(SRID_PLACEHOLDER); result->q_append((char) wkb_ndr); result->q_append((uint32) wkb_multilinestring); @@ -4099,7 +4103,9 @@ int Gis_multi_polygon::simplify(String *result, double max_distance) const return 1; result->length(0); - result->reserve(SRID_SIZE + WKB_HEADER_SIZE); + // Reserve room for the SRID, byte order, type, and polygon count. + if (result->reserve(SRID_SIZE + WKB_HEADER_SIZE + sizeof(uint32))) + return 1; result->q_append(SRID_PLACEHOLDER); result->q_append((char) wkb_ndr); result->q_append((uint32) wkb_multipolygon); @@ -4694,7 +4700,9 @@ int Gis_geometry_collection::simplify(String *result, return 1; result->length(0); - result->reserve(SRID_SIZE + BYTE_ORDER_SIZE + WKB_HEADER_SIZE); + // Reserve room for the SRID, byte order, type, and geometry count. + if (result->reserve(SRID_SIZE + WKB_HEADER_SIZE + sizeof(uint32))) + return 1; result->q_append(SRID_PLACEHOLDER); result->q_append((char) wkb_ndr); result->q_append((uint32) wkb_geometrycollection);