diff options
169 files changed, 4392 insertions, 2248 deletions
@@ -22,6 +22,7 @@ syntax: regexp ^pypy/module/cpyext/test/.+\.obj$ ^pypy/module/cpyext/test/.+\.manifest$ ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$ +^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$ ^pypy/module/cppyy/src/.+\.o$ ^pypy/module/cppyy/bench/.+\.so$ ^pypy/module/cppyy/bench/.+\.root$ @@ -35,7 +36,6 @@ syntax: regexp ^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$ ^pypy/doc/.+\.html$ ^pypy/doc/config/.+\.rst$ -^pypy/doc/basicblock\.asc$ ^pypy/doc/.+\.svninfo$ ^rpython/translator/c/src/libffi_msvc/.+\.obj$ ^rpython/translator/c/src/libffi_msvc/.+\.dll$ @@ -45,53 +45,33 @@ syntax: regexp ^rpython/translator/c/src/cjkcodecs/.+\.obj$ ^rpython/translator/c/src/stacklet/.+\.o$ ^rpython/translator/c/src/.+\.o$ -^rpython/translator/jvm/\.project$ -^rpython/translator/jvm/\.classpath$ -^rpython/translator/jvm/eclipse-bin$ -^rpython/translator/jvm/src/pypy/.+\.class$ -^rpython/translator/benchmark/docutils$ -^rpython/translator/benchmark/templess$ -^rpython/translator/benchmark/gadfly$ -^rpython/translator/benchmark/mako$ -^rpython/translator/benchmark/bench-custom\.benchmark_result$ -^rpython/translator/benchmark/shootout_benchmarks$ +^rpython/translator/llvm/.+\.so$ ^rpython/translator/goal/target.+-c$ ^rpython/translator/goal/.+\.exe$ ^rpython/translator/goal/.+\.dll$ ^pypy/goal/pypy-translation-snapshot$ ^pypy/goal/pypy-c -^pypy/goal/pypy-jvm -^pypy/goal/pypy-jvm.jar ^pypy/goal/.+\.exe$ ^pypy/goal/.+\.dll$ ^pypy/goal/.+\.lib$ ^pypy/_cache$ -^pypy/doc/statistic/.+\.html$ -^pypy/doc/statistic/.+\.eps$ -^pypy/doc/statistic/.+\.pdf$ -^rpython/translator/cli/src/pypylib\.dll$ -^rpython/translator/cli/src/query\.exe$ -^rpython/translator/cli/src/main\.exe$ +^lib-python/2.7/lib2to3/.+\.pickle$ ^lib_pypy/__pycache__$ ^lib_pypy/ctypes_config_cache/_.+_cache\.py$ ^lib_pypy/ctypes_config_cache/_.+_.+_\.py$ ^lib_pypy/_libmpdec/.+.o$ -^rpython/translator/cli/query-descriptions$ ^pypy/doc/discussion/.+\.html$ ^include/.+\.h$ ^include/.+\.inl$ ^pypy/doc/_build/.*$ ^pypy/doc/config/.+\.html$ ^pypy/doc/config/style\.css$ -^pypy/doc/jit/.+\.html$ -^pypy/doc/jit/style\.css$ ^pypy/doc/image/lattice1\.png$ ^pypy/doc/image/lattice2\.png$ ^pypy/doc/image/lattice3\.png$ ^pypy/doc/image/stackless_informal\.png$ ^pypy/doc/image/parsing_example.+\.png$ ^rpython/doc/_build/.*$ -^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$ ^compiled ^.git/ ^.hypothesis/ @@ -41,29 +41,29 @@ copyrighted by one or more of the following people and organizations: Amaury Forgeot d'Arc Antonio Cuni Samuele Pedroni + Matti Picus Alex Gaynor Brian Kearns - Matti Picus Philip Jenvey Michael Hudson + Ronan Lamy David Schneider + Manuel Jacob Holger Krekel Christian Tismer Hakan Ardo - Manuel Jacob - Ronan Lamy Benjamin Peterson + Richard Plangger Anders Chrigstrom Eric van Riet Paap Wim Lavrijsen - Richard Plangger Richard Emslie Alexander Schremmer Dan Villiom Podlaski Christiansen + Remi Meier Lukas Diekmann Sven Hager Anders Lehmann - Remi Meier Aurelien Campeas Niklaus Haldimann Camillo Bruni @@ -72,8 +72,8 @@ copyrighted by one or more of the following people and organizations: Romain Guillebert Leonardo Santagada Seo Sanghyeon - Justin Peel Ronny Pfannschmidt + Justin Peel David Edelsohn Anders Hammarquist Jakub Gustak @@ -95,6 +95,7 @@ copyrighted by one or more of the following people and organizations: Tyler Wade Michael Foord Stephan Diehl + Vincent Legoll Stefan Schwarzer Valentino Volonghi Tomek Meka @@ -105,9 +106,9 @@ copyrighted by one or more of the following people and organizations: Jean-Paul Calderone Timo Paulssen Squeaky + Marius Gedminas Alexandre Fayolle Simon Burton - Marius Gedminas Martin Matusiak Konstantin Lopuhin Wenzhu Man @@ -116,16 +117,20 @@ copyrighted by one or more of the following people and organizations: Ivan Sichmann Freitas Greg Price Dario Bertini + Stefano Rivera Mark Pearse Simon Cross Andreas Stührk - Stefano Rivera + Edd Barrett Jean-Philippe St. Pierre Guido van Rossum Pavel Vinogradov + Jeremy Thurgood Paweł Piotr Przeradowski + Spenser Bauman Paul deGrandis Ilya Osadchiy + marky1991 Tobias Oberstein Adrian Kuhn Boris Feigin @@ -134,14 +139,12 @@ copyrighted by one or more of the following people and organizations: Georg Brandl Bert Freudenberg Stian Andreassen - Edd Barrett + Tobias Pape Wanja Saatkamp Gerald Klix Mike Blume - Tobias Pape Oscar Nierstrasz Stefan H. Muller - Jeremy Thurgood Rami Chowdhury Eugene Oden Henry Mason @@ -153,6 +156,8 @@ copyrighted by one or more of the following people and organizations: Lukas Renggli Guenter Jantzen Ned Batchelder + Tim Felgentreff + Anton Gulenko Amit Regmi Ben Young Nicolas Chauvat @@ -162,12 +167,12 @@ copyrighted by one or more of the following people and organizations: Nicholas Riley Jason Chu Igor Trindade Oliveira - Tim Felgentreff + Yichao Yu Rocco Moretti Gintautas Miliauskas Michael Twomey Lucian Branescu Mihaila - Yichao Yu + Devin Jeanpierre Gabriel Lavoie Olivier Dormond Jared Grubb @@ -191,33 +196,33 @@ copyrighted by one or more of the following people and organizations: Stanislaw Halik Mikael Schönenberg Berkin Ilbeyi - Elmo M?ntynen + Elmo Mäntynen + Faye Zhao Jonathan David Riehl Anders Qvist Corbin Simpson Chirag Jadwani Beatrice During Alex Perry - Vincent Legoll + Vaibhav Sood Alan McIntyre - Spenser Bauman + William Leslie Alexander Sedov Attila Gobi + Jasper.Schulz Christopher Pope - Devin Jeanpierre - Vaibhav Sood Christian Tismer Marc Abramowitz Dan Stromberg Arjun Naik Valentina Mukhamedzhanova Stefano Parmesan + Mark Young Alexis Daboville Jens-Uwe Mager Carl Meyer Karl Ramm Pieter Zieschang - Anton Gulenko Gabriel Lukas Vacek Andrew Dalke @@ -225,6 +230,7 @@ copyrighted by one or more of the following people and organizations: Jakub Stasiak Nathan Taylor Vladimir Kryachko + Omer Katz Jacek Generowicz Alejandro J. Cura Jacob Oscarson @@ -239,6 +245,7 @@ copyrighted by one or more of the following people and organizations: Lars Wassermann Philipp Rustemeuer Henrik Vendelbo + Richard Lancaster Dan Buch Miguel de Val Borro Artur Lisiecki @@ -250,18 +257,18 @@ copyrighted by one or more of the following people and organizations: Tomo Cocoa Kim Jin Su Toni Mattis + Amber Brown Lucas Stadler Julian Berman Markus Holtermann roberto@goyle Yury V. Zaytsev Anna Katrina Dominguez - William Leslie Bobby Impollonia - Faye Zhao timo@eistee.fritz.box Andrew Thompson Yusei Tahara + Aaron Tubbs Ben Darnell Roberto De Ioris Juan Francisco Cantero Hurtado @@ -273,6 +280,7 @@ copyrighted by one or more of the following people and organizations: Christopher Armstrong Michael Hudson-Doyle Anders Sigfridsson + Nikolay Zinov Yasir Suhail Jason Michalski rafalgalczynski@gmail.com @@ -282,6 +290,7 @@ copyrighted by one or more of the following people and organizations: Gustavo Niemeyer Stephan Busemann Rafał Gałczyński + Matt Bogosian Christian Muirhead Berker Peksag James Lan @@ -316,9 +325,9 @@ copyrighted by one or more of the following people and organizations: Stefan Marr jiaaro Mads Kiilerich - Richard Lancaster opassembler.py Antony Lee + Jason Madden Yaroslav Fedevych Jim Hunziker Markus Unterwaditzer @@ -327,6 +336,7 @@ copyrighted by one or more of the following people and organizations: squeaky Zearin soareschen + Jonas Pfannschmidt Kurt Griffiths Mike Bayer Matthew Miller diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py index be237e8d72..1e1a0291f6 100644 --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -7,6 +7,7 @@ def get_hashed_dir(cfile): content = fid.read() # from cffi's Verifier() key = '\x00'.join([sys.version[:3], content]) + key += 'cpyext-gc-support-2' # this branch requires recompilation! if sys.version_info >= (3,): key = key.encode('utf-8') k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) @@ -62,7 +63,7 @@ def compile_shared(csource, modulename, output_dir=None): if sys.platform == 'win32': # XXX pyconfig.h uses a pragma to link to the import library, # which is currently python27.lib - library = os.path.join(thisdir, '..', 'include', 'python27') + library = os.path.join(thisdir, '..', 'libs', 'python27') if not os.path.exists(library + '.lib'): # For a local translation or nightly build library = os.path.join(thisdir, '..', 'pypy', 'goal', 'python27') diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py index 70ef93a22b..289bac34a9 100644 --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -550,6 +550,7 @@ class FFI(object): lst.append(value) # if '__pypy__' in sys.builtin_module_names: + import os if sys.platform == "win32": # we need 'libpypy-c.lib'. Current distributions of # pypy (>= 4.1) contain it as 'libs/python27.lib'. @@ -558,11 +559,15 @@ class FFI(object): ensure('library_dirs', os.path.join(sys.prefix, 'libs')) else: # we need 'libpypy-c.{so,dylib}', which should be by - # default located in 'sys.prefix/bin' + # default located in 'sys.prefix/bin' for installed + # systems. pythonlib = "pypy-c" if hasattr(sys, 'prefix'): - import os ensure('library_dirs', os.path.join(sys.prefix, 'bin')) + # On uninstalled pypy's, the libpypy-c is typically found in + # .../pypy/goal/. + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'pypy', 'goal')) else: if sys.platform == "win32": template = "python%d%d" diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py index cfefa00e7d..7f3f29c6da 100644 --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -36,11 +36,16 @@ working_modules.update([ "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend", - "_csv", "cppyy", "_pypyjson", "_vmprof", + "_csv", "cppyy", "_pypyjson", ]) -if os.uname()[4] == 's390x': - working_modules.remove("_vmprof") +from rpython.jit.backend import detect_cpu +try: + if detect_cpu.autodetect().startswith('x86'): + working_modules.add('_vmprof') +except detect_cpu.ProcessorAutodetectError: + pass + translation_modules = default_modules.copy() translation_modules.update([ @@ -165,12 +170,8 @@ pypy_optiondescription = OptionDescription("objspace", "Object Space Options", [ cmdline="--translationmodules", suggests=[("objspace.allworkingmodules", False)]), - BoolOption("usepycfiles", "Write and read pyc files when importing", - default=True), - BoolOption("lonepycfiles", "Import pyc files with no matching py file", - default=False, - requires=[("objspace.usepycfiles", True)]), + default=False), StrOption("soabi", "Tag to differentiate extension modules built for different Python interpreters", diff --git a/pypy/doc/discussion/rawrefcount.rst b/pypy/doc/discussion/rawrefcount.rst new file mode 100644 index 0000000000..d0a59aaaae --- /dev/null +++ b/pypy/doc/discussion/rawrefcount.rst @@ -0,0 +1,158 @@ +====================== +Rawrefcount and the GC +====================== + + +GC Interface +------------ + +"PyObject" is a raw structure with at least two fields, ob_refcnt and +ob_pypy_link. The ob_refcnt is the reference counter as used on +CPython. If the PyObject structure is linked to a live PyPy object, +its current address is stored in ob_pypy_link and ob_refcnt is bumped +by either the constant REFCNT_FROM_PYPY, or the constant +REFCNT_FROM_PYPY_LIGHT (== REFCNT_FROM_PYPY + SOME_HUGE_VALUE) +(to mean "light finalizer"). + +Most PyPy objects exist outside cpyext, and conversely in cpyext it is +possible that a lot of PyObjects exist without being seen by the rest +of PyPy. At the interface, however, we can "link" a PyPy object and a +PyObject. There are two kinds of link: + +rawrefcount.create_link_pypy(p, ob) + + Makes a link between an exising object gcref 'p' and a newly + allocated PyObject structure 'ob'. ob->ob_refcnt must be + initialized to either REFCNT_FROM_PYPY, or + REFCNT_FROM_PYPY_LIGHT. (The second case is an optimization: + when the GC finds the PyPy object and PyObject no longer + referenced, it can just free() the PyObject.) + +rawrefcount.create_link_pyobj(p, ob) + + Makes a link from an existing PyObject structure 'ob' to a newly + allocated W_CPyExtPlaceHolderObject 'p'. You must also add + REFCNT_FROM_PYPY to ob->ob_refcnt. For cases where the PyObject + contains all the data, and the PyPy object is just a proxy. The + W_CPyExtPlaceHolderObject should have only a field that contains + the address of the PyObject, but that's outside the scope of the + GC. + +rawrefcount.from_obj(p) + + If there is a link from object 'p' made with create_link_pypy(), + returns the corresponding 'ob'. Otherwise, returns NULL. + +rawrefcount.to_obj(Class, ob) + + Returns ob->ob_pypy_link, cast to an instance of 'Class'. + + +Collection logic +---------------- + +Objects existing purely on the C side have ob->ob_pypy_link == 0; +these are purely reference counted. On the other hand, if +ob->ob_pypy_link != 0, then ob->ob_refcnt is at least REFCNT_FROM_PYPY +and the object is part of a "link". + +The idea is that links whose 'p' is not reachable from other PyPy +objects *and* whose 'ob->ob_refcnt' is REFCNT_FROM_PYPY or +REFCNT_FROM_PYPY_LIGHT are the ones who die. But it is more messy +because PyObjects still (usually) need to have a tp_dealloc called, +and this cannot occur immediately (and can do random things like +accessing other references this object points to, or resurrecting the +object). + +Let P = list of links created with rawrefcount.create_link_pypy() +and O = list of links created with rawrefcount.create_link_pyobj(). +The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all +the data is in the PyObjects, and all outsite references (if any) are +in C, as "PyObject *" fields. + +So, during the collection we do this about P links: + + for (p, ob) in P: + if ob->ob_refcnt != REFCNT_FROM_PYPY + and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT: + mark 'p' as surviving, as well as all its dependencies + +At the end of the collection, the P and O links are both handled like +this: + + for (p, ob) in P + O: + if p is not surviving: # even if 'ob' might be surviving + unlink p and ob + if ob->ob_refcnt == REFCNT_FROM_PYPY_LIGHT: + free(ob) + elif ob->ob_refcnt > REFCNT_FROM_PYPY_LIGHT: + ob->ob_refcnt -= REFCNT_FROM_PYPY_LIGHT + else: + ob->ob_refcnt -= REFCNT_FROM_PYPY + if ob->ob_refcnt == 0: + invoke _Py_Dealloc(ob) later, outside the GC + + +GC Implementation +----------------- + +We need two copies of both the P list and O list, for young or old +objects. All four lists can be regular AddressLists of 'ob' objects. + +We also need an AddressDict mapping 'p' to 'ob' for all links in the P +list, and update it when PyPy objects move. + + +Further notes +------------- + +XXX +XXX the rest is the ideal world, but as a first step, we'll look +XXX for the minimal tweaks needed to adapt the existing cpyext +XXX + +For objects that are opaque in CPython, like <dict>, we always create +a PyPy object, and then when needed we make an empty PyObject and +attach it with create_link_pypy()/REFCNT_FROM_PYPY_LIGHT. + +For <int> and <float> objects, the corresponding PyObjects contain a +"long" or "double" field too. We link them with create_link_pypy() +and we can use REFCNT_FROM_PYPY_LIGHT too: 'tp_dealloc' doesn't +need to be called, and instead just calling free() is fine. + +For <type> objects, we need both a PyPy and a PyObject side. These +are made with create_link_pypy()/REFCNT_FROM_PYPY. + +For custom PyXxxObjects allocated from the C extension module, we +need create_link_pyobj(). + +For <str> or <unicode> objects coming from PyPy, we use +create_link_pypy()/REFCNT_FROM_PYPY_LIGHT with a PyObject +preallocated with the size of the string. We copy the string +lazily into that area if PyString_AS_STRING() is called. + +For <str>, <unicode>, <tuple> or <list> objects in the C extension +module, we first allocate it as only a PyObject, which supports +mutation of the data from C, like CPython. When it is exported to +PyPy we could make a W_CPyExtPlaceHolderObject with +create_link_pyobj(). + +For <tuple> objects coming from PyPy, if they are not specialized, +then the PyPy side holds a regular reference to the items. Then we +can allocate a PyTupleObject and store in it borrowed PyObject +pointers to the items. Such a case is created with +create_link_pypy()/REFCNT_FROM_PYPY_LIGHT. If it is specialized, +then it doesn't work because the items are created just-in-time on the +PyPy side. In this case, the PyTupleObject needs to hold real +references to the PyObject items, and we use create_link_pypy()/ +REFCNT_FROM_PYPY. In all cases, we have a C array of PyObjects +that we can directly return from PySequence_Fast_ITEMS, PyTuple_ITEMS, +PyTuple_GetItem, and so on. + +For <list> objects coming from PyPy, we can use a cpyext list +strategy. The list turns into a PyListObject, as if it had been +allocated from C in the first place. The special strategy can hold +(only) a direct reference to the PyListObject, and we can use either +create_link_pyobj() or create_link_pypy() (to be decided). +PySequence_Fast_ITEMS then works for lists too, and PyList_GetItem +can return a borrowed reference, and so on. diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst index 13ff02110a..8b8f70d17a 100644 --- a/pypy/doc/how-to-release.rst +++ b/pypy/doc/how-to-release.rst @@ -1,5 +1,20 @@ -Making a PyPy Release -===================== +The PyPy Release Process +======================== + +Release Policy +++++++++++++++ + +We try to create a stable release a few times a year. These are released on +a branch named like release-2.x or release-4.x, and each release is tagged, +for instance release-4.0.1. + +After release, inevitably there are bug fixes. It is the responsibility of +the commiter who fixes a bug to make sure this fix is on the release branch, +so that we can then create a tagged bug-fix release, which will hopefully +happen more often than stable releases. + +How to Create a PyPy Release +++++++++++++++++++++++++++++ Overview -------- diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py index b0b252bea8..28d351fe96 100644 --- a/pypy/doc/tool/makecontributor.py +++ b/pypy/doc/tool/makecontributor.py @@ -72,6 +72,7 @@ alias = { 'Anton Gulenko':['anton gulenko', 'anton_gulenko'], 'Richard Lancaster':['richardlancaster'], 'William Leslie':['William ML Leslie'], + 'Spenser Bauman':['Spenser Andrew Bauman'], } alias_map = {} diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst index 5e25d84e2c..ff6cba1e44 100644 --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -153,6 +153,37 @@ Refactor vmprof to work cross-operating-system. Seperate structmember.h from Python.h Also enhance creating api functions to specify which header file they appear in (previously only pypy_decl.h) +.. branch: llimpl + +Refactor register_external(), remove running_on_llinterp mechanism and +apply sandbox transform on externals at the end of annotation. + +.. branch: cffi-embedding-win32 + +.. branch: windows-vmprof-support + +vmprof should work on Windows. + + +.. branch: reorder-map-attributes + +When creating instances and adding attributes in several different orders +depending on some condition, the JIT would create too much code. This is now +fixed. + +.. branch: cpyext-gc-support-2 + +Improve CPython C API support, which means lxml now runs unmodified +(after removing pypy hacks, pending pull request) + +.. branch: look-inside-tuple-hash + +Look inside tuple hash, improving mdp benchmark + +.. branch: vlen-resume + +Compress resume data, saving 10-20% of memory consumed by the JIT + .. branch: memop-simplify3 Further simplifying the backend operations malloc_cond_varsize and zero_array. diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py index 0313d5bcce..8cfaaf1b9c 100644 --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -277,7 +277,6 @@ class PyPyTarget(object): if config.translation.sandbox: config.objspace.lonepycfiles = False - config.objspace.usepycfiles = False config.translating = True diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py index da52441d7c..d08bbee9e6 100644 --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -27,7 +27,7 @@ unpackiterable_driver = jit.JitDriver(name='unpackiterable', class W_Root(object): """This is the abstract root class of all wrapped objects that live in a 'normal' object space like StdObjSpace.""" - __slots__ = () + __slots__ = ('__weakref__',) user_overridden_class = False def getdict(self, space): diff --git a/pypy/interpreter/pyparser/pytokenizer.py b/pypy/interpreter/pyparser/pytokenizer.py index b4fe0ee64b..c707027a00 100644 --- a/pypy/interpreter/pyparser/pytokenizer.py +++ b/pypy/interpreter/pyparser/pytokenizer.py @@ -91,6 +91,7 @@ def generate_tokens(lines, flags): strstart = (0, 0, "") for line in lines: lnum = lnum + 1 + line = universal_newline(line) pos, max = 0, len(line) if contstr: @@ -259,3 +260,14 @@ def generate_tokens(lines, flags): token_list.append((tokens.ENDMARKER, '', lnum, pos, line)) return token_list + + +def universal_newline(line): + # show annotator that indexes below are non-negative + line_len_m2 = len(line) - 2 + if line_len_m2 >= 0 and line[-2] == '\r' and line[-1] == '\n': + return line[:line_len_m2] + '\n' + line_len_m1 = len(line) - 1 + if line_len_m1 >= 0 and line[-1] == '\r': + return line[:line_len_m1] + '\n' + return line diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py b/pypy/interpreter/pyparser/test/test_pyparse.py index 3dde0268ad..8f46d828a0 100644 --- a/pypy/interpreter/pyparser/test/test_pyparse.py +++ b/pypy/interpreter/pyparser/test/test_pyparse.py @@ -158,3 +158,10 @@ pass""" def test_print_function(self): self.parse("from __future__ import print_function\nx = print\n") + + def test_universal_newlines(self): + fmt = 'stuff = """hello%sworld"""' + expected_tree = self.parse(fmt % '\n') + for linefeed in ["\r\n","\r"]: + tree = self.parse(fmt % linefeed) + assert expected_tree == tree diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py index 959eb0d09b..b915d522ff 100644 --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -156,20 +156,6 @@ def get_unique_interplevel_subclass(config, cls, hasdict, wants_slots, get_unique_interplevel_subclass._annspecialcase_ = "specialize:memo" _subclass_cache = {} -def enum_interplevel_subclasses(config, cls): - """Return a list of all the extra interp-level subclasses of 'cls' that - can be built by get_unique_interplevel_subclass().""" - result = [] - for flag1 in (False, True): - for flag2 in (False, True): - for flag3 in (False, True): - for flag4 in (False, True): - result.append(get_unique_interplevel_subclass( - config, cls, flag1, flag2, flag3, flag4)) - result = dict.fromkeys(result) - assert len(result) <= 6 - return result.keys() - def _getusercls(config, cls, wants_dict, wants_slots, wants_del, weakrefable): typedef = cls.typedef if wants_dict and typedef.hasdict: @@ -262,7 +248,7 @@ def _builduserclswithfeature(config, supercls, *features): def user_setup(self, space, w_subtype): self.space = space self.w__class__ = w_subtype - self.user_setup_slots(w_subtype.nslots) + self.user_setup_slots(w_subtype.layout.nslots) def user_setup_slots(self, nslots): assert nslots == 0 diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py index c3b752affa..c4e504df0c 100644 --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -57,7 +57,7 @@ def patch_sys(space): # pypy_init_embedded_cffi_module(). if not glob.patched_sys: space.appexec([], """(): - import os + import os, sys sys.stdin = sys.__stdin__ = os.fdopen(0, 'rb', 0) sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0) sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0) diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py index e460c43ba2..4f75265553 100644 --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -5,14 +5,15 @@ from pypy.tool.pytest.objspace import gettestobjspace class AppTestVMProf(object): def setup_class(cls): cls.space = gettestobjspace(usemodules=['_vmprof', 'struct']) - cls.tmpfile = udir.join('test__vmprof.1').open('wb') - cls.w_tmpfileno = cls.space.wrap(cls.tmpfile.fileno()) - cls.w_tmpfilename = cls.space.wrap(cls.tmpfile.name) - cls.tmpfile2 = udir.join('test__vmprof.2').open('wb') - cls.w_tmpfileno2 = cls.space.wrap(cls.tmpfile2.fileno()) - cls.w_tmpfilename2 = cls.space.wrap(cls.tmpfile2.name) + cls.w_tmpfilename = cls.space.wrap(str(udir.join('test__vmprof.1'))) + cls.w_tmpfilename2 = cls.space.wrap(str(udir.join('test__vmprof.2'))) def test_import_vmprof(self): + tmpfile = open(self.tmpfilename, 'wb') + tmpfileno = tmpfile.fileno() + tmpfile2 = open(self.tmpfilename2, 'wb') + tmpfileno2 = tmpfile2.fileno() + import struct, sys WORD = struct.calcsize('l') @@ -45,7 +46,7 @@ class AppTestVMProf(object): return count import _vmprof - _vmprof.enable(self.tmpfileno, 0.01) + _vmprof.enable(tmpfileno, 0.01) _vmprof.disable() s = open(self.tmpfilename, 'rb').read() no_of_codes = count(s) @@ -56,7 +57,7 @@ class AppTestVMProf(object): pass """ in d - _vmprof.enable(self.tmpfileno2, 0.01) + _vmprof.enable(tmpfileno2, 0.01) exec """def foo2(): pass diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py index ceb21269ef..3bf8085479 100644 --- a/pypy/module/cpyext/__init__.py +++ b/pypy/module/cpyext/__init__.py @@ -34,7 +34,7 @@ import pypy.module.cpyext.pythonrun import pypy.module.cpyext.pyerrors import pypy.module.cpyext.typeobject import pypy.module.cpyext.object -import pypy.module.cpyext.stringobject +import pypy.module.cpyext.bytesobject import pypy.module.cpyext.tupleobject import pypy.module.cpyext.setobject import pypy.module.cpyext.dictobject @@ -60,7 +60,6 @@ import pypy.module.cpyext.weakrefobject import pypy.module.cpyext.funcobject import pypy.module.cpyext.frameobject import pypy.module.cpyext.classobject -import pypy.module.cpyext.pypyintf import pypy.module.cpyext.memoryobject import pypy.module.cpyext.codecs import pypy.module.cpyext.pyfile diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py index d807774807..2a1517a97f 100644 --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -9,7 +9,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import ll2ctypes from rpython.rtyper.annlowlevel import llhelper -from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import we_are_translated, keepalive_until_here from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.gensupp import NameManager @@ -30,13 +30,13 @@ from rpython.rlib.entrypoint import entrypoint_lowlevel from rpython.rlib.rposix import is_valid_fd, validate_fd from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize -from rpython.rlib.exports import export_struct from pypy.module import exceptions from pypy.module.exceptions import interp_exceptions # CPython 2.4 compatibility from py.builtin import BaseException from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rlib import rawrefcount DEBUG_WRAPPER = True @@ -194,7 +194,7 @@ cpyext_namespace = NameManager('cpyext_') class ApiFunction: def __init__(self, argtypes, restype, callable, error=_NOT_SPECIFIED, - c_name=None, gil=None): + c_name=None, gil=None, result_borrowed=False): self.argtypes = argtypes self.restype = restype self.functype = lltype.Ptr(lltype.FuncType(argtypes, restype)) @@ -211,17 +211,15 @@ class ApiFunction: self.argnames = argnames[1:] assert len(self.argnames) == len(self.argtypes) self.gil = gil + self.result_borrowed = result_borrowed + # + def get_llhelper(space): + return llhelper(self.functype, self.get_wrapper(space)) + self.get_llhelper = get_llhelper def _freeze_(self): return True - def get_llhelper(self, space): - llh = getattr(self, '_llhelper', None) - if llh is None: - llh = llhelper(self.functype, self.get_wrapper(space)) - self._llhelper = llh - return llh - @specialize.memo() def get_wrapper(self, space): wrapper = getattr(self, '_wrapper', None) @@ -234,7 +232,7 @@ class ApiFunction: return wrapper def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', - gil=None): + gil=None, result_borrowed=False): """ Declares a function to be exported. - `argtypes`, `restype` are lltypes and describe the function signature. @@ -263,13 +261,15 @@ def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', rffi.cast(restype, 0) == 0) def decorate(func): + func._always_inline_ = 'try' func_name = func.func_name if header is not None: c_name = None else: c_name = func_name api_function = ApiFunction(argtypes, restype, func, error, - c_name=c_name, gil=gil) + c_name=c_name, gil=gil, + result_borrowed=result_borrowed) func.api_func = api_function if header is not None: @@ -280,6 +280,10 @@ def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', raise ValueError("function %s has no return value for exceptions" % func) def make_unwrapper(catch_exception): + # ZZZ is this whole logic really needed??? It seems to be only + # for RPython code calling PyXxx() functions directly. I would + # think that usually directly calling the function is clean + # enough now names = api_function.argnames types_names_enum_ui = unrolling_iterable(enumerate( zip(api_function.argtypes, @@ -287,56 +291,58 @@ def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', @specialize.ll() def unwrapper(space, *args): - from pypy.module.cpyext.pyobject import Py_DecRef - from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import Reference + from pypy.module.cpyext.pyobject import Py_DecRef, is_pyobj + from pypy.module.cpyext.pyobject import from_ref, as_pyobj newargs = () - to_decref = [] + keepalives = () assert len(args) == len(api_function.argtypes) for i, (ARG, is_wrapped) in types_names_enum_ui: input_arg = args[i] if is_PyObject(ARG) and not is_wrapped: - # build a reference - if input_arg is None: - arg = lltype.nullptr(PyObject.TO) - elif isinstance(input_arg, W_Root): - ref = make_ref(space, input_arg) - to_decref.append(ref) - arg = rffi.cast(ARG, ref) + # build a 'PyObject *' (not holding a reference) + if not is_pyobj(input_arg): + keepalives += (input_arg,) + arg = rffi.cast(ARG, as_pyobj(space, input_arg)) else: - arg = input_arg + arg = rffi.cast(ARG, input_arg) elif is_PyObject(ARG) and is_wrapped: - # convert to a wrapped object - if input_arg is None: - arg = input_arg - elif isinstance(input_arg, W_Root): - arg = input_arg + # build a W_Root, possibly from a 'PyObject *' + if is_pyobj(input_arg): + arg = from_ref(space, input_arg) else: - try: - arg = from_ref(space, - rffi.cast(PyObject, input_arg)) - except TypeError, e: - err = OperationError(space.w_TypeError, - space.wrap( - "could not cast arg to PyObject")) - if not catch_exception: - raise err - state = space.fromcache(State) - state.set_exception(err) - if is_PyObject(restype): - return None - else: - return api_function.error_value + arg = input_arg + + ## ZZZ: for is_pyobj: + ## try: + ## arg = from_ref(space, + ## rffi.cast(PyObject, input_arg)) + ## except TypeError, e: + ## err = OperationError(space.w_TypeError, + ## space.wrap( + ## "could not cast arg to PyObject")) + ## if not catch_exception: + ## raise err + ## state = space.fromcache(State) + ## state.set_exception(err) + ## if is_PyObject(restype): + ## return None + ## else: + ## return api_function.error_value else: - # convert to a wrapped object + # arg is not declared as PyObject, no magic arg = input_arg newargs += (arg, ) - try: + if not catch_exception: + try: + res = func(space, *newargs) + finally: + keepalive_until_here(*keepalives) + else: + # non-rpython variant + assert not we_are_translated() try: res = func(space, *newargs) except OperationError, e: - if not catch_exception: - raise if not hasattr(api_function, "error_value"): raise state = space.fromcache(State) @@ -345,21 +351,13 @@ def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', return None else: return api_function.error_value - if not we_are_translated(): - got_integer = isinstance(res, (int, long, float)) - assert got_integer == expect_integer,'got %r not integer' % res - if res is None: - return None - elif isinstance(res, Reference): - return res.get_wrapped(space) - else: - return res - finally: - for arg in to_decref: - Py_DecRef(space, arg) + # 'keepalives' is alive here (it's not rpython) + got_integer = isinstance(res, (int, long, float)) + assert got_integer == expect_integer, ( + 'got %r not integer' % (res,)) + return res unwrapper.func = func unwrapper.api_func = api_function - unwrapper._always_inline_ = 'try' return unwrapper unwrapper_catch = make_unwrapper(True) @@ -501,7 +499,7 @@ def build_exported_objects(): GLOBALS['%s#' % (cpyname, )] = ('PyTypeObject*', pypyexpr) for cpyname in '''PyMethodObject PyListObject PyLongObject - PyDictObject PyTupleObject PyClassObject'''.split(): + PyDictObject PyClassObject'''.split(): FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s' % (cpyname, )) build_exported_objects() @@ -514,14 +512,16 @@ def get_structtype_for_ctype(ctype): "PyIntObject*": PyIntObject, "PyDateTime_CAPI*": lltype.Ptr(PyDateTime_CAPI)}[ctype] +# Note: as a special case, "PyObject" is the pointer type in RPython, +# corresponding to "PyObject *" in C. We do that only for PyObject. +# For example, "PyTypeObject" is the struct type even in RPython. PyTypeObject = lltype.ForwardReference() PyTypeObjectPtr = lltype.Ptr(PyTypeObject) -# It is important that these PyObjects are allocated in a raw fashion -# Thus we cannot save a forward pointer to the wrapped object -# So we need a forward and backward mapping in our State instance PyObjectStruct = lltype.ForwardReference() PyObject = lltype.Ptr(PyObjectStruct) -PyObjectFields = (("ob_refcnt", lltype.Signed), ("ob_type", PyTypeObjectPtr)) +PyObjectFields = (("ob_refcnt", lltype.Signed), + ("ob_pypy_link", lltype.Signed), + ("ob_type", PyTypeObjectPtr)) PyVarObjectFields = PyObjectFields + (("ob_size", Py_ssize_t), ) cpython_struct('PyObject', PyObjectFields, PyObjectStruct) PyVarObjectStruct = cpython_struct("PyVarObject", PyVarObjectFields) @@ -618,8 +618,8 @@ def make_wrapper(space, callable, gil=None): @specialize.ll() def wrapper(*args): - from pypy.module.cpyext.pyobject import make_ref, from_ref - from pypy.module.cpyext.pyobject import Reference + from pypy.module.cpyext.pyobject import make_ref, from_ref, is_pyobj + from pypy.module.cpyext.pyobject import as_pyobj # we hope that malloc removal removes the newtuple() that is # inserted exactly here by the varargs specializer if gil_acquire: @@ -628,6 +628,7 @@ def make_wrapper(space, callable, gil=None): llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py retval = fatal_value boxed_args = () + tb = None try: if not we_are_translated() and DEBUG_WRAPPER: print >>sys.stderr, callable, @@ -635,10 +636,8 @@ def make_wrapper(space, callable, gil=None): for i, (typ, is_wrapped) in argtypes_enum_ui: arg = args[i] if is_PyObject(typ) and is_wrapped: - if arg: - arg_conv = from_ref(space, rffi.cast(PyObject, arg)) - else: - arg_conv = None + assert is_pyobj(arg) + arg_conv = from_ref(space, rffi.cast(PyObject, arg)) else: arg_conv = arg boxed_args += (arg_conv, ) @@ -653,6 +652,7 @@ def make_wrapper(space, callable, gil=None): except BaseException, e: failed = True if not we_are_translated(): + tb = sys.exc_info()[2] message = repr(e) import traceback traceback.print_exc() @@ -671,29 +671,34 @@ def make_wrapper(space, callable, gil=None): retval = error_value elif is_PyObject(callable.api_func.restype): - if result is None: - retval = rffi.cast(callable.api_func.restype, - make_ref(space, None)) - elif isinstance(result, Reference): - retval = result.get_ref(space) - elif not rffi._isllptr(result): - retval = rffi.cast(callable.api_func.restype, - make_ref(space, result)) - else: + if is_pyobj(result): retval = result + else: + if result is not None: + if callable.api_func.result_borrowed: + retval = as_pyobj(space, result) + else: + retval = make_ref(space, result) + retval = rffi.cast(callable.api_func.restype, retval) + else: + retval = lltype.nullptr(PyObject.TO) elif callable.api_func.restype is not lltype.Void: retval = rffi.cast(callable.api_func.restype, result) except Exception, e: print 'Fatal error in cpyext, CPython compatibility layer, calling', callable.__name__ print 'Either report a bug or consider not using this particular extension' if not we_are_translated(): + if tb is None: + tb = sys.exc_info()[2] import traceback traceback.print_exc() - print str(e) + if sys.stdout == sys.__stdout__: + import pdb; pdb.post_mortem(tb) # we can't do much here, since we're in ctypes, swallow else: print str(e) pypy_debug_catch_fatal_exception() + assert False rffi.stackcounter.stacks_counter -= 1 if gil_release: rgil.release() @@ -827,6 +832,19 @@ def build_bridge(space): outputfilename=str(udir / "module_cache" / "pypyapi")) modulename = py.path.local(eci.libraries[-1]) + def dealloc_trigger(): + from pypy.module.cpyext.pyobject import _Py_Dealloc + print 'dealloc_trigger...' + while True: + ob = rawrefcount.next_dead(PyObject) + if not ob: + break + print ob + _Py_Dealloc(space, ob) + print 'dealloc_trigger DONE' + return "RETRY" + rawrefcount.init(dealloc_trigger) + run_bootstrap_functions(space) # load the bridge, and init structure @@ -836,9 +854,9 @@ def build_bridge(space): space.fromcache(State).install_dll(eci) # populate static data - builder = StaticObjectBuilder(space) + builder = space.fromcache(StaticObjectBuilder) for name, (typ, expr) in GLOBALS.iteritems(): - from pypy.module import cpyext + from pypy.module import cpyext # for the eval() below w_obj = eval(expr) if name.endswith('#'): name = name[:-1] @@ -894,27 +912,44 @@ def build_bridge(space): class StaticObjectBuilder: def __init__(self, space): self.space = space - self.to_attach = [] + self.static_pyobjs = [] + self.static_objs_w = [] + self.cpyext_type_init = None + # + # add a "method" that is overridden in setup_library() + # ('self.static_pyobjs' is completely ignored in that case) + self.get_static_pyobjs = lambda: self.static_pyobjs def prepare(self, py_obj, w_obj): - from pypy.module.cpyext.pyobject import track_reference - py_obj.c_ob_refcnt = 1 - track_reference(self.space, py_obj, w_obj) - self.to_attach.append((py_obj, w_obj)) + "NOT_RPYTHON" + if py_obj: + py_obj.c_ob_refcnt = 1 # 1 for kept immortal + self.static_pyobjs.append(py_obj) + self.static_objs_w.append(w_obj) def attach_all(self): + # this is RPython, called once in pypy-c when it imports cpyext from pypy.module.cpyext.pyobject import get_typedescr, make_ref from pypy.module.cpyext.typeobject import finish_type_1, finish_type_2 + from pypy.module.cpyext.pyobject import track_reference + # space = self.space - space._cpyext_type_init = [] - for py_obj, w_obj in self.to_attach: + static_pyobjs = self.get_static_pyobjs() + static_objs_w = self.static_objs_w + for i in range(len(static_objs_w)): + track_reference(space, static_pyobjs[i], static_objs_w[i]) + # + self.cpyext_type_init = [] + for i in range(len(static_objs_w)): + py_obj = static_pyobjs[i] + w_obj = static_objs_w[i] w_type = space.type(w_obj) - typedescr = get_typedescr(w_type.instancetypedef) + typedescr = get_typedescr(w_type.layout.typedef) py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) typedescr.attach(space, py_obj, w_obj) - cpyext_type_init = space._cpyext_type_init - del space._cpyext_type_init + cpyext_type_init = self.cpyext_type_init + self.cpyext_type_init = None for pto, w_type in cpyext_type_init: finish_type_1(space, pto) finish_type_2(space, pto, w_type) @@ -1067,7 +1102,7 @@ def build_eci(building_bridge, export_symbols, code): if name.endswith('#'): structs.append('%s %s;' % (typ[:-1], name[:-1])) elif name.startswith('PyExc_'): - structs.append('extern PyTypeObject _%s;' % (name,)) + structs.append('PyTypeObject _%s;' % (name,)) structs.append('PyObject* %s = (PyObject*)&_%s;' % (name, name)) elif typ == 'PyDateTime_CAPI*': structs.append('%s %s = NULL;' % (typ, name)) @@ -1107,7 +1142,7 @@ def setup_micronumpy(space): if not use_micronumpy: return use_micronumpy # import to register api functions by side-effect - import pypy.module.cpyext.ndarrayobject + import pypy.module.cpyext.ndarrayobject global GLOBALS, SYMBOLS_C, separate_module_files GLOBALS["PyArray_Type#"]= ('PyTypeObject*', "space.gettypeobject(W_NDimArray.typedef)") SYMBOLS_C += ['PyArray_Type', '_PyArray_FILLWBYTE', '_PyArray_ZEROS'] @@ -1116,10 +1151,8 @@ def setup_micronumpy(space): def setup_library(space): "NOT_RPYTHON" - from pypy.module.cpyext.pyobject import make_ref use_micronumpy = setup_micronumpy(space) - - export_symbols = list(FUNCTIONS) + SYMBOLS_C + list(GLOBALS) + export_symbols = sorted(FUNCTIONS) + sorted(SYMBOLS_C) + sorted(GLOBALS) from rpython.translator.c.database import LowLevelDatabase db = LowLevelDatabase() @@ -1135,41 +1168,37 @@ def setup_library(space): run_bootstrap_functions(space) setup_va_functions(eci) - from pypy.module import cpyext # for eval() below - - # Set up the types. Needs a special case, because of the - # immediate cycle involving 'c_ob_type', and because we don't - # want these types to be Py_TPFLAGS_HEAPTYPE. - static_types = {} - for name, (typ, expr) in GLOBALS.items(): - if typ == 'PyTypeObject*': - pto = lltype.malloc(PyTypeObject, immortal=True, - zero=True, flavor='raw') - pto.c_ob_refcnt = 1 - pto.c_tp_basicsize = -1 - static_types[name] = pto - builder = StaticObjectBuilder(space) - for name, pto in static_types.items(): - pto.c_ob_type = static_types['PyType_Type#'] - w_type = eval(GLOBALS[name][1]) - builder.prepare(rffi.cast(PyObject, pto), w_type) - builder.attach_all() - - # populate static data - for name, (typ, expr) in GLOBALS.iteritems(): - name = name.replace("#", "") - if name.startswith('PyExc_'): + # emit uninitialized static data + builder = space.fromcache(StaticObjectBuilder) + lines = ['PyObject *pypy_static_pyobjs[] = {\n'] + include_lines = ['RPY_EXTERN PyObject *pypy_static_pyobjs[];\n'] + for name, (typ, expr) in sorted(GLOBALS.items()): + if name.endswith('#'): + assert typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*') + typ, name = typ[:-1], name[:-1] + elif name.startswith('PyExc_'): + typ = 'PyTypeObject' name = '_' + name - w_obj = eval(expr) - if typ in ('PyObject*', 'PyTypeObject*', 'PyIntObject*'): - struct_ptr = make_ref(space, w_obj) elif typ == 'PyDateTime_CAPI*': continue else: assert False, "Unknown static data: %s %s" % (typ, name) - struct = rffi.cast(get_structtype_for_ctype(typ), struct_ptr)._obj - struct._compilation_info = eci - export_struct(name, struct) + + from pypy.module import cpyext # for the eval() below + w_obj = eval(expr) + builder.prepare(None, w_obj) + lines.append('\t(PyObject *)&%s,\n' % (name,)) + include_lines.append('RPY_EXPORTED %s %s;\n' % (typ, name)) + + lines.append('};\n') + eci2 = CConfig._compilation_info_.merge(ExternalCompilationInfo( + separate_module_sources = [''.join(lines)], + post_include_bits = [''.join(include_lines)], + )) + # override this method to return a pointer to this C array directly + builder.get_static_pyobjs = rffi.CExternVariable( + PyObjectP, 'pypy_static_pyobjs', eci2, c_type='PyObject **', + getter_only=True, declare_as_extern=False) for name, func in FUNCTIONS.iteritems(): newname = mangle_name('PyPy', name) or name @@ -1180,6 +1209,10 @@ def setup_library(space): trunk_include = pypydir.dirpath() / 'include' copy_header_files(trunk_include, use_micronumpy) +def init_static_data_translated(space): + builder = space.fromcache(StaticObjectBuilder) + builder.attach_all() + def _load_from_cffi(space, name, path, initptr): from pypy.module._cffi_backend import cffi1_module cffi1_module.load_cffi1_module(space, name, path, initptr) @@ -1262,22 +1295,18 @@ def load_cpyext_module(space, name, path, dll, initptr): @specialize.ll() def generic_cpy_call(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, True, False)(space, func, *args) - -@specialize.ll() -def generic_cpy_call_dont_decref(space, func, *args): - FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, False, False)(space, func, *args) + return make_generic_cpy_call(FT, False)(space, func, *args) @specialize.ll() def generic_cpy_call_expect_null(space, func, *args): FT = lltype.typeOf(func).TO - return make_generic_cpy_call(FT, True, True)(space, func, *args) + return make_generic_cpy_call(FT, True)(space, func, *args) @specialize.memo() -def make_generic_cpy_call(FT, decref_args, expect_null): +def make_generic_cpy_call(FT, expect_null): from pypy.module.cpyext.pyobject import make_ref, from_ref, Py_DecRef - from pypy.module.cpyext.pyobject import RefcountState + from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj + from pypy.module.cpyext.pyobject import get_w_obj_and_decref from pypy.module.cpyext.pyerrors import PyErr_Occurred unrolling_arg_types = unrolling_iterable(enumerate(FT.ARGS)) RESULT_TYPE = FT.RESULT @@ -1305,65 +1334,49 @@ def make_generic_cpy_call(FT, decref_args, expect_null): @specialize.ll() def generic_cpy_call(space, func, *args): boxed_args = () - to_decref = [] + keepalives = () assert len(args) == len(FT.ARGS) for i, ARG in unrolling_arg_types: arg = args[i] if is_PyObject(ARG): - if arg is None: - boxed_args += (lltype.nullptr(PyObject.TO),) - elif isinstance(arg, W_Root): - ref = make_ref(space, arg) - boxed_args += (ref,) - if decref_args: - to_decref.append(ref) - else: - boxed_args += (arg,) - else: - boxed_args += (arg,) + if not is_pyobj(arg): + keepalives += (arg,) + arg = as_pyobj(space, arg) + boxed_args += (arg,) try: - # create a new container for borrowed references - state = space.fromcache(RefcountState) - old_container = state.swap_borrow_container(None) - try: - # Call the function - result = call_external_function(func, *boxed_args) - finally: - state.swap_borrow_container(old_container) - - if is_PyObject(RESULT_TYPE): - if result is None: - ret = result - elif isinstance(result, W_Root): - ret = result - else: - ret = from_ref(space, result) - # The object reference returned from a C function - # that is called from Python must be an owned reference - # - ownership is transferred from the function to its caller. - if result: - Py_DecRef(space, result) - - # Check for exception consistency - has_error = PyErr_Occurred(space) is not None - has_result = ret is not None - if has_error and has_result: - raise OperationError(space.w_SystemError, space.wrap( - "An exception was set, but function returned a value")) - elif not expect_null and not has_error and not has_result: - raise OperationError(space.w_SystemError, space.wrap( - "Function returned a NULL result without setting an exception")) - - if has_error: - state = space.fromcache(State) - state.check_and_raise_exception() - - return ret - return result + # Call the function + result = call_external_function(func, *boxed_args) finally: - if decref_args: - for ref in to_decref: - Py_DecRef(space, ref) - return generic_cpy_call + keepalive_until_here(*keepalives) + if is_PyObject(RESULT_TYPE): + if not is_pyobj(result): + ret = result + else: + # The object reference returned from a C function + # that is called from Python must be an owned reference + # - ownership is transferred from the function to its caller. + if result: + ret = get_w_obj_and_decref(space, result) + else: + ret = None + + # Check for exception consistency + has_error = PyErr_Occurred(space) is not None + has_result = ret is not None + if has_error and has_result: + raise OperationError(space.w_SystemError, space.wrap( + "An exception was set, but function returned a value")) + elif not expect_null and not has_error and not has_result: + raise OperationError(space.w_SystemError, space.wrap( + "Function returned a NULL result without setting an exception")) + + if has_error: + state = space.fromcache(State) + state.check_and_raise_exception() + + return ret + return result + + return generic_cpy_call diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py index f60461bc44..11ea94cdf3 100644 --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -25,7 +25,7 @@ cpython_struct("PyBufferObject", PyBufferObjectFields, PyBufferObjectStruct) @bootstrap_function def init_bufferobject(space): "Type description of PyBufferObject" - make_typedescr(space.w_buffer.instancetypedef, + make_typedescr(space.w_buffer.layout.typedef, basestruct=PyBufferObject.TO, attach=buffer_attach, dealloc=buffer_dealloc, diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/bytesobject.py index 82e179c731..3a01515706 100644 --- a/pypy/module/cpyext/stringobject.py +++ b/pypy/module/cpyext/bytesobject.py @@ -59,7 +59,7 @@ cpython_struct("PyStringObject", PyStringObjectFields, PyStringObjectStruct) @bootstrap_function def init_stringobject(space): "Type description of PyStringObject" - make_typedescr(space.w_str.instancetypedef, + make_typedescr(space.w_str.layout.typedef, basestruct=PyStringObject.TO, attach=string_attach, dealloc=string_dealloc, @@ -69,11 +69,11 @@ PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_str") def new_empty_str(space, length): """ - Allocatse a PyStringObject and its buffer, but without a corresponding + Allocate a PyStringObject and its buffer, but without a corresponding interpreter object. The buffer may be mutated, until string_realize() is - called. + called. Refcount of the result is 1. """ - typedescr = get_typedescr(space.w_str.instancetypedef) + typedescr = get_typedescr(space.w_str.layout.typedef) py_obj = typedescr.allocate(space, space.w_str) py_str = rffi.cast(PyStringObject, py_obj) diff --git a/pypy/module/cpyext/complexobject.py b/pypy/module/cpyext/complexobject.py index a510f0da14..35c32670f7 100644 --- a/pypy/module/cpyext/complexobject.py +++ b/pypy/module/cpyext/complexobject.py @@ -43,7 +43,7 @@ def _PyComplex_FromCComplex(space, v): # lltype does not handle functions returning a structure. This implements a # helper function, which takes as argument a reference to the return value. -@cpython_api([PyObject, Py_complex_ptr], lltype.Void) +@cpython_api([PyObject, Py_complex_ptr], rffi.INT_real, error=-1) def _PyComplex_AsCComplex(space, w_obj, result): """Return the Py_complex value of the complex number op. @@ -60,7 +60,7 @@ def _PyComplex_AsCComplex(space, w_obj, result): # if the above did not work, interpret obj as a float giving the # real part of the result, and fill in the imaginary part as 0. result.c_real = PyFloat_AsDouble(space, w_obj) # -1 on failure - return + return 0 if not PyComplex_Check(space, w_obj): raise OperationError(space.w_TypeError, space.wrap( @@ -69,3 +69,4 @@ def _PyComplex_AsCComplex(space, w_obj, result): assert isinstance(w_obj, W_ComplexObject) result.c_real = w_obj.realval result.c_imag = w_obj.imagval + return 0 diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py index 0c133f05dc..761f1efb08 100644 --- a/pypy/module/cpyext/dictobject.py +++ b/pypy/module/cpyext/dictobject.py @@ -2,8 +2,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t, Py_ssize_tP, CONST_STRING) -from pypy.module.cpyext.pyobject import PyObject, PyObjectP, borrow_from -from pypy.module.cpyext.pyobject import RefcountState +from pypy.module.cpyext.pyobject import PyObject, PyObjectP, as_pyobj from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import specialize @@ -14,13 +13,17 @@ def PyDict_New(space): PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict") -@cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) +@cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL, + result_borrowed=True) def PyDict_GetItem(space, w_dict, w_key): try: w_res = space.getitem(w_dict, w_key) except: return None - return borrow_from(w_dict, w_res) + # NOTE: this works so far because all our dict strategies store + # *values* as full objects, which stay alive as long as the dict is + # alive and not modified. So we can return a borrowed ref. + return w_res @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1) def PyDict_SetItem(space, w_dict, w_key, w_obj): @@ -47,7 +50,8 @@ def PyDict_SetItemString(space, w_dict, key_ptr, w_obj): else: PyErr_BadInternalCall(space) -@cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL) +@cpython_api([PyObject, CONST_STRING], PyObject, error=CANNOT_FAIL, + result_borrowed=True) def PyDict_GetItemString(space, w_dict, key): """This is the same as PyDict_GetItem(), but key is specified as a char*, rather than a PyObject*.""" @@ -55,9 +59,10 @@ def PyDict_GetItemString(space, w_dict, key): w_res = space.finditem_str(w_dict, rffi.charp2str(key)) except: w_res = None - if w_res is None: - return None - return borrow_from(w_dict, w_res) + # NOTE: this works so far because all our dict strategies store + # *values* as full objects, which stay alive as long as the dict is + # alive and not modified. So we can return a borrowed ref. + return w_res @cpython_api([PyObject, CONST_STRING], rffi.INT_real, error=-1) def PyDict_DelItemString(space, w_dict, key_ptr): @@ -170,10 +175,13 @@ def PyDict_Next(space, w_dict, ppos, pkey, pvalue): if w_dict is None: return 0 - # Note: this is not efficient. Storing an iterator would probably + # XXX XXX PyDict_Next is not efficient. Storing an iterator would probably # work, but we can't work out how to not leak it if iteration does - # not complete. + # not complete. Alternatively, we could add some RPython-only + # dict-iterator method to move forward by N steps. + w_dict.ensure_object_strategy() # make sure both keys and values can + # be borrwed try: w_iter = space.call_method(space.w_dict, "iteritems", w_dict) pos = ppos[0] @@ -183,11 +191,10 @@ def PyDict_Next(space, w_dict, ppos, pkey, pvalue): w_item = space.call_method(w_iter, "next") w_key, w_value = space.fixedview(w_item, 2) - state = space.fromcache(RefcountState) if pkey: - pkey[0] = state.make_borrowed(w_dict, w_key) + pkey[0] = as_pyobj(space, w_key) if pvalue: - pvalue[0] = state.make_borrowed(w_dict, w_value) + pvalue[0] = as_pyobj(space, w_value) ppos[0] += 1 except OperationError, e: if not e.match(space, space.w_StopIteration): diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py index 6e18e0e337..449852f7e3 100644 --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -4,7 +4,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP, cpython_struct, is_valid_fp) -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.pyerrors import PyErr_SetFromErrno from pypy.module.cpyext.funcobject import PyCodeObject from pypy.module.__builtin__ import compiling @@ -23,7 +23,7 @@ PyCF_MASK = (consts.CO_FUTURE_DIVISION | def PyEval_CallObjectWithKeywords(space, w_obj, w_arg, w_kwds): return space.call(w_obj, w_arg, w_kwds) -@cpython_api([], PyObject) +@cpython_api([], PyObject, result_borrowed=True) def PyEval_GetBuiltins(space): """Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is @@ -36,25 +36,25 @@ def PyEval_GetBuiltins(space): w_builtins = w_builtins.getdict(space) else: w_builtins = space.builtin.getdict(space) - return borrow_from(None, w_builtins) + return w_builtins # borrowed ref in all cases -@cpython_api([], PyObject, error=CANNOT_FAIL) +@cpython_api([], PyObject, error=CANNOT_FAIL, result_borrowed=True) def PyEval_GetLocals(space): """Return a dictionary of the local variables in the current execution frame, or NULL if no frame is currently executing.""" caller = space.getexecutioncontext().gettopframe_nohidden() if caller is None: return None - return borrow_from(None, caller.getdictscope()) + return caller.getdictscope() # borrowed ref -@cpython_api([], PyObject, error=CANNOT_FAIL) +@cpython_api([], PyObject, error=CANNOT_FAIL, result_borrowed=True) def PyEval_GetGlobals(space): """Return a dictionary of the global variables in the current execution frame, or NULL if no frame is currently executing.""" caller = space.getexecutioncontext().gettopframe_nohidden() if caller is None: return None - return borrow_from(None, caller.get_w_globals()) + return caller.get_w_globals() # borrowed ref @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py index abc9b019e4..20f974d5d6 100644 --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import ( PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t, cpython_api, bootstrap_function, cpython_struct, build_type_checkers) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, borrow_from) + PyObject, make_ref, from_ref, Py_DecRef, make_typedescr) from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError from pypy.interpreter.function import Function, Method @@ -83,12 +83,12 @@ def code_dealloc(space, py_obj): from pypy.module.cpyext.object import PyObject_dealloc PyObject_dealloc(space, py_obj) -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyFunction_GetCode(space, w_func): """Return the code object associated with the function object op.""" func = space.interp_w(Function, w_func) w_code = space.wrap(func.code) - return borrow_from(w_func, w_code) + return w_code # borrowed ref @cpython_api([PyObject, PyObject, PyObject], PyObject) def PyMethod_New(space, w_func, w_self, w_cls): @@ -99,25 +99,25 @@ def PyMethod_New(space, w_func, w_self, w_cls): class which provides the unbound method.""" return Method(space, w_func, w_self, w_cls) -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyMethod_Function(space, w_method): """Return the function object associated with the method meth.""" assert isinstance(w_method, Method) - return borrow_from(w_method, w_method.w_function) + return w_method.w_function # borrowed ref -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyMethod_Self(space, w_method): """Return the instance associated with the method meth if it is bound, otherwise return NULL.""" assert isinstance(w_method, Method) - return borrow_from(w_method, w_method.w_instance) + return w_method.w_instance # borrowed ref -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyMethod_Class(space, w_method): """Return the class object from which the method meth was created; if this was created from an instance, it will be the class of the instance.""" assert isinstance(w_method, Method) - return borrow_from(w_method, w_method.w_class) + return w_method.w_class # borrowed ref def unwrap_list_of_strings(space, w_list): return [space.str_w(w_item) for w_item in space.fixedview(w_list)] diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py index bfe762669c..5ec4cbd3bd 100644 --- a/pypy/module/cpyext/import_.py +++ b/pypy/module/cpyext/import_.py @@ -1,7 +1,6 @@ from pypy.interpreter import module from pypy.module.cpyext.api import ( generic_cpy_call, cpython_api, PyObject, CONST_STRING) -from pypy.module.cpyext.pyobject import borrow_from from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError from pypy.interpreter.module import Module @@ -56,7 +55,7 @@ def PyImport_ReloadModule(space, w_mod): from pypy.module.imp.importing import reload return reload(space, w_mod) -@cpython_api([CONST_STRING], PyObject) +@cpython_api([CONST_STRING], PyObject, result_borrowed=True) def PyImport_AddModule(space, name): """Return the module object corresponding to a module name. The name argument may be of the form package.module. First check the modules @@ -74,14 +73,16 @@ def PyImport_AddModule(space, name): w_mod = check_sys_modules_w(space, modulename) if not w_mod or space.is_w(w_mod, space.w_None): w_mod = Module(space, space.wrap(modulename)) - return borrow_from(None, w_mod) + space.setitem(space.sys.get('modules'), space.wrap(modulename), w_mod) + # return a borrowed ref --- assumes one copy in sys.modules + return w_mod -@cpython_api([], PyObject) +@cpython_api([], PyObject, result_borrowed=True) def PyImport_GetModuleDict(space): """Return the dictionary used for the module administration (a.k.a. sys.modules). Note that this is a per-interpreter variable.""" w_modulesDict = space.sys.get('modules') - return borrow_from(None, w_modulesDict) + return w_modulesDict # borrowed ref @cpython_api([rffi.CCHARP, PyObject], PyObject) def PyImport_ExecCodeModule(space, name, w_code): diff --git a/pypy/module/cpyext/include/complexobject.h b/pypy/module/cpyext/include/complexobject.h index 16cf569aec..16e11e80cb 100644 --- a/pypy/module/cpyext/include/complexobject.h +++ b/pypy/module/cpyext/include/complexobject.h @@ -15,7 +15,7 @@ typedef struct Py_complex_t { } Py_complex; /* generated function */ -PyAPI_FUNC(void) _PyComplex_AsCComplex(PyObject *, Py_complex *); +PyAPI_FUNC(int) _PyComplex_AsCComplex(PyObject *, Py_complex *); PyAPI_FUNC(PyObject *) _PyComplex_FromCComplex(Py_complex *); Py_LOCAL_INLINE(Py_complex) PyComplex_AsCComplex(PyObject *obj) diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h index 3759f33b97..15ee723c6b 100644 --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -17,7 +17,8 @@ we have it for compatibility with CPython. #define staticforward static #define PyObject_HEAD \ - long ob_refcnt; \ + Py_ssize_t ob_refcnt; \ + Py_ssize_t ob_pypy_link; \ struct _typeobject *ob_type; #define PyObject_VAR_HEAD \ @@ -25,7 +26,7 @@ we have it for compatibility with CPython. Py_ssize_t ob_size; /* Number of items in variable part */ #define PyObject_HEAD_INIT(type) \ - 1, type, + 1, 0, type, #define PyVarObject_HEAD_INIT(type, size) \ PyObject_HEAD_INIT(type) size, @@ -40,19 +41,19 @@ typedef struct { #ifdef PYPY_DEBUG_REFCOUNT /* Slow version, but useful for debugging */ -#define Py_INCREF(ob) (Py_IncRef((PyObject *)ob)) -#define Py_DECREF(ob) (Py_DecRef((PyObject *)ob)) -#define Py_XINCREF(ob) (Py_IncRef((PyObject *)ob)) -#define Py_XDECREF(ob) (Py_DecRef((PyObject *)ob)) +#define Py_INCREF(ob) (Py_IncRef((PyObject *)(ob))) +#define Py_DECREF(ob) (Py_DecRef((PyObject *)(ob))) +#define Py_XINCREF(ob) (Py_IncRef((PyObject *)(ob))) +#define Py_XDECREF(ob) (Py_DecRef((PyObject *)(ob))) #else /* Fast version */ -#define Py_INCREF(ob) (((PyObject *)ob)->ob_refcnt++) -#define Py_DECREF(ob) \ +#define Py_INCREF(ob) (((PyObject *)(ob))->ob_refcnt++) +#define Py_DECREF(op) \ do { \ - if (((PyObject *)ob)->ob_refcnt > 1) \ - ((PyObject *)ob)->ob_refcnt--; \ + if (--((PyObject *)(op))->ob_refcnt != 0) \ + ; \ else \ - Py_DecRef((PyObject *)ob); \ + _Py_Dealloc((PyObject *)(op)); \ } while (0) #define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h index f2d99af634..2b0f78c94f 100644 --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -30,6 +30,13 @@ /* PyPy version as a string */ #define PYPY_VERSION "4.1.0-alpha0" +#define PYPY_VERSION_NUM 0x04010000 + +/* Defined to mean a PyPy where cpyext holds more regular references + to PyObjects, e.g. staying alive as long as the internal PyPy object + stays alive. */ +#define PYPY_CPYEXT_GC 1 +#define PyPy_Borrow(a, b) ((void) 0) /* Subversion Revision number of this file (not of the repository). * Empty since Mercurial migration. */ diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h index 8ffd9cfd1f..bcde5b3ca4 100644 --- a/pypy/module/cpyext/include/tupleobject.h +++ b/pypy/module/cpyext/include/tupleobject.h @@ -7,11 +7,21 @@ extern "C" { #endif +typedef struct { + PyObject_HEAD + Py_ssize_t ob_size; + PyObject **ob_item; /* XXX optimize to ob_item[] */ +} PyTupleObject; + /* defined in varargswrapper.c */ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); -#define PyTuple_SET_ITEM PyTuple_SetItem -#define PyTuple_GET_ITEM PyTuple_GetItem +/* Macro, trading safety for speed */ +#define PyTuple_GET_ITEM(op, i) (((PyTupleObject *)(op))->ob_item[i]) +#define PyTuple_GET_SIZE(op) Py_SIZE(op) + +/* Macro, *only* to be used to fill in brand new tuples */ +#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v) #ifdef __cplusplus diff --git a/pypy/module/cpyext/intobject.py b/pypy/module/cpyext/intobject.py index 2e27c70df6..4bf5c890c2 100644 --- a/pypy/module/cpyext/intobject.py +++ b/pypy/module/cpyext/intobject.py @@ -5,7 +5,7 @@ from pypy.module.cpyext.api import ( cpython_api, cpython_struct, build_type_checkers, bootstrap_function, PyObject, PyObjectFields, CONST_STRING, CANNOT_FAIL, Py_ssize_t) from pypy.module.cpyext.pyobject import ( - make_typedescr, track_reference, RefcountState, from_ref) + make_typedescr, track_reference, from_ref) from rpython.rlib.rarithmetic import r_uint, intmask, LONG_TEST, r_ulonglong from pypy.objspace.std.intobject import W_IntObject import sys @@ -19,7 +19,7 @@ cpython_struct("PyIntObject", PyIntObjectFields, PyIntObjectStruct) @bootstrap_function def init_intobject(space): "Type description of PyIntObject" - make_typedescr(space.w_int.instancetypedef, + make_typedescr(space.w_int.layout.typedef, basestruct=PyIntObject.TO, attach=int_attach, realize=int_realize) @@ -38,8 +38,6 @@ def int_realize(space, obj): w_obj = space.allocate_instance(W_IntObject, w_type) w_obj.__init__(intval) track_reference(space, obj, w_obj) - state = space.fromcache(RefcountState) - state.set_lifeline(w_obj, obj) return w_obj PyInt_Check, PyInt_CheckExact = build_type_checkers("Int") @@ -53,7 +51,7 @@ def PyInt_GetMax(space): @cpython_api([lltype.Signed], PyObject) def PyInt_FromLong(space, ival): """Create a new integer object with a value of ival. - + """ return space.wrap(ival) diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py index 2da4152188..4028be74fb 100644 --- a/pypy/module/cpyext/listobject.py +++ b/pypy/module/cpyext/listobject.py @@ -3,7 +3,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t, build_type_checkers) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall -from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, borrow_from +from pypy.module.cpyext.pyobject import Py_DecRef, PyObject from pypy.objspace.std.listobject import W_ListObject from pypy.interpreter.error import OperationError @@ -38,7 +38,7 @@ def PyList_SetItem(space, w_list, index, w_item): w_list.setitem(index, w_item) return 0 -@cpython_api([PyObject, Py_ssize_t], PyObject) +@cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True) def PyList_GetItem(space, w_list, index): """Return the object at position pos in the list pointed to by p. The position must be positive, indexing from the end of the list is not @@ -49,8 +49,10 @@ def PyList_GetItem(space, w_list, index): if index < 0 or index >= w_list.length(): raise OperationError(space.w_IndexError, space.wrap( "list index out of range")) - w_item = w_list.getitem(index) - return borrow_from(w_list, w_item) + w_list.ensure_object_strategy() # make sure we can return a borrowed obj + # XXX ^^^ how does this interact with CPyListStrategy? + w_res = w_list.getitem(index) + return w_res # borrowed ref @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py index ff4a9a62ec..c949946ad2 100644 --- a/pypy/module/cpyext/modsupport.py +++ b/pypy/module/cpyext/modsupport.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import cpython_api, cpython_struct, \ METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from pypy.interpreter.module import Module from pypy.module.cpyext.methodobject import ( W_PyCFunctionObject, PyCFunction_NewEx, PyDescr_NewMethod, @@ -34,7 +34,7 @@ def PyImport_AddModule(space, name): # This is actually the Py_InitModule4 function, # renamed to refuse modules built against CPython headers. @cpython_api([CONST_STRING, lltype.Ptr(PyMethodDef), CONST_STRING, - PyObject, rffi.INT_real], PyObject) + PyObject, rffi.INT_real], PyObject, result_borrowed=True) def _Py_InitPyPyModule(space, name, methods, doc, w_self, apiver): """ Create a new module object based on a name and table of functions, returning @@ -69,7 +69,7 @@ def _Py_InitPyPyModule(space, name, methods, doc, w_self, apiver): if doc: space.setattr(w_mod, space.wrap("__doc__"), space.wrap(rffi.charp2str(doc))) - return borrow_from(None, w_mod) + return w_mod # borrowed result kept alive in PyImport_AddModule() def convert_method_defs(space, dict_w, methods, w_type, w_self=None, name=None): @@ -114,12 +114,12 @@ def PyModule_Check(space, w_obj): return int(space.is_w(w_type, w_obj_type) or space.is_true(space.issubtype(w_obj_type, w_type))) -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyModule_GetDict(space, w_mod): if PyModule_Check(space, w_mod): assert isinstance(w_mod, Module) w_dict = w_mod.getdict(space) - return borrow_from(w_mod, w_dict) + return w_dict # borrowed reference, likely from w_mod.w_dict else: PyErr_BadInternalCall(space) diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py index ea78350361..d0b3939afe 100644 --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -6,7 +6,7 @@ from pypy.module.cpyext.api import ( Py_GE, CONST_STRING, FILEP, fwrite) from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, create_ref, from_ref, Py_IncRef, Py_DecRef, - track_reference, get_typedescr, _Py_NewReference, RefcountState) + get_typedescr, _Py_NewReference) from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall from pypy.objspace.std.typeobject import W_TypeObject @@ -31,9 +31,9 @@ def _PyObject_New(space, type): def _PyObject_NewVar(space, type, itemcount): w_type = from_ref(space, rffi.cast(PyObject, type)) assert isinstance(w_type, W_TypeObject) - typedescr = get_typedescr(w_type.instancetypedef) + typedescr = get_typedescr(w_type.layout.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - py_obj.c_ob_refcnt = 0 + #py_obj.c_ob_refcnt = 0 --- will be set to 1 again by PyObject_Init{Var} if type.c_tp_itemsize == 0: w_obj = PyObject_Init(space, py_obj, type) else: diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py index 4b2cd7d44b..ba15bf1643 100644 --- a/pypy/module/cpyext/pyerrors.py +++ b/pypy/module/cpyext/pyerrors.py @@ -6,7 +6,7 @@ from pypy.interpreter import pytraceback from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, CONST_STRING from pypy.module.exceptions.interp_exceptions import W_RuntimeWarning from pypy.module.cpyext.pyobject import ( - PyObject, PyObjectP, make_ref, from_ref, Py_DecRef, borrow_from) + PyObject, PyObjectP, make_ref, from_ref, Py_DecRef) from pypy.module.cpyext.state import State from pypy.module.cpyext.import_ import PyImport_Import from rpython.rlib import rposix, jit @@ -28,12 +28,12 @@ def PyErr_SetNone(space, w_type): """This is a shorthand for PyErr_SetObject(type, Py_None).""" PyErr_SetObject(space, w_type, space.w_None) -@cpython_api([], PyObject) +@cpython_api([], PyObject, result_borrowed=True) def PyErr_Occurred(space): state = space.fromcache(State) if state.operror is None: return None - return borrow_from(None, state.operror.w_type) + return state.operror.w_type # borrowed ref @cpython_api([], lltype.Void) def PyErr_Clear(space): diff --git a/pypy/module/cpyext/pyfile.py b/pypy/module/cpyext/pyfile.py index 628a90a2c7..5688f6ee37 100644 --- a/pypy/module/cpyext/pyfile.py +++ b/pypy/module/cpyext/pyfile.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, build_type_checkers) -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from pypy.module.cpyext.object import Py_PRINT_RAW from pypy.interpreter.error import OperationError from pypy.module._file.interp_file import W_File @@ -83,7 +83,8 @@ def PyFile_WriteObject(space, w_obj, w_p, flags): @cpython_api([PyObject], PyObject) def PyFile_Name(space, w_p): """Return the name of the file specified by p as a string object.""" - return borrow_from(w_p, space.getattr(w_p, space.wrap("name"))) + w_name = space.getattr(w_p, space.wrap("name")) + return w_name # borrowed ref, should be a W_StringObject from the file @cpython_api([PyObject, rffi.INT_real], rffi.INT_real, error=CANNOT_FAIL) def PyFile_SoftSpace(space, w_p, newflag): diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py index ceac295fab..793bac0354 100644 --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -2,15 +2,19 @@ import sys from pypy.interpreter.baseobjspace import W_Root, SpaceCache from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.extregistry import ExtRegistryEntry from pypy.module.cpyext.api import ( cpython_api, bootstrap_function, PyObject, PyObjectP, ADDR, - CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr) + CANNOT_FAIL, Py_TPFLAGS_HEAPTYPE, PyTypeObjectPtr, is_PyObject, + INTERPLEVEL_API) from pypy.module.cpyext.state import State from pypy.objspace.std.typeobject import W_TypeObject from pypy.objspace.std.objectobject import W_ObjectObject from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rweakref import RWeakKeyDictionary +from rpython.rlib.objectmodel import keepalive_until_here from rpython.rtyper.annlowlevel import llhelper +from rpython.rlib import rawrefcount + #________________________________________________________ # type description @@ -28,13 +32,15 @@ class BaseCpyTypedescr(object): def allocate(self, space, w_type, itemcount=0): # similar to PyType_GenericAlloc? # except that it's not related to any pypy object. + # this returns a PyObject with ob_refcnt == 1. - pytype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_type)) + pytype = as_pyobj(space, w_type) + pytype = rffi.cast(PyTypeObjectPtr, pytype) + assert pytype # Don't increase refcount for non-heaptypes - if pytype: - flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) - if not flags & Py_TPFLAGS_HEAPTYPE: - Py_DecRef(space, w_type) + flags = rffi.cast(lltype.Signed, pytype.c_tp_flags) + if flags & Py_TPFLAGS_HEAPTYPE: + Py_IncRef(space, w_type) if pytype: size = pytype.c_tp_basicsize @@ -42,6 +48,7 @@ class BaseCpyTypedescr(object): size = rffi.sizeof(self.basestruct) if itemcount: size += itemcount * pytype.c_tp_itemsize + assert size >= rffi.sizeof(PyObject.TO) buf = lltype.malloc(rffi.VOIDP.TO, size, flavor='raw', zero=True) pyobj = rffi.cast(PyObject, buf) @@ -56,9 +63,6 @@ class BaseCpyTypedescr(object): w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) w_obj = space.allocate_instance(self.W_BaseObject, w_type) track_reference(space, obj, w_obj) - if w_type is not space.gettypefor(self.W_BaseObject): - state = space.fromcache(RefcountState) - state.set_lifeline(w_obj, obj) return w_obj typedescr_cache = {} @@ -111,7 +115,7 @@ def make_typedescr(typedef, **kw): def init_pyobject(space): from pypy.module.cpyext.object import PyObject_dealloc # typedescr for the 'object' type - make_typedescr(space.w_object.instancetypedef, + make_typedescr(space.w_object.layout.typedef, dealloc=PyObject_dealloc) # almost all types, which should better inherit from object. make_typedescr(None) @@ -134,104 +138,6 @@ def get_typedescr(typedef): #________________________________________________________ # refcounted object support -class RefcountState: - def __init__(self, space): - self.space = space - self.py_objects_w2r = {} # { w_obj -> raw PyObject } - self.py_objects_r2w = {} # { addr of raw PyObject -> w_obj } - - self.lifeline_dict = RWeakKeyDictionary(W_Root, PyOLifeline) - - self.borrow_mapping = {None: {}} - # { w_container -> { w_containee -> None } } - # the None entry manages references borrowed during a call to - # generic_cpy_call() - - # For tests - self.non_heaptypes_w = [] - - def _cleanup_(self): - assert self.borrow_mapping == {None: {}} - self.py_objects_r2w.clear() # is not valid anymore after translation - - def init_r2w_from_w2r(self): - """Rebuilds the dict py_objects_r2w on startup""" - for w_obj, obj in self.py_objects_w2r.items(): - ptr = rffi.cast(ADDR, obj) - self.py_objects_r2w[ptr] = w_obj - - def print_refcounts(self): - print "REFCOUNTS" - for w_obj, obj in self.py_objects_w2r.items(): - print "%r: %i" % (w_obj, obj.c_ob_refcnt) - - def get_from_lifeline(self, w_obj): - lifeline = self.lifeline_dict.get(w_obj) - if lifeline is not None: # make old PyObject ready for use in C code - py_obj = lifeline.pyo - assert py_obj.c_ob_refcnt == 0 - return py_obj - else: - return lltype.nullptr(PyObject.TO) - - def set_lifeline(self, w_obj, py_obj): - self.lifeline_dict.set(w_obj, - PyOLifeline(self.space, py_obj)) - - def make_borrowed(self, w_container, w_borrowed): - """ - Create a borrowed reference, which will live as long as the container - has a living reference (as a PyObject!) - """ - ref = make_ref(self.space, w_borrowed) - obj_ptr = rffi.cast(ADDR, ref) - - borrowees = self.borrow_mapping.setdefault(w_container, {}) - if w_borrowed in borrowees: - Py_DecRef(self.space, w_borrowed) # cancel incref from make_ref() - else: - borrowees[w_borrowed] = None - - return ref - - def reset_borrowed_references(self): - "Used in tests" - for w_container, w_borrowed in self.borrow_mapping.items(): - Py_DecRef(self.space, w_borrowed) - self.borrow_mapping = {None: {}} - - def delete_borrower(self, w_obj): - """ - Called when a potential container for borrowed references has lost its - last reference. Removes the borrowed references it contains. - """ - if w_obj in self.borrow_mapping: # move to lifeline __del__ - for w_containee in self.borrow_mapping[w_obj]: - self.forget_borrowee(w_containee) - del self.borrow_mapping[w_obj] - - def swap_borrow_container(self, container): - """switch the current default contained with the given one.""" - if container is None: - old_container = self.borrow_mapping[None] - self.borrow_mapping[None] = {} - return old_container - else: - old_container = self.borrow_mapping[None] - self.borrow_mapping[None] = container - for w_containee in old_container: - self.forget_borrowee(w_containee) - - def forget_borrowee(self, w_obj): - "De-register an object from the list of borrowed references" - ref = self.py_objects_w2r.get(w_obj, lltype.nullptr(PyObject.TO)) - if not ref: - if DEBUG_REFCOUNT: - print >>sys.stderr, "Borrowed object is already gone!" - return - - Py_DecRef(self.space, ref) - class InvalidPointerException(Exception): pass @@ -249,72 +155,50 @@ def debug_refcount(*args, **kwargs): def create_ref(space, w_obj, itemcount=0): """ Allocates a PyObject, and fills its fields with info from the given - intepreter object. + interpreter object. """ - state = space.fromcache(RefcountState) w_type = space.type(w_obj) - if w_type.is_cpytype(): - py_obj = state.get_from_lifeline(w_obj) - if py_obj: - Py_IncRef(space, py_obj) - return py_obj - typedescr = get_typedescr(w_obj.typedef) py_obj = typedescr.allocate(space, w_type, itemcount=itemcount) - if w_type.is_cpytype(): - state.set_lifeline(w_obj, py_obj) + track_reference(space, py_obj, w_obj) + # + # py_obj.c_ob_refcnt should be exactly REFCNT_FROM_PYPY + 1 here, + # and we want only REFCNT_FROM_PYPY, i.e. only count as attached + # to the W_Root but not with any reference from the py_obj side. + assert py_obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY + py_obj.c_ob_refcnt -= 1 + # typedescr.attach(space, py_obj, w_obj) return py_obj -def track_reference(space, py_obj, w_obj, replace=False): +def track_reference(space, py_obj, w_obj): """ Ties together a PyObject and an interpreter object. + The PyObject's refcnt is increased by REFCNT_FROM_PYPY. + The reference in 'py_obj' is not stolen! Remember to Py_DecRef() + it is you need to. """ # XXX looks like a PyObject_GC_TRACK - ptr = rffi.cast(ADDR, py_obj) - state = space.fromcache(RefcountState) + assert py_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY + py_obj.c_ob_refcnt += rawrefcount.REFCNT_FROM_PYPY if DEBUG_REFCOUNT: debug_refcount("MAKREF", py_obj, w_obj) - if not replace: - assert w_obj not in state.py_objects_w2r - assert ptr not in state.py_objects_r2w - state.py_objects_w2r[w_obj] = py_obj - if ptr: # init_typeobject() bootstraps with NULL references - state.py_objects_r2w[ptr] = w_obj - -def make_ref(space, w_obj): - """ - Returns a new reference to an intepreter object. - """ - if w_obj is None: - return lltype.nullptr(PyObject.TO) - assert isinstance(w_obj, W_Root) - state = space.fromcache(RefcountState) - try: - py_obj = state.py_objects_w2r[w_obj] - except KeyError: - py_obj = create_ref(space, w_obj) - track_reference(space, py_obj, w_obj) - else: - Py_IncRef(space, py_obj) - return py_obj + assert w_obj + assert py_obj + rawrefcount.create_link_pypy(w_obj, py_obj) def from_ref(space, ref): """ Finds the interpreter object corresponding to the given reference. If the - object is not yet realized (see stringobject.py), creates it. + object is not yet realized (see bytesobject.py), creates it. """ - assert lltype.typeOf(ref) == PyObject + assert is_pyobj(ref) if not ref: return None - state = space.fromcache(RefcountState) - ptr = rffi.cast(ADDR, ref) - - try: - return state.py_objects_r2w[ptr] - except KeyError: - pass + w_obj = rawrefcount.to_obj(W_Root, ref) + if w_obj is not None: + return w_obj # This reference is not yet a real interpreter object. # Realize it. @@ -323,126 +207,135 @@ def from_ref(space, ref): raise InvalidPointerException(str(ref)) w_type = from_ref(space, ref_type) assert isinstance(w_type, W_TypeObject) - return get_typedescr(w_type.instancetypedef).realize(space, ref) + return get_typedescr(w_type.layout.typedef).realize(space, ref) -# XXX Optimize these functions and put them into macro definitions -@cpython_api([PyObject], lltype.Void) -def Py_DecRef(space, obj): - if not obj: - return - assert lltype.typeOf(obj) == PyObject +def debug_collect(): + rawrefcount._collect() - obj.c_ob_refcnt -= 1 - if DEBUG_REFCOUNT: - debug_refcount("DECREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) - if obj.c_ob_refcnt == 0: - state = space.fromcache(RefcountState) - ptr = rffi.cast(ADDR, obj) - if ptr not in state.py_objects_r2w: - # this is a half-allocated object, lets call the deallocator - # without modifying the r2w/w2r dicts - _Py_Dealloc(space, obj) - else: - w_obj = state.py_objects_r2w[ptr] - del state.py_objects_r2w[ptr] - w_type = space.type(w_obj) - if not w_type.is_cpytype(): + +def as_pyobj(space, w_obj): + """ + Returns a 'PyObject *' representing the given intepreter object. + This doesn't give a new reference, but the returned 'PyObject *' + is valid at least as long as 'w_obj' is. **To be safe, you should + use keepalive_until_here(w_obj) some time later.** In case of + doubt, use the safer make_ref(). + """ + if w_obj is not None: + assert not is_pyobj(w_obj) + py_obj = rawrefcount.from_obj(PyObject, w_obj) + if not py_obj: + py_obj = create_ref(space, w_obj) + return py_obj + else: + return lltype.nullptr(PyObject.TO) +as_pyobj._always_inline_ = 'try' +INTERPLEVEL_API['as_pyobj'] = as_pyobj + +def pyobj_has_w_obj(pyobj): + return rawrefcount.to_obj(W_Root, pyobj) is not None +INTERPLEVEL_API['pyobj_has_w_obj'] = staticmethod(pyobj_has_w_obj) + + +def is_pyobj(x): + if x is None or isinstance(x, W_Root): + return False + elif is_PyObject(lltype.typeOf(x)): + return True + else: + raise TypeError(repr(type(x))) +INTERPLEVEL_API['is_pyobj'] = staticmethod(is_pyobj) + +class Entry(ExtRegistryEntry): + _about_ = is_pyobj + def compute_result_annotation(self, s_x): + from rpython.rtyper.llannotation import SomePtr + return self.bookkeeper.immutablevalue(isinstance(s_x, SomePtr)) + def specialize_call(self, hop): + hop.exception_cannot_occur() + return hop.inputconst(lltype.Bool, hop.s_result.const) + +@specialize.ll() +def make_ref(space, obj): + """Increment the reference counter of the PyObject and return it. + Can be called with either a PyObject or a W_Root. + """ + if is_pyobj(obj): + pyobj = rffi.cast(PyObject, obj) + else: + pyobj = as_pyobj(space, obj) + if pyobj: + assert pyobj.c_ob_refcnt > 0 + pyobj.c_ob_refcnt += 1 + if not is_pyobj(obj): + keepalive_until_here(obj) + return pyobj +INTERPLEVEL_API['make_ref'] = make_ref + + +@specialize.ll() +def get_w_obj_and_decref(space, obj): + """Decrement the reference counter of the PyObject and return the + corresponding W_Root object (so the reference count is at least + REFCNT_FROM_PYPY and cannot be zero). Can be called with either + a PyObject or a W_Root. + """ + if is_pyobj(obj): + pyobj = rffi.cast(PyObject, obj) + w_obj = from_ref(space, pyobj) + else: + w_obj = obj + pyobj = as_pyobj(space, w_obj) + if pyobj: + pyobj.c_ob_refcnt -= 1 + assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY + keepalive_until_here(w_obj) + return w_obj +INTERPLEVEL_API['get_w_obj_and_decref'] = get_w_obj_and_decref + + +@specialize.ll() +def incref(space, obj): + make_ref(space, obj) +INTERPLEVEL_API['incref'] = incref + +@specialize.ll() +def decref(space, obj): + if is_pyobj(obj): + obj = rffi.cast(PyObject, obj) + if obj: + assert obj.c_ob_refcnt > 0 + obj.c_ob_refcnt -= 1 + if obj.c_ob_refcnt == 0: _Py_Dealloc(space, obj) - del state.py_objects_w2r[w_obj] - # if the object was a container for borrowed references - state.delete_borrower(w_obj) else: - if not we_are_translated() and obj.c_ob_refcnt < 0: - message = "Negative refcount for obj %s with type %s" % ( - obj, rffi.charp2str(obj.c_ob_type.c_tp_name)) - print >>sys.stderr, message - assert False, message + get_w_obj_and_decref(space, obj) +INTERPLEVEL_API['decref'] = decref + @cpython_api([PyObject], lltype.Void) def Py_IncRef(space, obj): - if not obj: - return - obj.c_ob_refcnt += 1 - assert obj.c_ob_refcnt > 0 - if DEBUG_REFCOUNT: - debug_refcount("INCREF", obj, obj.c_ob_refcnt, frame_stackdepth=3) + incref(space, obj) + +@cpython_api([PyObject], lltype.Void) +def Py_DecRef(space, obj): + decref(space, obj) @cpython_api([PyObject], lltype.Void) def _Py_NewReference(space, obj): obj.c_ob_refcnt = 1 w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type)) assert isinstance(w_type, W_TypeObject) - get_typedescr(w_type.instancetypedef).realize(space, obj) + get_typedescr(w_type.layout.typedef).realize(space, obj) +@cpython_api([PyObject], lltype.Void) def _Py_Dealloc(space, obj): - from pypy.module.cpyext.api import generic_cpy_call_dont_decref + from pypy.module.cpyext.api import generic_cpy_call pto = obj.c_ob_type #print >>sys.stderr, "Calling dealloc slot", pto.c_tp_dealloc, "of", obj, \ # "'s type which is", rffi.charp2str(pto.c_tp_name) - generic_cpy_call_dont_decref(space, pto.c_tp_dealloc, obj) - -#___________________________________________________________ -# Support for "lifelines" -# -# Object structure must stay alive even when not referenced -# by any C code. - -class PyOLifeline(object): - def __init__(self, space, pyo): - self.pyo = pyo - self.space = space - - def __del__(self): - if self.pyo: - assert self.pyo.c_ob_refcnt == 0 - _Py_Dealloc(self.space, self.pyo) - self.pyo = lltype.nullptr(PyObject.TO) - # XXX handle borrowed objects here - -#___________________________________________________________ -# Support for borrowed references - -def make_borrowed_ref(space, w_container, w_borrowed): - """ - Create a borrowed reference, which will live as long as the container - has a living reference (as a PyObject!) - """ - if w_borrowed is None: - return lltype.nullptr(PyObject.TO) - - state = space.fromcache(RefcountState) - return state.make_borrowed(w_container, w_borrowed) - -class Reference: - def __init__(self, pyobj): - assert not isinstance(pyobj, W_Root) - self.pyobj = pyobj - - def get_ref(self, space): - return self.pyobj - - def get_wrapped(self, space): - return from_ref(space, self.pyobj) - -class BorrowPair(Reference): - """ - Delays the creation of a borrowed reference. - """ - def __init__(self, w_container, w_borrowed): - self.w_container = w_container - self.w_borrowed = w_borrowed - - def get_ref(self, space): - return make_borrowed_ref(space, self.w_container, self.w_borrowed) - - def get_wrapped(self, space): - return self.w_borrowed - -def borrow_from(container, borrowed): - return BorrowPair(container, borrowed) - -#___________________________________________________________ + generic_cpy_call(space, pto.c_tp_dealloc, obj) @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL) def _Py_HashPointer(space, ptr): diff --git a/pypy/module/cpyext/pypyintf.py b/pypy/module/cpyext/pypyintf.py deleted file mode 100644 index 4723c4770c..0000000000 --- a/pypy/module/cpyext/pypyintf.py +++ /dev/null @@ -1,9 +0,0 @@ -from pypy.module.cpyext.api import cpython_api -from pypy.module.cpyext.pyobject import PyObject, borrow_from - - -@cpython_api([PyObject, PyObject], PyObject) -def PyPy_Borrow(space, w_parentobj, w_obj): - """Returns a borrowed reference to 'obj', borrowing from the 'parentobj'. - """ - return borrow_from(w_parentobj, w_obj) diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py index 066a084eb9..0d38408502 100644 --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -3,7 +3,7 @@ from pypy.module.cpyext.api import ( PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t, cpython_api, bootstrap_function, cpython_struct, build_type_checkers) from pypy.module.cpyext.pyobject import ( - PyObject, make_ref, from_ref, Py_DecRef, make_typedescr, borrow_from) + PyObject, make_ref, from_ref, Py_DecRef, make_typedescr) from pypy.module.cpyext.frameobject import PyFrameObject from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py index 1ed1890632..f21b8c7d37 100644 --- a/pypy/module/cpyext/sequence.py +++ b/pypy/module/cpyext/sequence.py @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.module.cpyext.api import ( cpython_api, CANNOT_FAIL, CONST_STRING, Py_ssize_t) -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from rpython.rtyper.lltypesystem import rffi, lltype from pypy.objspace.std import listobject, tupleobject @@ -42,15 +42,19 @@ def PySequence_Fast(space, w_obj, m): which case o is returned. Use PySequence_Fast_GET_ITEM() to access the members of the result. Returns NULL on failure. If the object is not a sequence, raises TypeError with m as the message text.""" - if (isinstance(w_obj, listobject.W_ListObject) or - isinstance(w_obj, tupleobject.W_TupleObject)): + if isinstance(w_obj, listobject.W_ListObject): + # make sure we can return a borrowed obj from PySequence_Fast_GET_ITEM + # XXX how does this interact with CPyListStrategy? + w_obj.ensure_object_strategy() + return w_obj + if isinstance(w_obj, tupleobject.W_TupleObject): return w_obj try: return tupleobject.W_TupleObject(space.fixedview(w_obj)) except OperationError: raise OperationError(space.w_TypeError, space.wrap(rffi.charp2str(m))) -@cpython_api([PyObject, Py_ssize_t], PyObject) +@cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True) def PySequence_Fast_GET_ITEM(space, w_obj, index): """Return the ith element of o, assuming that o was returned by PySequence_Fast(), o is not NULL, and that i is within bounds. @@ -60,7 +64,7 @@ def PySequence_Fast_GET_ITEM(space, w_obj, index): else: assert isinstance(w_obj, tupleobject.W_TupleObject) w_res = w_obj.wrappeditems[index] - return borrow_from(w_obj, w_res) + return w_res # borrowed ref @cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) def PySequence_Fast_GET_SIZE(space, w_obj): diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py index c82b3b6298..fac103117e 100644 --- a/pypy/module/cpyext/setobject.py +++ b/pypy/module/cpyext/setobject.py @@ -3,7 +3,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, build_type_checkers) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref) + make_ref, from_ref) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.setobject import W_SetObject, newset diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py index 7eb666daa9..9d6f327156 100644 --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -31,17 +31,16 @@ Py_GT = 4 Py_GE = 5 -def check_num_args(space, ob, n): - from pypy.module.cpyext.tupleobject import PyTuple_CheckExact, \ - PyTuple_GET_SIZE - if not PyTuple_CheckExact(space, ob): +def check_num_args(space, w_ob, n): + from pypy.module.cpyext.tupleobject import PyTuple_CheckExact + if not PyTuple_CheckExact(space, w_ob): raise OperationError(space.w_SystemError, space.wrap("PyArg_UnpackTuple() argument list is not a tuple")) - if n == PyTuple_GET_SIZE(space, ob): + if n == space.len_w(w_ob): return raise oefmt(space.w_TypeError, "expected %d arguments, got %d", - n, PyTuple_GET_SIZE(space, ob)) + n, space.len_w(w_ob)) def wrap_init(space, w_self, w_args, func, w_kwargs): func_init = rffi.cast(initproc, func) diff --git a/pypy/module/cpyext/src/getargs.c b/pypy/module/cpyext/src/getargs.c index 39ff59cd91..0055667210 100644 --- a/pypy/module/cpyext/src/getargs.c +++ b/pypy/module/cpyext/src/getargs.c @@ -442,7 +442,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, strncpy(msgbuf, "is not retrievable", bufsize); return msgbuf; } - PyPy_Borrow(arg, item); + //PyPy_Borrow(arg, item); msg = convertitem(item, &format, p_va, flags, levels+1, msgbuf, bufsize, freelist); /* PySequence_GetItem calls tp->sq_item, which INCREFs */ diff --git a/pypy/module/cpyext/state.py b/pypy/module/cpyext/state.py index 67a4e24d16..43e7bc77e3 100644 --- a/pypy/module/cpyext/state.py +++ b/pypy/module/cpyext/state.py @@ -1,8 +1,11 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rtyper.lltypesystem import rffi, lltype from pypy.interpreter.error import OperationError +from pypy.interpreter.executioncontext import AsyncAction from rpython.rtyper.lltypesystem import lltype +from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rdynload import DLLHANDLE +from rpython.rlib import rawrefcount import sys class State: @@ -11,6 +14,8 @@ class State: self.reset() self.programname = lltype.nullptr(rffi.CCHARP.TO) self.version = lltype.nullptr(rffi.CCHARP.TO) + pyobj_dealloc_action = PyObjDeallocAction(space) + self.dealloc_trigger = lambda: pyobj_dealloc_action.fire() def reset(self): from pypy.module.cpyext.modsupport import PyMethodDef @@ -74,13 +79,15 @@ class State: "This function is called when the program really starts" from pypy.module.cpyext.typeobject import setup_new_method_def - from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.api import INIT_FUNCTIONS + from pypy.module.cpyext.api import init_static_data_translated - setup_new_method_def(space) if we_are_translated(): - refcountstate = space.fromcache(RefcountState) - refcountstate.init_r2w_from_w2r() + rawrefcount.init(llhelper(rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER, + self.dealloc_trigger)) + init_static_data_translated(space) + + setup_new_method_def(space) for func in INIT_FUNCTIONS: func(space) @@ -133,3 +140,17 @@ class State: w_dict = w_mod.getdict(space) w_copy = space.call_method(w_dict, 'copy') self.extensions[path] = w_copy + + +class PyObjDeallocAction(AsyncAction): + """An action that invokes _Py_Dealloc() on the dying PyObjects. + """ + + def perform(self, executioncontext, frame): + from pypy.module.cpyext.pyobject import PyObject, _Py_Dealloc + + while True: + py_obj = rawrefcount.next_dead(PyObject) + if not py_obj: + break + _Py_Dealloc(self.space, py_obj) diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py index 5870ae82e7..a5b74996cb 100644 --- a/pypy/module/cpyext/structmember.py +++ b/pypy/module/cpyext/structmember.py @@ -6,7 +6,7 @@ from pypy.module.cpyext.api import ADDR, PyObjectP, cpython_api from pypy.module.cpyext.intobject import PyInt_AsLong, PyInt_AsUnsignedLong from pypy.module.cpyext.pyerrors import PyErr_Occurred from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, from_ref, make_ref -from pypy.module.cpyext.stringobject import ( +from pypy.module.cpyext.bytesobject import ( PyString_FromString, PyString_FromStringAndSize) from pypy.module.cpyext.floatobject import PyFloat_AsDouble from pypy.module.cpyext.longobject import ( diff --git a/pypy/module/cpyext/sysmodule.py b/pypy/module/cpyext/sysmodule.py index cc33c9246c..9511e53a6c 100644 --- a/pypy/module/cpyext/sysmodule.py +++ b/pypy/module/cpyext/sysmodule.py @@ -1,16 +1,16 @@ from pypy.interpreter.error import OperationError from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import CANNOT_FAIL, cpython_api, CONST_STRING -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject -@cpython_api([CONST_STRING], PyObject, error=CANNOT_FAIL) +@cpython_api([CONST_STRING], PyObject, error=CANNOT_FAIL, result_borrowed=True) def PySys_GetObject(space, name): """Return the object name from the sys module or NULL if it does not exist, without setting an exception.""" name = rffi.charp2str(name) w_dict = space.sys.getdict(space) w_obj = space.finditem_str(w_dict, name) - return borrow_from(None, w_obj) + return w_obj # borrowed ref: kept alive in space.sys.w_dict @cpython_api([CONST_STRING, PyObject], rffi.INT_real, error=-1) def PySys_SetObject(space, name, w_obj): diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c index 474d649c90..c9fededb2a 100644 --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -623,11 +623,17 @@ static PyTypeObject CustomType = { }; +static PyObject *size_of_instances(PyObject *self, PyObject *t) +{ + return PyInt_FromLong(((PyTypeObject *)t)->tp_basicsize); +} + /* List of functions exported by this module */ static PyMethodDef foo_functions[] = { {"new", (PyCFunction)foo_new, METH_NOARGS, NULL}, {"newCustom", (PyCFunction)newCustom, METH_NOARGS, NULL}, + {"size_of_instances", (PyCFunction)size_of_instances, METH_O, NULL}, {NULL, NULL} /* Sentinel */ }; diff --git a/pypy/module/cpyext/test/test_api.py b/pypy/module/cpyext/test/test_api.py index 5509d1049a..dbd49a4ff5 100644 --- a/pypy/module/cpyext/test/test_api.py +++ b/pypy/module/cpyext/test/test_api.py @@ -6,6 +6,7 @@ from pypy.module.cpyext import api from pypy.module.cpyext.test.test_cpyext import freeze_refcnts, LeakCheckingTest PyObject = api.PyObject from pypy.interpreter.error import OperationError +from rpython.rlib import rawrefcount import os @api.cpython_api([PyObject], lltype.Void) @@ -36,6 +37,9 @@ class BaseApiTest(LeakCheckingTest): cls.api = CAPI() CAPI.__dict__.update(api.INTERPLEVEL_API) + print 'DONT_FREE_ANY_MORE' + rawrefcount._dont_free_any_more() + def raises(self, space, api, expected_exc, f, *args): if not callable(f): raise Exception("%s is not callable" % (f,)) @@ -60,7 +64,7 @@ class BaseApiTest(LeakCheckingTest): raise try: - del self.space.getexecutioncontext().cpyext_threadstate + self.space.getexecutioncontext().cleanup_cpyext_threadstate() except AttributeError: pass diff --git a/pypy/module/cpyext/test/test_borrow.py b/pypy/module/cpyext/test/test_borrow.py index 74316d5403..56ec792228 100644 --- a/pypy/module/cpyext/test/test_borrow.py +++ b/pypy/module/cpyext/test/test_borrow.py @@ -1,20 +1,9 @@ import py from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.test.test_api import BaseApiTest -from pypy.module.cpyext.pyobject import make_ref, borrow_from, RefcountState +from pypy.module.cpyext.pyobject import make_ref -class TestBorrowing(BaseApiTest): - def test_borrowing(self, space, api): - w_int = space.wrap(1) - w_tuple = space.newtuple([w_int]) - api.Py_IncRef(w_tuple) - one_pyo = borrow_from(w_tuple, w_int).get_ref(space) - api.Py_DecRef(w_tuple) - state = space.fromcache(RefcountState) - state.print_refcounts() - py.test.raises(AssertionError, api.Py_DecRef, one_pyo) - class AppTestBorrow(AppTestCpythonExtensionBase): def test_tuple_borrowing(self): module = self.import_extension('foo', [ @@ -76,4 +65,5 @@ class AppTestBorrow(AppTestCpythonExtensionBase): ]) wr = module.run() # check that the set() object was deallocated + self.debug_collect() assert wr() is None diff --git a/pypy/module/cpyext/test/test_stringobject.py b/pypy/module/cpyext/test/test_bytesobject.py index 0b74e45df3..b81e297d47 100644 --- a/pypy/module/cpyext/test/test_stringobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from pypy.module.cpyext.stringobject import new_empty_str, PyStringObject +from pypy.module.cpyext.bytesobject import new_empty_str, PyStringObject from pypy.module.cpyext.api import PyObjectP, PyObject, Py_ssize_tP from pypy.module.cpyext.pyobject import Py_DecRef, from_ref, make_ref @@ -28,7 +28,7 @@ class AppTestStringObject(AppTestCpythonExtensionBase): if(PyString_Size(s) == 11) { result = 1; } - if(s->ob_type->tp_basicsize != sizeof(void*)*4) + if(s->ob_type->tp_basicsize != sizeof(void*)*5) result = 0; Py_DECREF(s); return PyBool_FromLong(result); @@ -231,7 +231,9 @@ class TestString(BaseApiTest): ref = make_ref(space, space.wrap('abc')) ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw') ptr[0] = ref + prev_refcnt = ref.c_ob_refcnt api.PyString_Concat(ptr, space.wrap('def')) + assert ref.c_ob_refcnt == prev_refcnt - 1 assert space.str_w(from_ref(space, ptr[0])) == 'abcdef' api.PyString_Concat(ptr, space.w_None) assert not ptr[0] @@ -244,14 +246,16 @@ class TestString(BaseApiTest): ref2 = make_ref(space, space.wrap('def')) ptr = lltype.malloc(PyObjectP.TO, 1, flavor='raw') ptr[0] = ref1 + prev_refcnf = ref2.c_ob_refcnt api.PyString_ConcatAndDel(ptr, ref2) assert space.str_w(from_ref(space, ptr[0])) == 'abcdef' - assert ref2.c_ob_refcnt == 0 + assert ref2.c_ob_refcnt == prev_refcnf - 1 Py_DecRef(space, ptr[0]) ptr[0] = lltype.nullptr(PyObject.TO) ref2 = make_ref(space, space.wrap('foo')) + prev_refcnf = ref2.c_ob_refcnt api.PyString_ConcatAndDel(ptr, ref2) # should not crash - assert ref2.c_ob_refcnt == 0 + assert ref2.c_ob_refcnt == prev_refcnf - 1 lltype.free(ptr, flavor='raw') def test_format(self, space, api): diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py index ef60d09b36..19ac9e117f 100644 --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -14,7 +14,7 @@ from rpython.translator.gensupp import uniquemodulename from rpython.tool.udir import udir from pypy.module.cpyext import api from pypy.module.cpyext.state import State -from pypy.module.cpyext.pyobject import RefcountState +from pypy.module.cpyext.pyobject import debug_collect from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from rpython.tool.identity_dict import identity_dict from rpython.tool import leakfinder @@ -73,7 +73,9 @@ def compile_extension_module(space, modname, **kwds): else: kwds["link_files"] = [str(api_library + '.so')] if sys.platform.startswith('linux'): - kwds["compile_extra"]=["-Werror=implicit-function-declaration"] + kwds["compile_extra"]=["-Werror=implicit-function-declaration", + "-g", "-O0"] + kwds["link_extra"]=["-g"] modname = modname.split('.')[-1] eci = ExternalCompilationInfo( @@ -92,6 +94,7 @@ def compile_extension_module(space, modname, **kwds): return str(pydname) def freeze_refcnts(self): + return #ZZZ state = self.space.fromcache(RefcountState) self.frozen_refcounts = {} for w_obj, obj in state.py_objects_w2r.iteritems(): @@ -109,6 +112,7 @@ class LeakCheckingTest(object): @staticmethod def cleanup_references(space): + return #ZZZ state = space.fromcache(RefcountState) import gc; gc.collect() @@ -127,10 +131,11 @@ class LeakCheckingTest(object): state.reset_borrowed_references() def check_and_print_leaks(self): + debug_collect() # check for sane refcnts import gc - if not self.enable_leak_checking: + if 1: #ZZZ not self.enable_leak_checking: leakfinder.stop_tracking_allocations(check=False) return False @@ -195,6 +200,9 @@ class AppTestApi(LeakCheckingTest): "the test actually passed in the first place; if it failed " "it is likely to reach this place.") + def test_only_import(self): + import cpyext + def test_load_error(self): import cpyext raises(ImportError, cpyext.load_module, "missing.file", "foo") @@ -212,8 +220,8 @@ class AppTestCpythonExtensionBase(LeakCheckingTest): cls.space.getbuiltinmodule("cpyext") from pypy.module.imp.importing import importhook importhook(cls.space, "os") # warm up reference counts - state = cls.space.fromcache(RefcountState) - state.non_heaptypes_w[:] = [] + #state = cls.space.fromcache(RefcountState) ZZZ + #state.non_heaptypes_w[:] = [] def setup_method(self, func): @unwrap_spec(name=str) @@ -348,7 +356,7 @@ class AppTestCpythonExtensionBase(LeakCheckingTest): interp2app(record_imported_module)) self.w_here = self.space.wrap( str(py.path.local(pypydir)) + '/module/cpyext/test/') - + self.w_debug_collect = self.space.wrap(interp2app(debug_collect)) # create the file lock before we count allocations self.space.call_method(self.space.sys.get("stdout"), "flush") @@ -638,8 +646,8 @@ class AppTestCpythonExtension(AppTestCpythonExtensionBase): static PyObject* foo_pi(PyObject* self, PyObject *args) { PyObject *true_obj = Py_True; - int refcnt = true_obj->ob_refcnt; - int refcnt_after; + Py_ssize_t refcnt = true_obj->ob_refcnt; + Py_ssize_t refcnt_after; Py_INCREF(true_obj); Py_INCREF(true_obj); PyBool_Check(true_obj); @@ -647,14 +655,14 @@ class AppTestCpythonExtension(AppTestCpythonExtensionBase): Py_DECREF(true_obj); Py_DECREF(true_obj); fprintf(stderr, "REFCNT %i %i\\n", refcnt, refcnt_after); - return PyBool_FromLong(refcnt_after == refcnt+2 && refcnt < 3); + return PyBool_FromLong(refcnt_after == refcnt + 2); } static PyObject* foo_bar(PyObject* self, PyObject *args) { PyObject *true_obj = Py_True; PyObject *tup = NULL; - int refcnt = true_obj->ob_refcnt; - int refcnt_after; + Py_ssize_t refcnt = true_obj->ob_refcnt; + Py_ssize_t refcnt_after; tup = PyTuple_New(1); Py_INCREF(true_obj); @@ -662,8 +670,10 @@ class AppTestCpythonExtension(AppTestCpythonExtensionBase): return NULL; refcnt_after = true_obj->ob_refcnt; Py_DECREF(tup); - fprintf(stderr, "REFCNT2 %i %i\\n", refcnt, refcnt_after); - return PyBool_FromLong(refcnt_after == refcnt); + fprintf(stderr, "REFCNT2 %i %i %i\\n", refcnt, refcnt_after, + true_obj->ob_refcnt); + return PyBool_FromLong(refcnt_after == refcnt + 1 && + refcnt == true_obj->ob_refcnt); } static PyMethodDef methods[] = { diff --git a/pypy/module/cpyext/test/test_getargs.py b/pypy/module/cpyext/test/test_getargs.py index f1fdee5e89..a89f3dfd8a 100644 --- a/pypy/module/cpyext/test/test_getargs.py +++ b/pypy/module/cpyext/test/test_getargs.py @@ -161,7 +161,9 @@ class AppTestGetargs(AppTestCpythonExtensionBase): freed.append('x') raises(TypeError, pybuffer, freestring("string"), freestring("other string"), 42) - import gc; gc.collect() + self.debug_collect() # gc.collect() is not enough in this test: + # we need to check and free the PyObject + # linked to the freestring object as well assert freed == ['x', 'x'] diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py index a7cb85590b..48ed8b3c28 100644 --- a/pypy/module/cpyext/test/test_ndarrayobject.py +++ b/pypy/module/cpyext/test/test_ndarrayobject.py @@ -80,7 +80,7 @@ class TestNDArrayObject(BaseApiTest): a0 = scalar(space) assert a0.get_scalar_value().value == 10. - a = api._PyArray_FromAny(a0, NULL, 0, 0, 0, NULL) + a = api._PyArray_FromAny(a0, None, 0, 0, 0, NULL) assert api._PyArray_NDIM(a) == 0 ptr = rffi.cast(rffi.DOUBLEP, api._PyArray_DATA(a)) @@ -88,10 +88,10 @@ class TestNDArrayObject(BaseApiTest): def test_FromAny(self, space, api): a = array(space, [10, 5, 3]) - assert api._PyArray_FromAny(a, NULL, 0, 0, 0, NULL) is a - assert api._PyArray_FromAny(a, NULL, 1, 4, 0, NULL) is a + assert api._PyArray_FromAny(a, None, 0, 0, 0, NULL) is a + assert api._PyArray_FromAny(a, None, 1, 4, 0, NULL) is a self.raises(space, api, ValueError, api._PyArray_FromAny, - a, NULL, 4, 5, 0, NULL) + a, None, 4, 5, 0, NULL) def test_FromObject(self, space, api): a = array(space, [10, 5, 3]) diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py index 3c36edd39b..ea435cb233 100644 --- a/pypy/module/cpyext/test/test_tupleobject.py +++ b/pypy/module/cpyext/test/test_tupleobject.py @@ -1,7 +1,9 @@ import py from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref +from pypy.module.cpyext.tupleobject import PyTupleObject from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from rpython.rtyper.lltypesystem import rffi, lltype @@ -10,38 +12,47 @@ class TestTupleObject(BaseApiTest): def test_tupleobject(self, space, api): assert not api.PyTuple_Check(space.w_None) assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1 - atuple = space.newtuple([0, 1, 'yay']) + atuple = space.newtuple([space.wrap(0), space.wrap(1), + space.wrap('yay')]) assert api.PyTuple_Size(atuple) == 3 - assert api.PyTuple_GET_SIZE(atuple) == 3 + #assert api.PyTuple_GET_SIZE(atuple) == 3 --- now a C macro raises(TypeError, api.PyTuple_Size(space.newlist([]))) api.PyErr_Clear() def test_tuple_resize(self, space, api): - py_tuple = api.PyTuple_New(3) + w_42 = space.wrap(42) ar = lltype.malloc(PyObjectP.TO, 1, flavor='raw') - ar[0] = rffi.cast(PyObject, make_ref(space, py_tuple)) + + py_tuple = api.PyTuple_New(3) + # inside py_tuple is an array of "PyObject *" items which each hold + # a reference + rffi.cast(PyTupleObject, py_tuple).c_ob_item[0] = make_ref(space, w_42) + ar[0] = py_tuple api._PyTuple_Resize(ar, 2) - py_tuple = from_ref(space, ar[0]) - assert space.int_w(space.len(py_tuple)) == 2 - + w_tuple = from_ref(space, ar[0]) + assert space.int_w(space.len(w_tuple)) == 2 + assert space.int_w(space.getitem(w_tuple, space.wrap(0))) == 42 + api.Py_DecRef(ar[0]) + + py_tuple = api.PyTuple_New(3) + rffi.cast(PyTupleObject, py_tuple).c_ob_item[0] = make_ref(space, w_42) + ar[0] = py_tuple api._PyTuple_Resize(ar, 10) - py_tuple = from_ref(space, ar[0]) - assert space.int_w(space.len(py_tuple)) == 10 - + w_tuple = from_ref(space, ar[0]) + assert space.int_w(space.len(w_tuple)) == 10 + assert space.int_w(space.getitem(w_tuple, space.wrap(0))) == 42 api.Py_DecRef(ar[0]) + lltype.free(ar, flavor='raw') def test_setitem(self, space, api): - atuple = space.newtuple([space.wrap(0), space.wrap("hello")]) - assert api.PyTuple_Size(atuple) == 2 - assert space.eq_w(space.getitem(atuple, space.wrap(0)), space.wrap(0)) - assert space.eq_w(space.getitem(atuple, space.wrap(1)), space.wrap("hello")) - w_obj = space.wrap(1) - api.Py_IncRef(w_obj) - api.PyTuple_SetItem(atuple, 1, w_obj) - assert api.PyTuple_Size(atuple) == 2 - assert space.eq_w(space.getitem(atuple, space.wrap(0)), space.wrap(0)) - assert space.eq_w(space.getitem(atuple, space.wrap(1)), space.wrap(1)) + py_tuple = api.PyTuple_New(2) + api.PyTuple_SetItem(py_tuple, 0, make_ref(space, space.wrap(42))) + api.PyTuple_SetItem(py_tuple, 1, make_ref(space, space.wrap(43))) + + w_tuple = from_ref(space, py_tuple) + assert space.eq_w(w_tuple, space.newtuple([space.wrap(42), + space.wrap(43)])) def test_getslice(self, space, api): w_tuple = space.newtuple([space.wrap(i) for i in range(10)]) @@ -49,3 +60,71 @@ class TestTupleObject(BaseApiTest): assert space.eq_w(w_slice, space.newtuple([space.wrap(i) for i in range(3, 7)])) + +class AppTestTuple(AppTestCpythonExtensionBase): + def test_refcounts(self): + module = self.import_extension('foo', [ + ("run", "METH_NOARGS", + """ + PyObject *item = PyTuple_New(0); + PyObject *t = PyTuple_New(1); + if (t->ob_refcnt != 1 || item->ob_refcnt != 1) { + PyErr_SetString(PyExc_SystemError, "bad initial refcnt"); + return NULL; + } + + PyTuple_SetItem(t, 0, item); + if (t->ob_refcnt != 1) { + PyErr_SetString(PyExc_SystemError, "SetItem: t refcnt != 1"); + return NULL; + } + if (item->ob_refcnt != 1) { + PyErr_SetString(PyExc_SystemError, "SetItem: item refcnt != 1"); + return NULL; + } + + if (PyTuple_GetItem(t, 0) != item || + PyTuple_GetItem(t, 0) != item) { + PyErr_SetString(PyExc_SystemError, "GetItem: bogus item"); + return NULL; + } + + if (t->ob_refcnt != 1) { + PyErr_SetString(PyExc_SystemError, "GetItem: t refcnt != 1"); + return NULL; + } + if (item->ob_refcnt != 1) { + PyErr_SetString(PyExc_SystemError, "GetItem: item refcnt != 1"); + return NULL; + } + return t; + """), + ]) + x = module.run() + assert x == ((),) + + def test_refcounts_more(self): + module = self.import_extension('foo', [ + ("run", "METH_NOARGS", + """ + long prev, next; + PyObject *t = PyTuple_New(1); + prev = Py_True->ob_refcnt; + Py_INCREF(Py_True); + PyTuple_SetItem(t, 0, Py_True); + if (Py_True->ob_refcnt != prev + 1) { + PyErr_SetString(PyExc_SystemError, + "SetItem: Py_True refcnt != prev + 1"); + return NULL; + } + Py_DECREF(t); + if (Py_True->ob_refcnt != prev) { + PyErr_SetString(PyExc_SystemError, + "after: Py_True refcnt != prev"); + return NULL; + } + Py_INCREF(Py_None); + return Py_None; + """), + ]) + module.run() diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py index cf3a030b0d..23008cb2fb 100644 --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -744,3 +744,25 @@ class AppTestSlots(AppTestCpythonExtensionBase): module = self.import_module(name='foo3') print('calling module.Type()...') module.Type("X", (object,), {}) + + def test_app_subclass_of_c_type(self): + module = self.import_module(name='foo') + size = module.size_of_instances(module.fooType) + class f1(object): + pass + class f2(module.fooType): + pass + class bar(f1, f2): + pass + assert bar.__base__ is f2 + assert module.size_of_instances(bar) == size + + def test_app_cant_subclass_two_types(self): + module = self.import_module(name='foo') + try: + class bar(module.fooType, module.Property): + pass + except TypeError as e: + assert str(e) == 'instance layout conflicts in multiple inheritance' + else: + raise AssertionError("did not get TypeError!") diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py index 0ae2491c0a..31ffdc67a2 100644 --- a/pypy/module/cpyext/test/test_unicodeobject.py +++ b/pypy/module/cpyext/test/test_unicodeobject.py @@ -24,7 +24,7 @@ class AppTestUnicodeObject(AppTestCpythonExtensionBase): if(PyUnicode_GetSize(s) == 11) { result = 1; } - if(s->ob_type->tp_basicsize != sizeof(void*)*4) + if(s->ob_type->tp_basicsize != sizeof(void*)*5) result = 0; Py_DECREF(s); return PyBool_FromLong(result); diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py index de354f9b34..65e2dfff1f 100644 --- a/pypy/module/cpyext/test/test_version.py +++ b/pypy/module/cpyext/test/test_version.py @@ -23,6 +23,7 @@ class AppTestVersion(AppTestCpythonExtensionBase): PyModule_AddIntConstant(m, "py_minor_version", PY_MINOR_VERSION); PyModule_AddIntConstant(m, "py_micro_version", PY_MICRO_VERSION); PyModule_AddStringConstant(m, "pypy_version", PYPY_VERSION); + PyModule_AddIntConstant(m, "pypy_version_num", PYPY_VERSION_NUM); } """ module = self.import_module(name='foo', init=init) @@ -35,3 +36,6 @@ class AppTestVersion(AppTestCpythonExtensionBase): if v.releaselevel != 'final': s += '-%s%d' % (v[3], v[4]) assert module.pypy_version == s + assert module.pypy_version_num == ((v[0] << 24) | + (v[1] << 16) | + (v[2] << 8)) diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py index ef23ba9803..938fb50b91 100644 --- a/pypy/module/cpyext/tupleobject.py +++ b/pypy/module/cpyext/tupleobject.py @@ -1,59 +1,166 @@ from pypy.interpreter.error import OperationError from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.api import (cpython_api, Py_ssize_t, CANNOT_FAIL, - build_type_checkers) + build_type_checkers, PyObjectFields, + cpython_struct, bootstrap_function) from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, Py_DecRef, - borrow_from, make_ref, from_ref) + make_ref, from_ref, decref, + track_reference, make_typedescr, get_typedescr) from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall from pypy.objspace.std.tupleobject import W_TupleObject +## +## Implementation of PyTupleObject +## =============================== +## +## Similar to stringobject.py. The reason is only the existance of +## W_SpecialisedTupleObject_ii and W_SpecialisedTupleObject_ff. +## These two PyPy classes implement getitem() by returning a freshly +## constructed W_IntObject or W_FloatObject. This is not compatible +## with PyTuple_GetItem, which returns a borrowed reference. +## +## So we use this more advanced (but also likely faster) solution: +## tuple_attach makes a real PyTupleObject with an array of N +## 'PyObject *', which are created immediately and own a reference. +## Then the macro PyTuple_GET_ITEM can be implemented like CPython. +## + +PyTupleObjectStruct = lltype.ForwardReference() +PyTupleObject = lltype.Ptr(PyTupleObjectStruct) +ObjectItems = rffi.CArray(PyObject) +PyTupleObjectFields = PyObjectFields + \ + (("ob_size", Py_ssize_t), ("ob_item", lltype.Ptr(ObjectItems))) +cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct) + +@bootstrap_function +def init_stringobject(space): + "Type description of PyTupleObject" + make_typedescr(space.w_tuple.layout.typedef, + basestruct=PyTupleObject.TO, + attach=tuple_attach, + dealloc=tuple_dealloc, + realize=tuple_realize) + PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple") +def tuple_check_ref(space, ref): + w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type)) + return (w_type is space.w_tuple or + space.is_true(space.issubtype(w_type, space.w_tuple))) + +def new_empty_tuple(space, length): + """ + Allocate a PyTupleObject and its array of PyObject *, but without a + corresponding interpreter object. The array may be mutated, until + tuple_realize() is called. Refcount of the result is 1. + """ + typedescr = get_typedescr(space.w_tuple.layout.typedef) + py_obj = typedescr.allocate(space, space.w_tuple) + py_tup = rffi.cast(PyTupleObject, py_obj) + + py_tup.c_ob_item = lltype.malloc(ObjectItems, length, + flavor='raw', zero=True) + py_tup.c_ob_size = length + return py_tup + +def tuple_attach(space, py_obj, w_obj): + """ + Fills a newly allocated PyTupleObject with the given tuple object. The + buffer must not be modified. + """ + items_w = space.fixedview(w_obj) + l = len(items_w) + p = lltype.malloc(ObjectItems, l, flavor='raw') + i = 0 + try: + while i < l: + p[i] = make_ref(space, items_w[i]) + i += 1 + except: + while i > 0: + i -= 1 + decref(space, p[i]) + lltype.free(p, flavor='raw') + raise + py_tup = rffi.cast(PyTupleObject, py_obj) + py_tup.c_ob_size = l + py_tup.c_ob_item = p + +def tuple_realize(space, py_obj): + """ + Creates the tuple in the interpreter. The PyTupleObject must not + be modified after this call. + """ + py_tup = rffi.cast(PyTupleObject, py_obj) + l = py_tup.c_ob_size + p = py_tup.c_ob_item + items_w = [None] * l + for i in range(l): + items_w[i] = from_ref(space, p[i]) + w_obj = space.newtuple(items_w) + track_reference(space, py_obj, w_obj) + return w_obj + +@cpython_api([PyObject], lltype.Void, header=None) +def tuple_dealloc(space, py_obj): + """Frees allocated PyTupleObject resources. + """ + py_tup = rffi.cast(PyTupleObject, py_obj) + p = py_tup.c_ob_item + if p: + for i in range(py_tup.c_ob_size): + decref(space, p[i]) + lltype.free(p, flavor="raw") + from pypy.module.cpyext.object import PyObject_dealloc + PyObject_dealloc(space, py_obj) + +#_______________________________________________________________________ + @cpython_api([Py_ssize_t], PyObject) def PyTuple_New(space, size): - return W_TupleObject([space.w_None] * size) + return rffi.cast(PyObject, new_empty_tuple(space, size)) @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1) -def PyTuple_SetItem(space, w_t, pos, w_obj): - if not PyTuple_Check(space, w_t): - # XXX this should also steal a reference, test it!!! +def PyTuple_SetItem(space, ref, index, py_obj): + # XXX this will not complain when changing tuples that have + # already been realized as a W_TupleObject, but won't update the + # W_TupleObject + if not tuple_check_ref(space, ref): + decref(space, py_obj) PyErr_BadInternalCall(space) - _setitem_tuple(w_t, pos, w_obj) - Py_DecRef(space, w_obj) # SetItem steals a reference! + ref = rffi.cast(PyTupleObject, ref) + size = ref.c_ob_size + if index < 0 or index >= size: + raise OperationError(space.w_IndexError, + space.wrap("tuple assignment index out of range")) + old_ref = ref.c_ob_item[index] + ref.c_ob_item[index] = py_obj # consumes a reference + if old_ref: + decref(space, old_ref) return 0 -def _setitem_tuple(w_t, pos, w_obj): - # this function checks that w_t is really a W_TupleObject. It - # should only ever be called with a freshly built tuple from - # PyTuple_New(), which always return a W_TupleObject, even if there - # are also other implementations of tuples. - assert isinstance(w_t, W_TupleObject) - w_t.wrappeditems[pos] = w_obj - -@cpython_api([PyObject, Py_ssize_t], PyObject) -def PyTuple_GetItem(space, w_t, pos): - if not PyTuple_Check(space, w_t): +@cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True) +def PyTuple_GetItem(space, ref, index): + if not tuple_check_ref(space, ref): PyErr_BadInternalCall(space) - w_obj = space.getitem(w_t, space.wrap(pos)) - return borrow_from(w_t, w_obj) - -@cpython_api([PyObject], Py_ssize_t, error=CANNOT_FAIL) -def PyTuple_GET_SIZE(space, w_t): - """Return the size of the tuple p, which must be non-NULL and point to a tuple; - no error checking is performed. """ - return space.int_w(space.len(w_t)) + ref = rffi.cast(PyTupleObject, ref) + size = ref.c_ob_size + if index < 0 or index >= size: + raise OperationError(space.w_IndexError, + space.wrap("tuple index out of range")) + return ref.c_ob_item[index] # borrowed ref @cpython_api([PyObject], Py_ssize_t, error=-1) def PyTuple_Size(space, ref): """Take a pointer to a tuple object, and return the size of that tuple.""" - if not PyTuple_Check(space, ref): - raise OperationError(space.w_TypeError, - space.wrap("expected tuple object")) - return PyTuple_GET_SIZE(space, ref) + if not tuple_check_ref(space, ref): + PyErr_BadInternalCall(space) + ref = rffi.cast(PyTupleObject, ref) + return ref.c_ob_size @cpython_api([PyObjectP, Py_ssize_t], rffi.INT_real, error=-1) -def _PyTuple_Resize(space, ref, newsize): +def _PyTuple_Resize(space, p_ref, newsize): """Can be used to resize a tuple. newsize will be the new length of the tuple. Because tuples are supposed to be immutable, this should only be used if there is only one reference to the object. Do not use this if the tuple may already @@ -64,19 +171,27 @@ def _PyTuple_Resize(space, ref, newsize): this function. If the object referenced by *p is replaced, the original *p is destroyed. On failure, returns -1 and sets *p to NULL, and raises MemoryError or SystemError.""" - py_tuple = from_ref(space, ref[0]) - if not PyTuple_Check(space, py_tuple): + ref = p_ref[0] + if not tuple_check_ref(space, ref): PyErr_BadInternalCall(space) - py_newtuple = PyTuple_New(space, newsize) - - to_cp = newsize - oldsize = space.int_w(space.len(py_tuple)) - if oldsize < newsize: - to_cp = oldsize - for i in range(to_cp): - _setitem_tuple(py_newtuple, i, space.getitem(py_tuple, space.wrap(i))) - Py_DecRef(space, ref[0]) - ref[0] = make_ref(space, py_newtuple) + ref = rffi.cast(PyTupleObject, ref) + oldsize = ref.c_ob_size + oldp = ref.c_ob_item + newp = lltype.malloc(ObjectItems, newsize, zero=True, flavor='raw') + try: + if oldsize < newsize: + to_cp = oldsize + else: + to_cp = newsize + for i in range(to_cp): + newp[i] = oldp[i] + except: + lltype.free(newp, flavor='raw') + raise + ref.c_ob_item = newp + ref.c_ob_size = newsize + lltype.free(oldp, flavor='raw') + # in this version, p_ref[0] never needs to be updated return 0 @cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject) diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py index fda7de0778..e8fa4e9e00 100644 --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -15,13 +15,13 @@ from pypy.module.cpyext.api import ( cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL, - Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers) + Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers, StaticObjectBuilder) from pypy.module.cpyext.methodobject import ( PyDescr_NewWrapper, PyCFunction_NewEx, PyCFunction_typedef) from pypy.module.cpyext.modsupport import convert_method_defs from pypy.module.cpyext.pyobject import ( PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr, - track_reference, RefcountState, borrow_from, Py_DecRef) + track_reference, Py_DecRef, as_pyobj) from pypy.module.cpyext.slotdefs import ( slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function) from pypy.module.cpyext.state import State @@ -116,7 +116,7 @@ def convert_member_defs(space, dict_w, members, w_type): def update_all_slots(space, w_type, pto): # XXX fill slots in pto - typedef = w_type.instancetypedef + typedef = w_type.layout.typedef for method_name, slot_name, slot_names, slot_func in slotdefs_for_tp_slots: w_descr = w_type.lookup(method_name) if w_descr is None: @@ -235,6 +235,9 @@ def add_tp_new_wrapper(space, dict_w, pto): def inherit_special(space, pto, base_pto): # XXX missing: copy basicsize and flags in a magical way + # (minimally, if tp_basicsize is zero we copy it from the base) + if not pto.c_tp_basicsize: + pto.c_tp_basicsize = base_pto.c_tp_basicsize flags = rffi.cast(lltype.Signed, pto.c_tp_flags) base_object_pyo = make_ref(space, space.w_object) base_object_pto = rffi.cast(PyTypeObjectPtr, base_object_pyo) @@ -294,7 +297,7 @@ class W_PyCTypeObject(W_TypeObject): name = rffi.charp2str(pto.c_tp_name) W_TypeObject.__init__(self, space, name, - bases_w or [space.w_object], dict_w) + bases_w or [space.w_object], dict_w, force_new_layout=True) if not space.is_true(space.issubtype(self, space.w_type)): self.flag_cpytype = True self.flag_heaptype = False @@ -303,7 +306,7 @@ class W_PyCTypeObject(W_TypeObject): @bootstrap_function def init_typeobject(space): - make_typedescr(space.w_type.instancetypedef, + make_typedescr(space.w_type.layout.typedef, basestruct=PyTypeObject, alloc=type_alloc, attach=type_attach, @@ -337,7 +340,7 @@ def str_segcount(space, w_obj, ref): @cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, header=None, error=-1) def str_getreadbuffer(space, w_str, segment, ref): - from pypy.module.cpyext.stringobject import PyString_AsString + from pypy.module.cpyext.bytesobject import PyString_AsString if segment != 0: raise OperationError(space.w_SystemError, space.wrap ("accessing non-existent string segment")) @@ -350,7 +353,7 @@ def str_getreadbuffer(space, w_str, segment, ref): @cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, header=None, error=-1) def str_getcharbuffer(space, w_str, segment, ref): - from pypy.module.cpyext.stringobject import PyString_AsString + from pypy.module.cpyext.bytesobject import PyString_AsString if segment != 0: raise OperationError(space.w_SystemError, space.wrap ("accessing non-existent string segment")) @@ -442,7 +445,7 @@ def type_attach(space, py_obj, w_type): pto = rffi.cast(PyTypeObjectPtr, py_obj) - typedescr = get_typedescr(w_type.instancetypedef) + typedescr = get_typedescr(w_type.layout.typedef) # dealloc pto.c_tp_dealloc = typedescr.get_dealloc(space) @@ -460,7 +463,7 @@ def type_attach(space, py_obj, w_type): w_typename = space.getattr(w_type, space.wrap('__name__')) heaptype = rffi.cast(PyHeapTypeObject, pto) heaptype.c_ht_name = make_ref(space, w_typename) - from pypy.module.cpyext.stringobject import PyString_AsString + from pypy.module.cpyext.bytesobject import PyString_AsString pto.c_tp_name = PyString_AsString(space, heaptype.c_ht_name) else: pto.c_tp_name = rffi.str2charp(w_type.name) @@ -471,8 +474,9 @@ def type_attach(space, py_obj, w_type): w_base = best_base(space, w_type.bases_w) pto.c_tp_base = rffi.cast(PyTypeObjectPtr, make_ref(space, w_base)) - if hasattr(space, '_cpyext_type_init'): - space._cpyext_type_init.append((pto, w_type)) + builder = space.fromcache(StaticObjectBuilder) + if builder.cpyext_type_init is not None: + builder.cpyext_type_init.append((pto, w_type)) else: finish_type_1(space, pto) finish_type_2(space, pto, w_type) @@ -502,6 +506,7 @@ def PyType_Ready(space, pto): def type_realize(space, py_obj): pto = rffi.cast(PyTypeObjectPtr, py_obj) + assert pto.c_tp_flags & Py_TPFLAGS_READY == 0 assert pto.c_tp_flags & Py_TPFLAGS_READYING == 0 pto.c_tp_flags |= Py_TPFLAGS_READYING try: @@ -512,13 +517,13 @@ def type_realize(space, py_obj): return w_obj def solid_base(space, w_type): - typedef = w_type.instancetypedef + typedef = w_type.layout.typedef return space.gettypeobject(typedef) def best_base(space, bases_w): if not bases_w: return None - return find_best_base(space, bases_w) + return find_best_base(bases_w) def inherit_slots(space, pto, w_base): # XXX missing: nearly everything @@ -553,8 +558,7 @@ def _type_realize(space, py_obj): if not py_type.c_tp_base: # borrowed reference, but w_object is unlikely to disappear - base = make_ref(space, space.w_object) - Py_DecRef(space, base) + base = as_pyobj(space, space.w_object) py_type.c_tp_base = rffi.cast(PyTypeObjectPtr, base) finish_type_1(space, py_type) @@ -568,9 +572,6 @@ def _type_realize(space, py_obj): finish_type_2(space, py_type, w_obj) - state = space.fromcache(RefcountState) - state.non_heaptypes_w.append(w_obj) - return w_obj def finish_type_1(space, pto): @@ -636,7 +637,8 @@ def PyType_GenericNew(space, type, w_args, w_kwds): return generic_cpy_call( space, type.c_tp_alloc, type, 0) -@cpython_api([PyTypeObjectPtr, PyObject], PyObject, error=CANNOT_FAIL) +@cpython_api([PyTypeObjectPtr, PyObject], PyObject, error=CANNOT_FAIL, + result_borrowed=True) def _PyType_Lookup(space, type, w_name): """Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception!""" @@ -647,7 +649,9 @@ def _PyType_Lookup(space, type, w_name): return None name = space.str_w(w_name) w_obj = w_type.lookup(name) - return borrow_from(w_type, w_obj) + # this assumes that w_obj is not dynamically created, but will stay alive + # until w_type is modified or dies. Assuming this, we return a borrowed ref + return w_obj @cpython_api([PyTypeObjectPtr], lltype.Void) def PyType_Modified(space, w_obj): diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py index 5d0f21d744..fb0d39eea0 100644 --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -9,7 +9,7 @@ from pypy.module.cpyext.pyerrors import PyErr_BadArgument from pypy.module.cpyext.pyobject import ( PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference, make_typedescr, get_typedescr) -from pypy.module.cpyext.stringobject import PyString_Check +from pypy.module.cpyext.bytesobject import PyString_Check from pypy.module.sys.interp_encoding import setdefaultencoding from pypy.module._codecs.interp_codecs import CodecState from pypy.objspace.std import unicodeobject @@ -17,7 +17,7 @@ from rpython.rlib import rstring, runicode from rpython.tool.sourcetools import func_renamer import sys -## See comment in stringobject.py. +## See comment in bytesobject.py. PyUnicodeObjectStruct = lltype.ForwardReference() PyUnicodeObject = lltype.Ptr(PyUnicodeObjectStruct) @@ -27,7 +27,7 @@ cpython_struct("PyUnicodeObject", PyUnicodeObjectFields, PyUnicodeObjectStruct) @bootstrap_function def init_unicodeobject(space): - make_typedescr(space.w_unicode.instancetypedef, + make_typedescr(space.w_unicode.layout.typedef, basestruct=PyUnicodeObject.TO, attach=unicode_attach, dealloc=unicode_dealloc, @@ -44,11 +44,11 @@ Py_UNICODE = lltype.UniChar def new_empty_unicode(space, length): """ - Allocatse a PyUnicodeObject and its buffer, but without a corresponding + Allocate a PyUnicodeObject and its buffer, but without a corresponding interpreter object. The buffer may be mutated, until unicode_realize() is - called. + called. Refcount of the result is 1. """ - typedescr = get_typedescr(space.w_unicode.instancetypedef) + typedescr = get_typedescr(space.w_unicode.layout.typedef) py_obj = typedescr.allocate(space, space.w_unicode) py_uni = rffi.cast(PyUnicodeObject, py_obj) diff --git a/pypy/module/cpyext/weakrefobject.py b/pypy/module/cpyext/weakrefobject.py index 4c5389719b..fe93dd337b 100644 --- a/pypy/module/cpyext/weakrefobject.py +++ b/pypy/module/cpyext/weakrefobject.py @@ -1,5 +1,5 @@ from pypy.module.cpyext.api import cpython_api -from pypy.module.cpyext.pyobject import PyObject, borrow_from +from pypy.module.cpyext.pyobject import PyObject from pypy.module._weakref.interp__weakref import W_Weakref, proxy @cpython_api([PyObject, PyObject], PyObject) @@ -30,24 +30,26 @@ def PyWeakref_NewProxy(space, w_obj, w_callback): """ return proxy(space, w_obj, w_callback) -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyWeakref_GetObject(space, w_ref): """Return the referenced object from a weak reference. If the referent is no longer live, returns None. This function returns a borrowed reference. """ - return PyWeakref_GET_OBJECT(space, w_ref) + return space.call_function(w_ref) # borrowed ref -@cpython_api([PyObject], PyObject) +@cpython_api([PyObject], PyObject, result_borrowed=True) def PyWeakref_GET_OBJECT(space, w_ref): """Similar to PyWeakref_GetObject(), but implemented as a macro that does no error checking. """ - return borrow_from(w_ref, space.call_function(w_ref)) + return space.call_function(w_ref) # borrowed ref @cpython_api([PyObject], PyObject) def PyWeakref_LockObject(space, w_ref): """Return the referenced object from a weak reference. If the referent is no longer live, returns None. This function returns a new reference. + + (A PyPy extension that may not be useful any more: use + PyWeakref_GetObject() and Py_INCREF().) """ return space.call_function(w_ref) - diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py index 67aa8c96ad..1b2fcbaa30 100644 --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -38,7 +38,7 @@ SO = '.pyd' if _WIN32 else '.so' # and cffi so's. If we do have to update it, we'd likely need a way to # split the two usages again. #DEFAULT_SOABI = 'pypy-%d%d' % PYPY_VERSION[:2] -DEFAULT_SOABI = 'pypy-26' +DEFAULT_SOABI = 'pypy-41' @specialize.memo() def get_so_extension(space): @@ -85,7 +85,7 @@ def find_modtype(space, filepart): # The "imp" module does not respect this, and is allowed to find # lone .pyc files. # check the .pyc file - if space.config.objspace.usepycfiles and space.config.objspace.lonepycfiles: + if space.config.objspace.lonepycfiles: pycfile = filepart + ".pyc" if file_exists(pycfile): # existing .pyc file @@ -888,17 +888,11 @@ def load_source_module(space, w_modulename, w_mod, pathname, source, fd, """ w = space.wrap - if space.config.objspace.usepycfiles: - src_stat = os.fstat(fd) - cpathname = pathname + 'c' - mtime = int(src_stat[stat.ST_MTIME]) - mode = src_stat[stat.ST_MODE] - stream = check_compiled_module(space, cpathname, mtime) - else: - cpathname = None - mtime = 0 - mode = 0 - stream = None + src_stat = os.fstat(fd) + cpathname = pathname + 'c' + mtime = int(src_stat[stat.ST_MTIME]) + mode = src_stat[stat.ST_MODE] + stream = check_compiled_module(space, cpathname, mtime) if stream: # existing and up-to-date .pyc file @@ -913,7 +907,7 @@ def load_source_module(space, w_modulename, w_mod, pathname, source, fd, else: code_w = parse_source_module(space, pathname, source) - if space.config.objspace.usepycfiles and write_pyc: + if write_pyc: if not space.is_true(space.sys.get('dont_write_bytecode')): write_compiled_module(space, code_w, cpathname, mode, mtime) diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py index ae94b8e35b..134eba3502 100644 --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -98,6 +98,10 @@ def setup_directory_structure(space): 'a=5\nb=6\rc="""hello\r\nworld"""\r', mode='wb') p.join('mod.py').write( 'a=15\nb=16\rc="""foo\r\nbar"""\r', mode='wb') + setuppkg("test_bytecode", + a = '', + b = '', + c = '') # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") @@ -119,7 +123,7 @@ def setup_directory_structure(space): stream.try_to_find_file_descriptor()) finally: stream.close() - if space.config.objspace.usepycfiles: + if not space.config.translation.sandbox: # also create a lone .pyc file p.join('lone.pyc').write(p.join('x.pyc').read(mode='rb'), mode='wb') @@ -146,6 +150,8 @@ def _setup(space): """) def _teardown(space, w_saved_modules): + p = udir.join('impsubdir') + p.remove() space.appexec([w_saved_modules], """ ((saved_path, saved_modules)): import sys @@ -646,11 +652,13 @@ class AppTestImport: # one in sys.path. import sys assert '_md5' not in sys.modules - import _md5 - assert hasattr(_md5, 'hello_world') - assert not hasattr(_md5, 'count') - assert '(built-in)' not in repr(_md5) - del sys.modules['_md5'] + try: + import _md5 + assert hasattr(_md5, 'hello_world') + assert not hasattr(_md5, 'digest_size') + assert '(built-in)' not in repr(_md5) + finally: + sys.modules.pop('_md5', None) def test_shadow_extension_2(self): if self.runappdirect: skip("hard to test: module is already imported") @@ -669,7 +677,7 @@ class AppTestImport: assert '(built-in)' in repr(_md5) finally: sys.path.insert(0, sys.path.pop()) - del sys.modules['_md5'] + sys.modules.pop('_md5', None) def test_invalid_pathname(self): import imp @@ -1061,12 +1069,12 @@ def test_PYTHONPATH_takes_precedence(space): py.test.skip("unresolved issues with win32 shell quoting rules") from pypy.interpreter.test.test_zpy import pypypath extrapath = udir.ensure("pythonpath", dir=1) - extrapath.join("urllib.py").write("print 42\n") + extrapath.join("sched.py").write("print 42\n") old = os.environ.get('PYTHONPATH', None) oldlang = os.environ.pop('LANG', None) try: os.environ['PYTHONPATH'] = str(extrapath) - output = py.process.cmdexec('''"%s" "%s" -c "import urllib"''' % + output = py.process.cmdexec('''"%s" "%s" -c "import sched"''' % (sys.executable, pypypath)) assert output.strip() == '42' finally: @@ -1342,15 +1350,56 @@ class AppTestPyPyExtension(object): assert isinstance(importer, zipimport.zipimporter) -class AppTestNoPycFile(object): +class AppTestWriteBytecode(object): spaceconfig = { - "objspace.usepycfiles": False, - "objspace.lonepycfiles": False + "translation.sandbox": False } + + def setup_class(cls): + cls.saved_modules = _setup(cls.space) + sandbox = cls.spaceconfig['translation.sandbox'] + cls.w_sandbox = cls.space.wrap(sandbox) + + def teardown_class(cls): + _teardown(cls.space, cls.saved_modules) + cls.space.appexec([], """ + (): + import sys + sys.dont_write_bytecode = False + """) + + def test_default(self): + import os.path + from test_bytecode import a + assert a.__file__.endswith('a.py') + assert os.path.exists(a.__file__ + 'c') == (not self.sandbox) + + def test_write_bytecode(self): + import os.path + import sys + sys.dont_write_bytecode = False + from test_bytecode import b + assert b.__file__.endswith('b.py') + assert os.path.exists(b.__file__ + 'c') + + def test_dont_write_bytecode(self): + import os.path + import sys + sys.dont_write_bytecode = True + from test_bytecode import c + assert c.__file__.endswith('c.py') + assert not os.path.exists(c.__file__ + 'c') + + +class AppTestWriteBytecodeSandbox(AppTestWriteBytecode): + spaceconfig = { + "translation.sandbox": True + } + + +class _AppTestLonePycFileBase(object): def setup_class(cls): - usepycfiles = cls.spaceconfig['objspace.usepycfiles'] lonepycfiles = cls.spaceconfig['objspace.lonepycfiles'] - cls.w_usepycfiles = cls.space.wrap(usepycfiles) cls.w_lonepycfiles = cls.space.wrap(lonepycfiles) cls.saved_modules = _setup(cls.space) @@ -1359,10 +1408,7 @@ class AppTestNoPycFile(object): def test_import_possibly_from_pyc(self): from compiled import x - if self.usepycfiles: - assert x.__file__.endswith('x.pyc') - else: - assert x.__file__.endswith('x.py') + assert x.__file__.endswith('x.pyc') try: from compiled import lone except ImportError: @@ -1371,15 +1417,13 @@ class AppTestNoPycFile(object): assert self.lonepycfiles, "should not have found 'lone.pyc'" assert lone.__file__.endswith('lone.pyc') -class AppTestNoLonePycFile(AppTestNoPycFile): +class AppTestNoLonePycFile(_AppTestLonePycFileBase): spaceconfig = { - "objspace.usepycfiles": True, "objspace.lonepycfiles": False } -class AppTestLonePycFile(AppTestNoPycFile): +class AppTestLonePycFile(_AppTestLonePycFileBase): spaceconfig = { - "objspace.usepycfiles": True, "objspace.lonepycfiles": True } diff --git a/pypy/module/micronumpy/compile.py b/pypy/module/micronumpy/compile.py index 1de8307610..18b50a6746 100644 --- a/pypy/module/micronumpy/compile.py +++ b/pypy/module/micronumpy/compile.py @@ -196,6 +196,10 @@ class FakeSpace(ObjSpace): def newfloat(self, f): return self.float(f) + def newslice(self, start, stop, step): + return SliceObject(self.int_w(start), self.int_w(stop), + self.int_w(step)) + def le(self, w_obj1, w_obj2): assert isinstance(w_obj1, boxes.W_GenericBox) assert isinstance(w_obj2, boxes.W_GenericBox) diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py index d62f7a398c..02e82c517f 100644 --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -12,8 +12,8 @@ from pypy.module.micronumpy.base import convert_to_array, W_NDimArray, \ ArrayArgumentException, W_NumpyObject from pypy.module.micronumpy.iterators import ArrayIter from pypy.module.micronumpy.strides import ( - IntegerChunk, SliceChunk, NewAxisChunk, EllipsisChunk, new_view, - calc_strides, calc_new_strides, shape_agreement, + IntegerChunk, SliceChunk, NewAxisChunk, EllipsisChunk, BooleanChunk, + new_view, calc_strides, calc_new_strides, shape_agreement, calculate_broadcast_strides, calc_backstrides, calc_start, is_c_contiguous, is_f_contiguous) from rpython.rlib.objectmodel import keepalive_until_here @@ -236,6 +236,7 @@ class BaseConcreteArray(object): @jit.unroll_safe def _prepare_slice_args(self, space, w_idx): + from pypy.module.micronumpy import boxes if space.isinstance_w(w_idx, space.w_str): raise oefmt(space.w_IndexError, "only integers, slices (`:`), " "ellipsis (`...`), numpy.newaxis (`None`) and integer or " @@ -258,6 +259,7 @@ class BaseConcreteArray(object): result = [] i = 0 has_ellipsis = False + has_filter = False for w_item in space.fixedview(w_idx): if space.is_w(w_item, space.w_Ellipsis): if has_ellipsis: @@ -272,6 +274,16 @@ class BaseConcreteArray(object): elif space.isinstance_w(w_item, space.w_slice): result.append(SliceChunk(w_item)) i += 1 + elif isinstance(w_item, W_NDimArray) and w_item.get_dtype().is_bool(): + if has_filter: + # in CNumPy, the support for this is incomplete + raise oefmt(space.w_ValueError, + "an index can only have a single boolean mask; " + "use np.take or create a sinlge mask array") + has_filter = True + result.append(BooleanChunk(w_item)) + elif isinstance(w_item, boxes.W_GenericBox): + result.append(IntegerChunk(w_item.descr_int(space))) else: result.append(IntegerChunk(w_item)) i += 1 diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py index 5c9c43505e..0d784bd43a 100644 --- a/pypy/module/micronumpy/ndarray.py +++ b/pypy/module/micronumpy/ndarray.py @@ -107,8 +107,9 @@ class __extend__(W_NDimArray): arr = W_NDimArray(self.implementation.transpose(self, None)) return space.wrap(loop.tostring(space, arr)) - def getitem_filter(self, space, arr): - if arr.ndims() > 1 and arr.get_shape() != self.get_shape(): + def getitem_filter(self, space, arr, axis=0): + shape = self.get_shape() + if arr.ndims() > 1 and arr.get_shape() != shape: raise OperationError(space.w_IndexError, space.wrap( "boolean index array should have 1 dimension")) if arr.get_size() > self.get_size(): @@ -116,14 +117,14 @@ class __extend__(W_NDimArray): "index out of range for array")) size = loop.count_all_true(arr) if arr.ndims() == 1: - if self.ndims() > 1 and arr.get_shape()[0] != self.get_shape()[0]: + if self.ndims() > 1 and arr.get_shape()[0] != shape[axis]: msg = ("boolean index did not match indexed array along" - " dimension 0; dimension is %d but corresponding" - " boolean dimension is %d" % (self.get_shape()[0], + " dimension %d; dimension is %d but corresponding" + " boolean dimension is %d" % (axis, shape[axis], arr.get_shape()[0])) #warning = space.gettypefor(support.W_VisibleDeprecationWarning) space.warn(space.wrap(msg), space.w_VisibleDeprecationWarning) - res_shape = [size] + self.get_shape()[1:] + res_shape = shape[:axis] + [size] + shape[axis+1:] else: res_shape = [size] w_res = W_NDimArray.from_shape(space, res_shape, self.get_dtype(), @@ -149,6 +150,8 @@ class __extend__(W_NDimArray): def _prepare_array_index(self, space, w_index): if isinstance(w_index, W_NDimArray): return [], w_index.get_shape(), w_index.get_shape(), [w_index] + if isinstance(w_index, boxes.W_GenericBox): + return [], [1], [1], [w_index] w_lst = space.listview(w_index) for w_item in w_lst: if not (space.isinstance_w(w_item, space.w_int) or space.isinstance_w(w_item, space.w_float)): @@ -162,7 +165,14 @@ class __extend__(W_NDimArray): arr_index_in_shape = False prefix = [] for i, w_item in enumerate(w_lst): - if (isinstance(w_item, W_NDimArray) or + if isinstance(w_item, W_NDimArray) and w_item.get_dtype().is_bool(): + if w_item.ndims() > 0: + indexes_w[i] = w_item + else: + raise oefmt(space.w_IndexError, + "in the future, 0-d boolean arrays will be " + "interpreted as a valid boolean index") + elif (isinstance(w_item, W_NDimArray) or space.isinstance_w(w_item, space.w_list)): w_item = convert_to_array(space, w_item) if shape is None: @@ -232,6 +242,8 @@ class __extend__(W_NDimArray): raise oefmt(space.w_IndexError, "in the future, 0-d boolean arrays will be " "interpreted as a valid boolean index") + elif isinstance(w_idx, boxes.W_GenericBox): + w_ret = self.getitem_array_int(space, w_idx) else: try: w_ret = self.implementation.descr_getitem(space, self, w_idx) diff --git a/pypy/module/micronumpy/strides.py b/pypy/module/micronumpy/strides.py index 02c13a4d90..acb3ff484e 100644 --- a/pypy/module/micronumpy/strides.py +++ b/pypy/module/micronumpy/strides.py @@ -77,14 +77,42 @@ class EllipsisChunk(BaseChunk): backstride = base_stride * max(0, base_length - 1) return 0, base_length, base_stride, backstride +class BooleanChunk(BaseChunk): + input_dim = 1 + out_dim = 1 + def __init__(self, w_idx): + self.w_idx = w_idx + + def compute(self, space, base_length, base_stride): + raise oefmt(space.w_NotImplementedError, 'cannot reach') def new_view(space, w_arr, chunks): arr = w_arr.implementation - r = calculate_slice_strides(space, arr.shape, arr.start, arr.get_strides(), - arr.get_backstrides(), chunks) + dim = -1 + for i, c in enumerate(chunks): + if isinstance(c, BooleanChunk): + dim = i + break + if dim >= 0: + # filter by axis dim + filtr = chunks[dim] + assert isinstance(filtr, BooleanChunk) + w_arr = w_arr.getitem_filter(space, filtr.w_idx, axis=dim) + arr = w_arr.implementation + chunks[dim] = SliceChunk(space.newslice(space.wrap(0), + space.wrap(-1), space.w_None)) + r = calculate_slice_strides(space, arr.shape, arr.start, + arr.get_strides(), arr.get_backstrides(), chunks) + else: + r = calculate_slice_strides(space, arr.shape, arr.start, + arr.get_strides(), arr.get_backstrides(), chunks) shape, start, strides, backstrides = r - return W_NDimArray.new_slice(space, start, strides[:], backstrides[:], + w_ret = W_NDimArray.new_slice(space, start, strides[:], backstrides[:], shape[:], arr, w_arr) + if dim == 0: + # Do not return a view + return w_ret.descr_copy(space, space.wrap(w_ret.get_order())) + return w_ret @jit.unroll_safe def _extend_shape(old_shape, chunks): @@ -127,7 +155,7 @@ def enumerate_chunks(chunks): jit.isconstant(len(chunks))) def calculate_slice_strides(space, shape, start, strides, backstrides, chunks): """ - Note: `chunks` must contain exactly one EllipsisChunk object. + Note: `chunks` can contain at most one EllipsisChunk object. """ size = 0 used_dims = 0 diff --git a/pypy/module/micronumpy/test/test_deprecations.py b/pypy/module/micronumpy/test/test_deprecations.py index 036252ee01..8d275e0bf0 100644 --- a/pypy/module/micronumpy/test/test_deprecations.py +++ b/pypy/module/micronumpy/test/test_deprecations.py @@ -24,7 +24,7 @@ class AppTestDeprecations(BaseNumpyAppTest): # boolean indexing matches the dims in index # to the first index.ndims in arr, not implemented in pypy yet raises(IndexError, arr.__getitem__, index) - raises(TypeError, arr.__getitem__, (slice(None), index)) + raises(IndexError, arr.__getitem__, (slice(None), index)) else: raises(np.VisibleDeprecationWarning, arr.__getitem__, index) raises(np.VisibleDeprecationWarning, arr.__getitem__, (slice(None), index)) diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py index aa5b3c6116..62c182a1bd 100644 --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -2573,6 +2573,25 @@ class AppTestNumArray(BaseNumpyAppTest): a[b] = np.array([[4.]]) assert (a == [[4., 4., 4.]]).all() + def test_indexing_by_boolean(self): + import numpy as np + a = np.arange(6).reshape(2,3) + assert (a[[True, False], :] == [[3, 4, 5], [0, 1, 2]]).all() + b = a[np.array([True, False]), :] + assert (b == [[0, 1, 2]]).all() + assert b.base is None + b = a[:, np.array([True, False, True])] + assert b.base is not None + b = a[np.array([True, False]), 0] + assert (b ==[0]).all() + + def test_scalar_indexing(self): + import numpy as np + a = np.arange(6).reshape(2,3) + i = np.dtype('int32').type(0) + assert (a[0] == a[i]).all() + + def test_ellipsis_indexing(self): import numpy as np import sys diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py index 2c8afba53e..a5f51a50b9 100644 --- a/pypy/module/pypyjit/test_pypy_c/test_string.py +++ b/pypy/module/pypyjit/test_pypy_c/test_string.py @@ -28,7 +28,6 @@ class TestString(BaseTestPyPyC): guard_true(i14, descr=...) guard_not_invalidated(descr=...) i16 = int_eq(i6, %d) - guard_false(i16, descr=...) i15 = int_mod(i6, i10) i17 = int_rshift(i15, %d) i18 = int_and(i10, i17) @@ -68,7 +67,6 @@ class TestString(BaseTestPyPyC): guard_true(i11, descr=...) guard_not_invalidated(descr=...) i13 = int_eq(i6, %d) # value provided below - guard_false(i13, descr=...) i15 = int_mod(i6, 10) i17 = int_rshift(i15, %d) # value provided below i18 = int_and(10, i17) @@ -144,43 +142,6 @@ class TestString(BaseTestPyPyC): jump(..., descr=...) """) - def test_getattr_promote(self): - def main(n): - class A(object): - def meth_a(self): - return 1 - def meth_b(self): - return 2 - a = A() - - l = ['a', 'b'] - s = 0 - for i in range(n): - name = 'meth_' + l[i & 1] - meth = getattr(a, name) # ID: getattr - s += meth() - return s - - log = self.run(main, [1000]) - assert log.result == main(1000) - loops = log.loops_by_filename(self.filepath) - assert len(loops) == 1 - for loop in loops: - assert loop.match_by_id('getattr',''' - guard_not_invalidated? - i32 = strlen(p31) - i34 = int_add(5, i32) - p35 = newstr(i34) - strsetitem(p35, 0, 109) - strsetitem(p35, 1, 101) - strsetitem(p35, 2, 116) - strsetitem(p35, 3, 104) - strsetitem(p35, 4, 95) - copystrcontent(p31, p35, 0, 5, i32) - i49 = call_i(ConstClass(_ll_2_str_eq_nonnull__rpy_stringPtr_rpy_stringPtr), p35, ConstPtr(ptr48), descr=<Calli [48] rr EF=0 OS=28>) - guard_value(i49, 1, descr=...) - ''') - def test_remove_duplicate_method_calls(self): def main(n): lst = [] diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py index 7b09adbeb5..9462a1b0d7 100644 --- a/pypy/module/sys/__init__.py +++ b/pypy/module/sys/__init__.py @@ -77,7 +77,7 @@ class Module(MixedModule): 'meta_path' : 'space.wrap([])', 'path_hooks' : 'space.wrap([])', 'path_importer_cache' : 'space.wrap({})', - 'dont_write_bytecode' : 'space.w_False', + 'dont_write_bytecode' : 'space.wrap(space.config.translation.sandbox)', 'getdefaultencoding' : 'interp_encoding.getdefaultencoding', 'setdefaultencoding' : 'interp_encoding.setdefaultencoding', diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py index 9dc556aabd..08eeed9bb5 100644 --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -70,11 +70,11 @@ def callstats(): return None copyright_str = """ -Copyright 2003-2014 PyPy development team. +Copyright 2003-2016 PyPy development team. All Rights Reserved. For further information, see <http://pypy.org> -Portions Copyright (c) 2001-2014 Python Software Foundation. +Portions Copyright (c) 2001-2016 Python Software Foundation. All Rights Reserved. Portions Copyright (c) 2000 BeOpen.com. diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py index bcb05e76b5..9e78ec99f9 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_zintegration.py @@ -12,7 +12,9 @@ if sys.version_info < (2, 7): def create_venv(name): tmpdir = udir.join(name) try: - subprocess.check_call(['virtualenv', '--distribute', + subprocess.check_call(['virtualenv', + #'--never-download', <= could be added, but causes failures + # in random cases on random machines '-p', os.path.abspath(sys.executable), str(tmpdir)]) except OSError as e: diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c index 1328d1a628..abac15338a 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c @@ -1,3 +1,4 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> extern int add1(int, int); diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c index 96208436ec..3a10a42c19 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add2-test.c @@ -1,3 +1,4 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> extern int add1(int, int); diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c index cd29b790ff..2b89ee7e00 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add_recursive-test.c @@ -1,3 +1,4 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> #ifdef _MSC_VER diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c index 2931186c3f..66bbeb51b3 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/perf-test.c @@ -1,10 +1,12 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> #include <assert.h> #include <sys/time.h> #ifdef PTEST_USE_THREAD # include <pthread.h> -# include <semaphore.h> -static sem_t done; +static pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond1 = PTHREAD_COND_INITIALIZER; +static int remaining; #endif @@ -54,8 +56,11 @@ static void *start_routine(void *arg) printf("time per call: %.3g\n", t); #ifdef PTEST_USE_THREAD - int status = sem_post(&done); - assert(status == 0); + pthread_mutex_lock(&mutex1); + remaining -= 1; + if (!remaining) + pthread_cond_signal(&cond1); + pthread_mutex_unlock(&mutex1); #endif return arg; @@ -68,19 +73,19 @@ int main(void) start_routine(0); #else pthread_t th; - int i, status = sem_init(&done, 0, 0); - assert(status == 0); + int i, status; add1(0, 0); /* this is the main thread */ + remaining = PTEST_USE_THREAD; for (i = 0; i < PTEST_USE_THREAD; i++) { status = pthread_create(&th, NULL, start_routine, NULL); assert(status == 0); } - for (i = 0; i < PTEST_USE_THREAD; i++) { - status = sem_wait(&done); - assert(status == 0); - } + pthread_mutex_lock(&mutex1); + while (remaining) + pthread_cond_wait(&cond1, &mutex1); + pthread_mutex_unlock(&mutex1); #endif return 0; } diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py index 2ccd8a7671..1969e27aa5 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py @@ -28,11 +28,14 @@ def check_lib_python_found(tmpdir): def prefix_pythonpath(): cffi_base = os.path.dirname(os.path.dirname(local_dir)) - pythonpath = os.environ.get('PYTHONPATH', '').split(os.pathsep) + pythonpath = org_env.get('PYTHONPATH', '').split(os.pathsep) if cffi_base not in pythonpath: pythonpath.insert(0, cffi_base) return os.pathsep.join(pythonpath) +def setup_module(mod): + mod.org_env = os.environ.copy() + class EmbeddingTests: _compiled_modules = {} @@ -46,14 +49,12 @@ class EmbeddingTests: def get_path(self): return str(self._path.ensure(dir=1)) - def _run_base(self, args, env_extra={}, **kwds): - print('RUNNING:', args, env_extra, kwds) - env = os.environ.copy() - env.update(env_extra) - return subprocess.Popen(args, env=env, **kwds) + def _run_base(self, args, **kwds): + print('RUNNING:', args, kwds) + return subprocess.Popen(args, **kwds) - def _run(self, args, env_extra={}): - popen = self._run_base(args, env_extra, cwd=self.get_path(), + def _run(self, args): + popen = self._run_base(args, cwd=self.get_path(), stdout=subprocess.PIPE, universal_newlines=True) output = popen.stdout.read() @@ -65,6 +66,7 @@ class EmbeddingTests: return output def prepare_module(self, name): + self.patch_environment() if name not in self._compiled_modules: path = self.get_path() filename = '%s.py' % name @@ -74,9 +76,8 @@ class EmbeddingTests: # find a solution to that: we could hack sys.path inside the # script run here, but we can't hack it in the same way in # execute(). - env_extra = {'PYTHONPATH': prefix_pythonpath()} - output = self._run([sys.executable, os.path.join(local_dir, filename)], - env_extra=env_extra) + output = self._run([sys.executable, + os.path.join(local_dir, filename)]) match = re.compile(r"\bFILENAME: (.+)").search(output) assert match dynamic_lib_name = match.group(1) @@ -120,28 +121,35 @@ class EmbeddingTests: finally: os.chdir(curdir) - def execute(self, name): + def patch_environment(self): path = self.get_path() + # for libpypy-c.dll or Python27.dll + path = os.path.split(sys.executable)[0] + os.path.pathsep + path env_extra = {'PYTHONPATH': prefix_pythonpath()} if sys.platform == 'win32': - _path = os.environ.get('PATH') - # for libpypy-c.dll or Python27.dll - _path = os.path.split(sys.executable)[0] + ';' + _path - env_extra['PATH'] = _path + envname = 'PATH' else: - libpath = os.environ.get('LD_LIBRARY_PATH') - if libpath: - libpath = path + ':' + libpath - else: - libpath = path - env_extra['LD_LIBRARY_PATH'] = libpath + envname = 'LD_LIBRARY_PATH' + libpath = org_env.get(envname) + if libpath: + libpath = path + os.path.pathsep + libpath + else: + libpath = path + env_extra[envname] = libpath + for key, value in sorted(env_extra.items()): + if os.environ.get(key) != value: + print '* setting env var %r to %r' % (key, value) + os.environ[key] = value + + def execute(self, name): + path = self.get_path() print('running %r in %r' % (name, path)) executable_name = name if sys.platform == 'win32': executable_name = os.path.join(path, executable_name + '.exe') else: executable_name = os.path.join('.', executable_name) - popen = self._run_base([executable_name], env_extra, cwd=path, + popen = self._run_base([executable_name], cwd=path, stdout=subprocess.PIPE, universal_newlines=True) result = popen.stdout.read() diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h index 33a7e480cd..68bad8a8dc 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread-test.h @@ -1,10 +1,45 @@ +/* Generated by pypy/tool/import_cffi.py */ /************************************************************/ #ifndef _MSC_VER /************************************************************/ #include <pthread.h> -#include <semaphore.h> + +/* don't include <semaphore.h>, it is not available on OS/X */ + +typedef struct { + pthread_mutex_t mutex1; + pthread_cond_t cond1; + unsigned int value; +} sem_t; + +static int sem_init(sem_t *sem, int pshared, unsigned int value) +{ + assert(pshared == 0); + sem->value = value; + return (pthread_mutex_init(&sem->mutex1, NULL) || + pthread_cond_init(&sem->cond1, NULL)); +} + +static int sem_post(sem_t *sem) +{ + pthread_mutex_lock(&sem->mutex1); + sem->value += 1; + pthread_cond_signal(&sem->cond1); + pthread_mutex_unlock(&sem->mutex1); + return 0; +} + +static int sem_wait(sem_t *sem) +{ + pthread_mutex_lock(&sem->mutex1); + while (sem->value == 0) + pthread_cond_wait(&sem->cond1, &sem->mutex1); + sem->value -= 1; + pthread_mutex_unlock(&sem->mutex1); + return 0; +} /************************************************************/ @@ -22,7 +57,7 @@ typedef HANDLE sem_t; typedef HANDLE pthread_t; -int sem_init(sem_t *sem, int pshared, unsigned int value) +static int sem_init(sem_t *sem, int pshared, unsigned int value) { assert(pshared == 0); assert(value == 0); @@ -30,26 +65,26 @@ int sem_init(sem_t *sem, int pshared, unsigned int value) return *sem ? 0 : -1; } -int sem_post(sem_t *sem) +static int sem_post(sem_t *sem) { return ReleaseSemaphore(*sem, 1, NULL) ? 0 : -1; } -int sem_wait(sem_t *sem) +static int sem_wait(sem_t *sem) { WaitForSingleObject(*sem, INFINITE); return 0; } -DWORD WINAPI myThreadProc(LPVOID lpParameter) +static DWORD WINAPI myThreadProc(LPVOID lpParameter) { void *(* start_routine)(void *) = (void *(*)(void *))lpParameter; start_routine(NULL); return 0; } -int pthread_create(pthread_t *thread, void *attr, - void *start_routine(void *), void *arg) +static int pthread_create(pthread_t *thread, void *attr, + void *start_routine(void *), void *arg) { assert(arg == NULL); *thread = CreateThread(NULL, 0, myThreadProc, start_routine, 0, NULL); diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c index 70bb861a26..4cadb4e195 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread1-test.c @@ -1,3 +1,4 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> #include <assert.h> #include "thread-test.h" diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c index 62f5ec849a..a139e1bc6b 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread2-test.c @@ -1,3 +1,4 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> #include <assert.h> #include "thread-test.h" diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c index fa549af213..2f4b370cb9 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/thread3-test.c @@ -1,3 +1,4 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> #include <assert.h> #include "thread-test.h" diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c index b78a03d45a..4a41e16813 100644 --- a/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c +++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/tlocal-test.c @@ -1,3 +1,4 @@ +/* Generated by pypy/tool/import_cffi.py */ #include <stdio.h> #include <assert.h> #include "thread-test.h" diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py index 423f233b6f..15301c0ca9 100644 --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -239,14 +239,12 @@ class AppTestThread(GenericTestThread): if waiting: thread.interrupt_main() return - print 'tock...', x # <-force the GIL to be released, as - time.sleep(0.1) # time.sleep doesn't do non-translated + time.sleep(0.1) def busy_wait(): waiting.append(None) for x in range(50): - print 'tick...', x # <-force the GIL to be released, as - time.sleep(0.1) # time.sleep doesn't do non-translated + time.sleep(0.1) waiting.pop() # This is normally called by app_main.py diff --git a/pypy/module/time/interp_time.py b/pypy/module/time/interp_time.py index 0abafebd88..425a60d7e9 100644 --- a/pypy/module/time/interp_time.py +++ b/pypy/module/time/interp_time.py @@ -4,7 +4,7 @@ from pypy.interpreter.error import OperationError, oefmt from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import lltype from rpython.rlib.rarithmetic import intmask -from rpython.rlib import rposix +from rpython.rlib import rposix, rtime from rpython.translator.tool.cbuild import ExternalCompilationInfo import os import sys @@ -316,13 +316,13 @@ if sys.platform != 'win32': if secs < 0: raise OperationError(space.w_IOError, space.wrap("Invalid argument: negative time in sleep")) - pytime.sleep(secs) + rtime.sleep(secs) else: from rpython.rlib import rwin32 from errno import EINTR def _simple_sleep(space, secs, interruptible): if secs == 0.0 or not interruptible: - pytime.sleep(secs) + rtime.sleep(secs) else: millisecs = int(secs * 1000) interrupt_event = space.fromcache(State).get_interrupt_event() @@ -331,7 +331,7 @@ else: if rc == rwin32.WAIT_OBJECT_0: # Yield to make sure real Python signal handler # called. - pytime.sleep(0.001) + rtime.sleep(0.001) raise wrap_oserror(space, OSError(EINTR, "sleep() interrupted")) @unwrap_spec(secs=float) diff --git a/pypy/objspace/std/celldict.py b/pypy/objspace/std/celldict.py index e9b9bafd89..f16a486b0e 100644 --- a/pypy/objspace/std/celldict.py +++ b/pypy/objspace/std/celldict.py @@ -64,6 +64,9 @@ class ModuleDictStrategy(DictStrategy): def setitem_str(self, w_dict, key, w_value): cell = self.getdictvalue_no_unwrapping(w_dict, key) + return self._setitem_str_cell_known(cell, w_dict, key, w_value) + + def _setitem_str_cell_known(self, cell, w_dict, key, w_value): w_value = write_cell(self.space, cell, w_value) if w_value is None: return @@ -74,10 +77,11 @@ class ModuleDictStrategy(DictStrategy): space = self.space if space.is_w(space.type(w_key), space.w_str): key = space.str_w(w_key) - w_result = self.getitem_str(w_dict, key) + cell = self.getdictvalue_no_unwrapping(w_dict, key) + w_result = unwrap_cell(self.space, cell) if w_result is not None: return w_result - self.setitem_str(w_dict, key, w_default) + self._setitem_str_cell_known(cell, w_dict, key, w_default) return w_default else: self.switch_to_object_strategy(w_dict) diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py index 15869e3702..04e77924d9 100644 --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -350,6 +350,12 @@ class W_DictMultiObject(W_Root): F: D[k] = F[k]""" init_or_update(space, self, __args__, 'dict.update') + def ensure_object_strategy(self): # for cpyext + object_strategy = self.space.fromcache(ObjectDictStrategy) + strategy = self.get_strategy() + if strategy is not object_strategy: + strategy.switch_to_object_strategy(self) + class W_DictObject(W_DictMultiObject): """ a regular dict object """ diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py index d6ff58ad81..d2b602279a 100644 --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -222,6 +222,10 @@ class W_ListObject(W_Root): self.strategy = object_strategy object_strategy.init_from_list_w(self, list_w) + def ensure_object_strategy(self): # for cpyext + if self.strategy is not self.space.fromcache(ObjectListStrategy): + self.switch_to_object_strategy() + def _temporarily_as_objects(self): if self.strategy is self.space.fromcache(ObjectListStrategy): return self diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py index b14b3c14df..6e071dec2a 100644 --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -1,4 +1,4 @@ -import weakref +import weakref, sys from rpython.rlib import jit, objectmodel, debug, rerased from rpython.rlib.rarithmetic import intmask, r_uint @@ -12,6 +12,11 @@ from pypy.objspace.std.dictmultiobject import ( from pypy.objspace.std.typeobject import MutableCell +erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage item") +erase_map, unerase_map = rerased.new_erasing_pair("map") +erase_list, unerase_list = rerased.new_erasing_pair("mapdict storage list") + + # ____________________________________________________________ # attribute shapes @@ -20,6 +25,7 @@ NUM_DIGITS_POW2 = 1 << NUM_DIGITS # note: we use "x * NUM_DIGITS_POW2" instead of "x << NUM_DIGITS" because # we want to propagate knowledge that the result cannot be negative + class AbstractAttribute(object): _immutable_fields_ = ['terminator'] cache_attrs = None @@ -59,21 +65,8 @@ class AbstractAttribute(object): def delete(self, obj, name, index): pass - def find_map_attr(self, name, index): - if jit.we_are_jitted(): - # hack for the jit: - # the _find_map_attr method is pure too, but its argument is never - # constant, because it is always a new tuple - return self._find_map_attr_jit_pure(name, index) - else: - return self._find_map_attr_indirection(name, index) - @jit.elidable - def _find_map_attr_jit_pure(self, name, index): - return self._find_map_attr_indirection(name, index) - - @jit.dont_look_inside - def _find_map_attr_indirection(self, name, index): + def find_map_attr(self, name, index): if (self.space.config.objspace.std.withmethodcache): return self._find_map_attr_cache(name, index) return self._find_map_attr(name, index) @@ -151,29 +144,124 @@ class AbstractAttribute(object): cache[name, index] = attr return attr - @jit.look_inside_iff(lambda self, obj, name, index, w_value: - jit.isconstant(self) and - jit.isconstant(name) and - jit.isconstant(index)) + @jit.elidable + def _get_cache_attr(self, name, index): + key = name, index + # this method is not actually elidable, but it's fine anyway + if self.cache_attrs is not None: + return self.cache_attrs.get(key, None) + return None + def add_attr(self, obj, name, index, w_value): - attr = self._get_new_attr(name, index) - oldattr = obj._get_mapdict_map() + self._reorder_and_add(obj, name, index, w_value) if not jit.we_are_jitted(): + oldattr = self + attr = obj._get_mapdict_map() size_est = (oldattr._size_estimate + attr.size_estimate() - oldattr.size_estimate()) assert size_est >= (oldattr.length() * NUM_DIGITS_POW2) oldattr._size_estimate = size_est - if attr.length() > obj._mapdict_storage_length(): - # note that attr.size_estimate() is always at least attr.length() - new_storage = [None] * attr.size_estimate() + + def _add_attr_without_reordering(self, obj, name, index, w_value): + attr = self._get_new_attr(name, index) + attr._switch_map_and_write_storage(obj, w_value) + + @jit.unroll_safe + def _switch_map_and_write_storage(self, obj, w_value): + if self.length() > obj._mapdict_storage_length(): + # note that self.size_estimate() is always at least self.length() + new_storage = [None] * self.size_estimate() for i in range(obj._mapdict_storage_length()): new_storage[i] = obj._mapdict_read_storage(i) - obj._set_mapdict_storage_and_map(new_storage, attr) + obj._set_mapdict_storage_and_map(new_storage, self) # the order is important here: first change the map, then the storage, # for the benefit of the special subclasses - obj._set_mapdict_map(attr) - obj._mapdict_write_storage(attr.storageindex, w_value) + obj._set_mapdict_map(self) + obj._mapdict_write_storage(self.storageindex, w_value) + + + @jit.elidable + def _find_branch_to_move_into(self, name, index): + # walk up the map chain to find an ancestor with lower order that + # already has the current name as a child inserted + current_order = sys.maxint + number_to_readd = 0 + current = self + key = (name, index) + while True: + attr = None + if current.cache_attrs is not None: + attr = current.cache_attrs.get(key, None) + if attr is None or attr.order > current_order: + # we reached the top, so we didn't find it anywhere, + # just add it to the top attribute + if not isinstance(current, PlainAttribute): + return 0, self._get_new_attr(name, index) + + else: + return number_to_readd, attr + # if not found try parent + number_to_readd += 1 + current_order = current.order + current = current.back + + @jit.look_inside_iff(lambda self, obj, name, index, w_value: + jit.isconstant(self) and + jit.isconstant(name) and + jit.isconstant(index)) + def _reorder_and_add(self, obj, name, index, w_value): + # the idea is as follows: the subtrees of any map are ordered by + # insertion. the invariant is that subtrees that are inserted later + # must not contain the name of the attribute of any earlier inserted + # attribute anywhere + # m______ + # inserted first / \ ... \ further attributes + # attrname a 0/ 1\ n\ + # m a must not appear here anywhere + # + # when inserting a new attribute in an object we check whether any + # parent of lower order has seen that attribute yet. if yes, we follow + # that branch. if not, we normally append that attribute. When we + # follow a prior branch, we necessarily remove some attributes to be + # able to do that. They need to be re-added, which has to follow the + # reordering procedure recusively. + + # we store the to-be-readded attribute in the stack, with the map and + # the value paired up those are lazily initialized to a list large + # enough to store all current attributes + stack = None + stack_index = 0 + while True: + current = self + number_to_readd = 0 + number_to_readd, attr = self._find_branch_to_move_into(name, index) + # we found the attributes further up, need to save the + # previous values of the attributes we passed + if number_to_readd: + if stack is None: + stack = [erase_map(None)] * (self.length() * 2) + current = self + for i in range(number_to_readd): + assert isinstance(current, PlainAttribute) + w_self_value = obj._mapdict_read_storage( + current.storageindex) + stack[stack_index] = erase_map(current) + stack[stack_index + 1] = erase_item(w_self_value) + stack_index += 2 + current = current.back + attr._switch_map_and_write_storage(obj, w_value) + + if not stack_index: + return + + # readd the current top of the stack + stack_index -= 2 + next_map = unerase_map(stack[stack_index]) + w_value = unerase_item(stack[stack_index + 1]) + name = next_map.name + index = next_map.index + self = obj._get_mapdict_map() def materialize_r_dict(self, space, obj, dict_w): raise NotImplementedError("abstract base class") @@ -279,7 +367,7 @@ class DevolvedDictTerminator(Terminator): return Terminator.set_terminator(self, obj, terminator) class PlainAttribute(AbstractAttribute): - _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?'] + _immutable_fields_ = ['name', 'index', 'storageindex', 'back', 'ever_mutated?', 'order'] def __init__(self, name, index, back): AbstractAttribute.__init__(self, back.space, back.terminator) @@ -289,6 +377,7 @@ class PlainAttribute(AbstractAttribute): self.back = back self._size_estimate = self.length() * NUM_DIGITS_POW2 self.ever_mutated = False + self.order = len(back.cache_attrs) if back.cache_attrs else 0 def _copy_attr(self, obj, new_obj): w_value = self.read(obj, self.name, self.index) @@ -542,9 +631,6 @@ def memo_get_subclass_of_correct_size(space, supercls): memo_get_subclass_of_correct_size._annspecialcase_ = "specialize:memo" _subclass_cache = {} -erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage item") -erase_list, unerase_list = rerased.new_erasing_pair("mapdict storage list") - def _make_subclass_size_n(supercls, n): from rpython.rlib import unroll rangen = unroll.unrolling_iterable(range(n)) diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py index a680dbf26d..5b5e8a74fe 100644 --- a/pypy/objspace/std/objspace.py +++ b/pypy/objspace/std/objspace.py @@ -359,7 +359,8 @@ class StdObjSpace(ObjSpace): subcls = get_subclass_of_correct_size(self, cls, w_subtype) else: subcls = get_unique_interplevel_subclass( - self.config, cls, w_subtype.hasdict, w_subtype.nslots != 0, + self.config, cls, w_subtype.hasdict, + w_subtype.layout.nslots != 0, w_subtype.needsdel, w_subtype.weakrefable) instance = instantiate(subcls) assert isinstance(instance, cls) diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py index 7172f64ec4..b80762cb3d 100644 --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -1076,7 +1076,7 @@ class AbstractUnwrappedSetStrategy(object): if self is w_other.strategy: strategy = self if w_set.length() > w_other.length(): - # swap operants + # swap operands storage = self._intersect_unwrapped(w_other, w_set) else: storage = self._intersect_unwrapped(w_set, w_other) @@ -1086,7 +1086,7 @@ class AbstractUnwrappedSetStrategy(object): else: strategy = self.space.fromcache(ObjectSetStrategy) if w_set.length() > w_other.length(): - # swap operants + # swap operands storage = w_other.strategy._intersect_wrapped(w_other, w_set) else: storage = self._intersect_wrapped(w_set, w_other) diff --git a/pypy/objspace/std/test/test_celldict.py b/pypy/objspace/std/test/test_celldict.py index 28e2a148d1..54f961f8b0 100644 --- a/pypy/objspace/std/test/test_celldict.py +++ b/pypy/objspace/std/test/test_celldict.py @@ -108,22 +108,11 @@ class AppTestModuleDict(object): class TestModuleDictImplementation(BaseTestRDictImplementation): StrategyClass = ModuleDictStrategy - -class TestModuleDictImplementationWithBuiltinNames(BaseTestRDictImplementation): - StrategyClass = ModuleDictStrategy - - string = "int" - string2 = "isinstance" - + setdefault_hash_count = 2 class TestDevolvedModuleDictImplementation(BaseTestDevolvedDictImplementation): StrategyClass = ModuleDictStrategy - -class TestDevolvedModuleDictImplementationWithBuiltinNames(BaseTestDevolvedDictImplementation): - StrategyClass = ModuleDictStrategy - - string = "int" - string2 = "isinstance" + setdefault_hash_count = 2 class AppTestCellDict(object): diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py index e02144e25f..8b72c83fc1 100644 --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -1248,6 +1248,9 @@ class BaseTestRDictImplementation: impl.setitem(x, x) assert type(impl.get_strategy()) is ObjectDictStrategy + + setdefault_hash_count = 1 + def test_setdefault_fast(self): on_pypy = "__pypy__" in sys.builtin_module_names impl = self.impl @@ -1255,11 +1258,11 @@ class BaseTestRDictImplementation: x = impl.setdefault(key, 1) assert x == 1 if on_pypy: - assert key.hash_count == 1 + assert key.hash_count == self.setdefault_hash_count x = impl.setdefault(key, 2) assert x == 1 if on_pypy: - assert key.hash_count == 2 + assert key.hash_count == self.setdefault_hash_count + 1 def test_fallback_evil_key(self): class F(object): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py index 5f1772afd0..1f051e8753 100644 --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -107,6 +107,153 @@ def test_add_attribute(): assert obj2.getdictvalue(space, "b") == 60 assert obj2.map is obj.map +def test_insert_different_orders(): + cls = Class() + obj = cls.instantiate() + obj.setdictvalue(space, "a", 10) + obj.setdictvalue(space, "b", 20) + + obj2 = cls.instantiate() + obj2.setdictvalue(space, "b", 30) + obj2.setdictvalue(space, "a", 40) + + assert obj.map is obj2.map + +def test_insert_different_orders_2(): + cls = Class() + obj = cls.instantiate() + obj2 = cls.instantiate() + + obj.setdictvalue(space, "a", 10) + + obj2.setdictvalue(space, "b", 20) + obj2.setdictvalue(space, "a", 30) + + obj.setdictvalue(space, "b", 40) + assert obj.map is obj2.map + +def test_insert_different_orders_3(): + cls = Class() + obj = cls.instantiate() + obj2 = cls.instantiate() + obj3 = cls.instantiate() + obj4 = cls.instantiate() + obj5 = cls.instantiate() + obj6 = cls.instantiate() + + obj.setdictvalue(space, "a", 10) + obj.setdictvalue(space, "b", 20) + obj.setdictvalue(space, "c", 30) + + obj2.setdictvalue(space, "a", 30) + obj2.setdictvalue(space, "c", 40) + obj2.setdictvalue(space, "b", 50) + + obj3.setdictvalue(space, "c", 30) + obj3.setdictvalue(space, "a", 40) + obj3.setdictvalue(space, "b", 50) + + obj4.setdictvalue(space, "c", 30) + obj4.setdictvalue(space, "b", 40) + obj4.setdictvalue(space, "a", 50) + + obj5.setdictvalue(space, "b", 30) + obj5.setdictvalue(space, "a", 40) + obj5.setdictvalue(space, "c", 50) + + obj6.setdictvalue(space, "b", 30) + obj6.setdictvalue(space, "c", 40) + obj6.setdictvalue(space, "a", 50) + + assert obj.map is obj2.map + assert obj.map is obj3.map + assert obj.map is obj4.map + assert obj.map is obj5.map + assert obj.map is obj6.map + + +def test_insert_different_orders_4(): + cls = Class() + obj = cls.instantiate() + obj2 = cls.instantiate() + + obj.setdictvalue(space, "a", 10) + obj.setdictvalue(space, "b", 20) + obj.setdictvalue(space, "c", 30) + obj.setdictvalue(space, "d", 40) + + obj2.setdictvalue(space, "d", 50) + obj2.setdictvalue(space, "c", 50) + obj2.setdictvalue(space, "b", 50) + obj2.setdictvalue(space, "a", 50) + + assert obj.map is obj2.map + +def test_insert_different_orders_5(): + cls = Class() + obj = cls.instantiate() + obj2 = cls.instantiate() + + obj.setdictvalue(space, "a", 10) + obj.setdictvalue(space, "b", 20) + obj.setdictvalue(space, "c", 30) + obj.setdictvalue(space, "d", 40) + + obj2.setdictvalue(space, "d", 50) + obj2.setdictvalue(space, "c", 50) + obj2.setdictvalue(space, "b", 50) + obj2.setdictvalue(space, "a", 50) + + obj3 = cls.instantiate() + obj3.setdictvalue(space, "d", 50) + obj3.setdictvalue(space, "c", 50) + obj3.setdictvalue(space, "b", 50) + obj3.setdictvalue(space, "a", 50) + + assert obj.map is obj3.map + + +def test_bug_stack_overflow_insert_attributes(): + cls = Class() + obj = cls.instantiate() + + for i in range(1000): + obj.setdictvalue(space, str(i), i) + + +def test_insert_different_orders_perm(): + from itertools import permutations + cls = Class() + seen_maps = {} + for preexisting in ['', 'x', 'xy']: + for i, attributes in enumerate(permutations("abcdef")): + obj = cls.instantiate() + for i, attr in enumerate(preexisting): + obj.setdictvalue(space, attr, i*1000) + key = preexisting + for j, attr in enumerate(attributes): + obj.setdictvalue(space, attr, i*10+j) + key = "".join(sorted(key+attr)) + if key in seen_maps: + assert obj.map is seen_maps[key] + else: + seen_maps[key] = obj.map + + print len(seen_maps) + + +def test_bug_infinite_loop(): + cls = Class() + obj = cls.instantiate() + obj.setdictvalue(space, "e", 1) + obj2 = cls.instantiate() + obj2.setdictvalue(space, "f", 2) + obj3 = cls.instantiate() + obj3.setdictvalue(space, "a", 3) + obj3.setdictvalue(space, "e", 4) + obj3.setdictvalue(space, "f", 5) + + def test_attr_immutability(monkeypatch): cls = Class() obj = cls.instantiate() @@ -359,9 +506,15 @@ def get_impl(self): class TestMapDictImplementation(BaseTestRDictImplementation): StrategyClass = MapDictStrategy get_impl = get_impl + def test_setdefault_fast(self): + # mapdict can't pass this, which is fine + pass class TestDevolvedMapDictImplementation(BaseTestDevolvedDictImplementation): get_impl = get_impl StrategyClass = MapDictStrategy + def test_setdefault_fast(self): + # mapdict can't pass this, which is fine + pass # ___________________________________________________________ # tests that check the obj interface after the dict has devolved @@ -1132,3 +1285,7 @@ def test_newdict_instance(): class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation): StrategyClass = MapDictStrategy # NB: the get_impl method is not overwritten here, as opposed to above + + def test_setdefault_fast(self): + # mapdict can't pass this, which is fine + pass diff --git a/pypy/objspace/std/transparent.py b/pypy/objspace/std/transparent.py index d9dba6d6c3..e63a3b5483 100644 --- a/pypy/objspace/std/transparent.py +++ b/pypy/objspace/std/transparent.py @@ -62,7 +62,7 @@ completely controlled by the controller.""" return W_TransparentGenerator(space, w_type, w_controller) if space.is_true(space.issubtype(w_type, space.gettypeobject(PyCode.typedef))): return W_TransparentCode(space, w_type, w_controller) - if w_type.instancetypedef is space.w_object.instancetypedef: + if w_type.layout.typedef is space.w_object.layout.typedef: return W_Transparent(space, w_type, w_controller) else: raise OperationError(space.w_TypeError, space.wrap("type expected as first argument")) diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py index e9c09065ae..54243af7dc 100644 --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -30,6 +30,11 @@ def _unroll_condition_cmp(self, space, other): contains_jmp = jit.JitDriver(greens = ['tp'], reds = 'auto', name = 'tuple.contains') +hash_driver = jit.JitDriver( + name='tuple.hash', + greens=['w_type'], + reds='auto') + class W_AbstractTupleObject(W_Root): __slots__ = () @@ -262,12 +267,32 @@ class W_TupleObject(W_AbstractTupleObject): def length(self): return len(self.wrappeditems) - @jit.look_inside_iff(lambda self, _1: _unroll_condition(self)) def descr_hash(self, space): + if _unroll_condition(self): + return self._descr_hash_unroll(space) + else: + return self._descr_hash_jitdriver(space) + + @jit.unroll_safe + def _descr_hash_unroll(self, space): + mult = 1000003 + x = 0x345678 + z = len(self.wrappeditems) + for w_item in self.wrappeditems: + y = space.hash_w(w_item) + x = (x ^ y) * mult + z -= 1 + mult += 82520 + z + z + x += 97531 + return space.wrap(intmask(x)) + + def _descr_hash_jitdriver(self, space): mult = 1000003 x = 0x345678 z = len(self.wrappeditems) + w_type = space.type(self.wrappeditems[0]) for w_item in self.wrappeditems: + hash_driver.jit_merge_point(w_type=w_type) y = space.hash_w(w_item) x = (x ^ y) * mult z -= 1 diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py index 154618fcad..874a7c91df 100644 --- a/pypy/objspace/std/typeobject.py +++ b/pypy/objspace/std/typeobject.py @@ -7,7 +7,7 @@ from pypy.interpreter.typedef import weakref_descr, GetSetProperty,\ from pypy.interpreter.astcompiler.misc import mangle from rpython.rlib.jit import (promote, elidable_promote, we_are_jitted, - promote_string, elidable, dont_look_inside, unroll_safe) + elidable, dont_look_inside, unroll_safe) from rpython.rlib.objectmodel import current_object_addr_as_int, compute_hash from rpython.rlib.rarithmetic import intmask, r_uint @@ -87,6 +87,29 @@ class MethodCache(object): for i in range(len(self.lookup_where)): self.lookup_where[i] = None_None + +class Layout(object): + """A Layout is attached to every W_TypeObject to represent the + layout of instances. Some W_TypeObjects share the same layout. + If a W_TypeObject is a base of another, then the layout of + the first is either the same or a parent layout of the second. + The Layouts have single inheritance, unlike W_TypeObjects. + """ + _immutable_ = True + + def __init__(self, typedef, nslots, base_layout=None): + self.typedef = typedef + self.nslots = nslots + self.base_layout = base_layout + + def issublayout(self, parent): + while self is not parent: + self = self.base_layout + if self is None: + return False + return True + + # possible values of compares_by_identity_status UNKNOWN = 0 COMPARES_BY_IDENTITY = 1 @@ -106,8 +129,7 @@ class W_TypeObject(W_Root): 'needsdel', 'weakrefable', 'hasdict', - 'nslots', - 'instancetypedef', + 'layout', 'terminator', '_version_tag?', 'name?', @@ -126,12 +148,11 @@ class W_TypeObject(W_Root): @dont_look_inside def __init__(w_self, space, name, bases_w, dict_w, - overridetypedef=None): + overridetypedef=None, force_new_layout=False): w_self.space = space w_self.name = name w_self.bases_w = bases_w w_self.dict_w = dict_w - w_self.nslots = 0 w_self.hasdict = False w_self.needsdel = False w_self.weakrefable = False @@ -141,13 +162,13 @@ class W_TypeObject(W_Root): w_self.flag_cpytype = False w_self.flag_abstract = False w_self.flag_sequence_bug_compat = False - w_self.instancetypedef = overridetypedef if overridetypedef is not None: - setup_builtin_type(w_self) + assert not force_new_layout + layout = setup_builtin_type(w_self, overridetypedef) else: - setup_user_defined_type(w_self) - w_self.w_same_layout_as = get_parent_layout(w_self) + layout = setup_user_defined_type(w_self, force_new_layout) + w_self.layout = layout if space.config.objspace.std.withtypeversion: if not is_mro_purely_of_types(w_self.mro_w): @@ -164,6 +185,10 @@ class W_TypeObject(W_Root): else: w_self.terminator = NoDictTerminator(space, w_self) + def __repr__(self): + "NOT_RPYTHON" + return '<W_TypeObject %r at 0x%x>' % (self.name, id(self)) + def mutated(w_self, key): """ The type is being mutated. key is either the string containing the @@ -264,8 +289,8 @@ class W_TypeObject(W_Root): # compute a tuple that fully describes the instance layout def get_full_instance_layout(w_self): - w_layout = w_self.w_same_layout_as or w_self - return (w_layout, w_self.hasdict, w_self.needsdel, w_self.weakrefable) + layout = w_self.layout + return (layout, w_self.hasdict, w_self.needsdel, w_self.weakrefable) def compute_default_mro(w_self): return compute_C3_mro(w_self.space, w_self) @@ -402,7 +427,6 @@ class W_TypeObject(W_Root): if version_tag is None: tup = w_self._lookup_where(name) return tup - name = promote_string(name) tup_w = w_self._pure_lookup_where_with_method_cache(name, version_tag) w_class, w_value = tup_w if (space.config.objspace.std.withtypeversion and @@ -463,7 +487,7 @@ class W_TypeObject(W_Root): raise oefmt(space.w_TypeError, "%N.__new__(%N): %N is not a subtype of %N", w_self, w_subtype, w_subtype, w_self) - if w_self.instancetypedef is not w_subtype.instancetypedef: + if w_self.layout.typedef is not w_subtype.layout.typedef: raise oefmt(space.w_TypeError, "%N.__new__(%N) is not safe, use %N.__new__()", w_self, w_subtype, w_subtype) @@ -817,11 +841,10 @@ def descr_set__bases__(space, w_type, w_value): for w_subclass in w_type.get_subclasses(): if isinstance(w_subclass, W_TypeObject): w_subclass._version_tag = None - assert w_type.w_same_layout_as is get_parent_layout(w_type) # invariant def descr__base(space, w_type): w_type = _check(space, w_type) - return find_best_base(space, w_type.bases_w) + return find_best_base(w_type.bases_w) def descr__doc(space, w_type): if space.is_w(w_type, space.w_type): @@ -924,48 +947,7 @@ W_TypeObject.typedef = TypeDef("type", # ____________________________________________________________ # Initialization of type objects -def get_parent_layout(w_type): - """Compute the most parent class of 'w_type' whose layout - is the same as 'w_type', or None if all parents of 'w_type' - have a different layout than 'w_type'. - """ - w_starttype = w_type - while len(w_type.bases_w) > 0: - w_bestbase = find_best_base(w_type.space, w_type.bases_w) - if w_type.instancetypedef is not w_bestbase.instancetypedef: - break - if w_type.nslots != w_bestbase.nslots: - break - w_type = w_bestbase - if w_type is not w_starttype: - return w_type - else: - return None - -def issublayout(w_layout1, w_layout2): - space = w_layout2.space - while w_layout1 is not w_layout2: - w_layout1 = find_best_base(space, w_layout1.bases_w) - if w_layout1 is None: - return False - w_layout1 = w_layout1.w_same_layout_as or w_layout1 - return True - -@unroll_safe -def issubtypedef(a, b): - from pypy.objspace.std.objectobject import W_ObjectObject - if b is W_ObjectObject.typedef: - return True - if a is None: - return False - if a is b: - return True - for a1 in a.bases: - if issubtypedef(a1, b): - return True - return False - -def find_best_base(space, bases_w): +def find_best_base(bases_w): """The best base is one of the bases in the given list: the one whose layout a new type should use as a starting point. """ @@ -976,14 +958,10 @@ def find_best_base(space, bases_w): if w_bestbase is None: w_bestbase = w_candidate # for now continue - candtypedef = w_candidate.instancetypedef - besttypedef = w_bestbase.instancetypedef - if candtypedef is besttypedef: - # two candidates with the same typedef are equivalent unless - # one has extra slots over the other - if w_candidate.nslots > w_bestbase.nslots: - w_bestbase = w_candidate - elif issubtypedef(candtypedef, besttypedef): + cand_layout = w_candidate.layout + best_layout = w_bestbase.layout + if (cand_layout is not best_layout and + cand_layout.issublayout(best_layout)): w_bestbase = w_candidate return w_bestbase @@ -992,20 +970,21 @@ def check_and_find_best_base(space, bases_w): whose layout a new type should use as a starting point. This version checks that bases_w is an acceptable tuple of bases. """ - w_bestbase = find_best_base(space, bases_w) + w_bestbase = find_best_base(bases_w) if w_bestbase is None: raise oefmt(space.w_TypeError, "a new-style class can't have only classic bases") - if not w_bestbase.instancetypedef.acceptable_as_base_class: + if not w_bestbase.layout.typedef.acceptable_as_base_class: raise oefmt(space.w_TypeError, "type '%N' is not an acceptable base class", w_bestbase) - # check that all other bases' layouts are superclasses of the bestbase - w_bestlayout = w_bestbase.w_same_layout_as or w_bestbase + # check that all other bases' layouts are "super-layouts" of the + # bestbase's layout + best_layout = w_bestbase.layout for w_base in bases_w: if isinstance(w_base, W_TypeObject): - w_layout = w_base.w_same_layout_as or w_base - if not issublayout(w_bestlayout, w_layout): + layout = w_base.layout + if not best_layout.issublayout(layout): raise oefmt(space.w_TypeError, "instance layout conflicts in multiple inheritance") return w_bestbase @@ -1019,10 +998,11 @@ def copy_flags_from_bases(w_self, w_bestbase): w_self.hasdict = w_self.hasdict or w_base.hasdict w_self.needsdel = w_self.needsdel or w_base.needsdel w_self.weakrefable = w_self.weakrefable or w_base.weakrefable - w_self.nslots = w_bestbase.nslots return hasoldstylebase -def create_all_slots(w_self, hasoldstylebase, w_bestbase): +def create_all_slots(w_self, hasoldstylebase, w_bestbase, force_new_layout): + base_layout = w_bestbase.layout + index_next_extra_slot = base_layout.nslots space = w_self.space dict_w = w_self.dict_w if '__slots__' not in dict_w: @@ -1050,7 +1030,8 @@ def create_all_slots(w_self, hasoldstylebase, w_bestbase): "__weakref__ slot disallowed: we already got one") wantweakref = True else: - create_slot(w_self, slot_name) + index_next_extra_slot = create_slot(w_self, slot_name, + index_next_extra_slot) wantdict = wantdict or hasoldstylebase if wantdict: create_dict_slot(w_self) @@ -1058,8 +1039,14 @@ def create_all_slots(w_self, hasoldstylebase, w_bestbase): create_weakref_slot(w_self) if '__del__' in dict_w: w_self.needsdel = True + # + if index_next_extra_slot == base_layout.nslots and not force_new_layout: + return base_layout + else: + return Layout(base_layout.typedef, index_next_extra_slot, + base_layout=base_layout) -def create_slot(w_self, slot_name): +def create_slot(w_self, slot_name, index_next_extra_slot): space = w_self.space if not valid_slot_name(slot_name): raise oefmt(space.w_TypeError, "__slots__ must be identifiers") @@ -1069,9 +1056,10 @@ def create_slot(w_self, slot_name): # Force interning of slot names. slot_name = space.str_w(space.new_interned_str(slot_name)) # in cpython it is ignored less, but we probably don't care - member = Member(w_self.nslots, slot_name, w_self) + member = Member(index_next_extra_slot, slot_name, w_self) + index_next_extra_slot += 1 w_self.dict_w[slot_name] = space.wrap(member) - w_self.nslots += 1 + return index_next_extra_slot def create_dict_slot(w_self): if not w_self.hasdict: @@ -1093,11 +1081,10 @@ def valid_slot_name(slot_name): return False return True -def setup_user_defined_type(w_self): +def setup_user_defined_type(w_self, force_new_layout): if len(w_self.bases_w) == 0: w_self.bases_w = [w_self.space.w_object] w_bestbase = check_and_find_best_base(w_self.space, w_self.bases_w) - w_self.instancetypedef = w_bestbase.instancetypedef w_self.flag_heaptype = True for w_base in w_self.bases_w: if not isinstance(w_base, W_TypeObject): @@ -1106,16 +1093,29 @@ def setup_user_defined_type(w_self): w_self.flag_abstract |= w_base.flag_abstract hasoldstylebase = copy_flags_from_bases(w_self, w_bestbase) - create_all_slots(w_self, hasoldstylebase, w_bestbase) + layout = create_all_slots(w_self, hasoldstylebase, w_bestbase, + force_new_layout) ensure_common_attributes(w_self) + return layout -def setup_builtin_type(w_self): - w_self.hasdict = w_self.instancetypedef.hasdict - w_self.weakrefable = w_self.instancetypedef.weakrefable - w_self.w_doc = w_self.space.wrap(w_self.instancetypedef.doc) +def setup_builtin_type(w_self, instancetypedef): + w_self.hasdict = instancetypedef.hasdict + w_self.weakrefable = instancetypedef.weakrefable + w_self.w_doc = w_self.space.wrap(instancetypedef.doc) ensure_common_attributes(w_self) - w_self.flag_heaptype = w_self.instancetypedef.heaptype + w_self.flag_heaptype = instancetypedef.heaptype + # + # usually 'instancetypedef' is new, i.e. not seen in any base, + # but not always (see Exception class) + w_bestbase = find_best_base(w_self.bases_w) + if w_bestbase is None: + parent_layout = None + else: + parent_layout = w_bestbase.layout + if parent_layout.typedef is instancetypedef: + return parent_layout + return Layout(instancetypedef, 0, base_layout=parent_layout) def ensure_common_attributes(w_self): ensure_static_new(w_self) diff --git a/pypy/tool/import_cffi.py b/pypy/tool/import_cffi.py index 2176baa901..eb747d8ac3 100755 --- a/pypy/tool/import_cffi.py +++ b/pypy/tool/import_cffi.py @@ -7,11 +7,18 @@ import_cffi.py <path-to-cffi> import sys, py -def mangle(lines): - yield "# Generated by pypy/tool/import_cffi.py\n" - for line in lines: - line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') - yield line +def mangle(lines, ext): + if ext == '.py': + yield "# Generated by pypy/tool/import_cffi.py\n" + for line in lines: + line = line.replace('from testing', 'from pypy.module.test_lib_pypy.cffi_tests') + yield line + elif ext in ('.c', '.h'): + yield "/* Generated by pypy/tool/import_cffi.py */\n" + for line in lines: + yield line + else: + raise AssertionError(ext) def main(cffi_dir): cffi_dir = py.path.local(cffi_dir) @@ -23,10 +30,12 @@ def main(cffi_dir): for p in (list(cffi_dir.join('cffi').visit(fil='*.py')) + list(cffi_dir.join('cffi').visit(fil='*.h'))): cffi_dest.join('..', p.relto(cffi_dir)).write(p.read()) - for p in cffi_dir.join('testing').visit(fil='*.py'): + for p in (list(cffi_dir.join('testing').visit(fil='*.py')) + + list(cffi_dir.join('testing').visit(fil='*.h')) + + list(cffi_dir.join('testing').visit(fil='*.c'))): path = test_dest.join(p.relto(cffi_dir.join('testing'))) path.join('..').ensure(dir=1) - path.write(''.join(mangle(p.readlines()))) + path.write(''.join(mangle(p.readlines(), p.ext))) if __name__ == '__main__': if len(sys.argv) != 2: diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py index 84f506e555..749db2f8b1 100644 --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -58,6 +58,9 @@ class AppFrame(py.code.Frame): self.w_locals = space.getattr(pyframe, space.wrap('f_locals')) self.f_locals = self.w_locals # for py.test's recursion detection + def get_w_globals(self): + return self.w_globals + def eval(self, code, **vars): space = self.space for key, w_value in vars.items(): diff --git a/pypy/tool/test/test_tab.py b/pypy/tool/test/test_tab.py index 05fdb2ef12..14fd63b812 100644 --- a/pypy/tool/test/test_tab.py +++ b/pypy/tool/test/test_tab.py @@ -6,7 +6,8 @@ import os from pypy.conftest import pypydir ROOT = os.path.abspath(os.path.join(pypydir, '..')) -EXCLUDE = {} +RPYTHONDIR = os.path.join(ROOT, "rpython") +EXCLUDE = {'/virt_test/lib/python2.7/site-packages/setuptools'} def test_no_tabs(): @@ -28,3 +29,27 @@ def test_no_tabs(): if not entry.startswith('.'): walk('%s/%s' % (reldir, entry)) walk('') + +def test_no_pypy_import_in_rpython(): + def walk(reldir): + print reldir + if reldir: + path = os.path.join(RPYTHONDIR, *reldir.split('/')) + else: + path = RPYTHONDIR + if os.path.isfile(path): + if not path.lower().endswith('.py'): + return + with file(path) as f: + for line in f: + if "import" not in line: + continue + assert "from pypy." not in line + assert "import pypy." not in line + elif os.path.isdir(path) and not os.path.islink(path): + for entry in os.listdir(path): + if not entry.startswith('.'): + walk('%s/%s' % (reldir, entry)) + + walk('') + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..a04799b455 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# hypothesis is used for test generation on untranslated jit tests +hypothesis + diff --git a/rpython/annotator/policy.py b/rpython/annotator/policy.py index 5a33b79083..93c3dccdd1 100644 --- a/rpython/annotator/policy.py +++ b/rpython/annotator/policy.py @@ -3,6 +3,9 @@ from rpython.annotator.specialize import default_specialize as default from rpython.annotator.specialize import ( specialize_argvalue, specialize_argtype, specialize_arglistitemtype, specialize_arg_or_var, memo, specialize_call_location) +from rpython.flowspace.operation import op +from rpython.flowspace.model import Constant +from rpython.annotator.model import SomeTuple class AnnotatorPolicy(object): @@ -64,7 +67,34 @@ class AnnotatorPolicy(object): return LowLevelAnnotatorPolicy.specialize__ll_and_arg(*args) def no_more_blocks_to_annotate(pol, annotator): + bk = annotator.bookkeeper # hint to all pending specializers that we are done - for callback in annotator.bookkeeper.pending_specializations: + for callback in bk.pending_specializations: callback() - del annotator.bookkeeper.pending_specializations[:] + del bk.pending_specializations[:] + if annotator.added_blocks is not None: + all_blocks = annotator.added_blocks + else: + all_blocks = annotator.annotated + for block in list(all_blocks): + for i, instr in enumerate(block.operations): + if not isinstance(instr, (op.simple_call, op.call_args)): + continue + v_func = instr.args[0] + s_func = annotator.annotation(v_func) + if not hasattr(s_func, 'needs_sandboxing'): + continue + key = ('sandboxing', s_func.const) + if key not in bk.emulated_pbc_calls: + params_s = s_func.args_s + s_result = s_func.s_result + from rpython.translator.sandbox.rsandbox import make_sandbox_trampoline + sandbox_trampoline = make_sandbox_trampoline( + s_func.name, params_s, s_result) + sandbox_trampoline._signature_ = [SomeTuple(items=params_s)], s_result + bk.emulate_pbc_call(key, bk.immutablevalue(sandbox_trampoline), params_s) + else: + s_trampoline = bk.emulated_pbc_calls[key][0] + sandbox_trampoline = s_trampoline.const + new = instr.replace({instr.args[0]: Constant(sandbox_trampoline)}) + block.operations[i] = new diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py index 14bd0d6e85..436d8ec635 100644 --- a/rpython/annotator/specialize.py +++ b/rpython/annotator/specialize.py @@ -317,18 +317,6 @@ def cartesian_product(lstlst): yield (value,) + tuple_tail -def make_constgraphbuilder(n, v=None, factory=None, srcmodule=None): - def constgraphbuilder(translator, ignore): - args = ','.join(["arg%d" % i for i in range(n)]) - if factory is not None: - computed_v = factory() - else: - computed_v = v - miniglobals = {'v': computed_v, '__name__': srcmodule} - exec py.code.Source("constf = lambda %s: v" % args).compile() in miniglobals - return translator.buildflowgraph(miniglobals['constf']) - return constgraphbuilder - def maybe_star_args(funcdesc, key, args_s): args_s, key1, builder = flatten_star_args(funcdesc, args_s) if key1 is not None: diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py index 50ea9b7769..27629f5639 100644 --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -113,8 +113,9 @@ def contains_number(annotator, number, element): @op.simple_call.register(SomeObject) def simple_call_SomeObject(annotator, func, *args): - return annotator.annotation(func).call( - simple_args([annotator.annotation(arg) for arg in args])) + s_func = annotator.annotation(func) + argspec = simple_args([annotator.annotation(arg) for arg in args]) + return s_func.call(argspec) @op.call_args.register_transform(SomeObject) def transform_varargs(annotator, v_func, v_shape, *data_v): diff --git a/rpython/jit/backend/ppc/test/test_runner.py b/rpython/jit/backend/ppc/test/test_runner.py index 3d2c0ef959..9137407b84 100644 --- a/rpython/jit/backend/ppc/test/test_runner.py +++ b/rpython/jit/backend/ppc/test/test_runner.py @@ -134,7 +134,7 @@ class TestPPC(LLtypeBackendTest): def test_debugger_on(self): py.test.skip("XXX") - from pypy.rlib import debug + from rpython.rlib import debug targettoken, preambletoken = TargetToken(), TargetToken() loop = """ diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py index 2e8e683dd9..12357425a9 100644 --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -2042,6 +2042,11 @@ class Transformer(object): self.vable_flags[op.args[0]] = op.args[2].value return [] + def rewrite_op_jit_enter_portal_frame(self, op): + return [op] + def rewrite_op_jit_leave_portal_frame(self, op): + return [op] + # --------- # ll_math.sqrt_nonneg() diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py index 177a507e69..e319a87883 100644 --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -246,12 +246,12 @@ def _ll_1_jit_force_virtual(inst): def _ll_2_int_floordiv_ovf_zer(x, y): if y == 0: raise ZeroDivisionError - if x == -sys.maxint - 1 and y == -1: - raise OverflowError - return llop.int_floordiv(lltype.Signed, x, y) + return _ll_2_int_floordiv_ovf(x, y) def _ll_2_int_floordiv_ovf(x, y): - if x == -sys.maxint - 1 and y == -1: + # intentionally not short-circuited to produce only one guard + # and to remove the check fully if one of the arguments is known + if (x == -sys.maxint - 1) & (y == -1): raise OverflowError return llop.int_floordiv(lltype.Signed, x, y) @@ -263,12 +263,11 @@ def _ll_2_int_floordiv_zer(x, y): def _ll_2_int_mod_ovf_zer(x, y): if y == 0: raise ZeroDivisionError - if x == -sys.maxint - 1 and y == -1: - raise OverflowError - return llop.int_mod(lltype.Signed, x, y) + return _ll_2_int_mod_ovf(x, y) def _ll_2_int_mod_ovf(x, y): - if x == -sys.maxint - 1 and y == -1: + #see comment in _ll_2_int_floordiv_ovf + if (x == -sys.maxint - 1) & (y == -1): raise OverflowError return llop.int_mod(lltype.Signed, x, y) diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py index 47e5ff0950..20a69597a5 100644 --- a/rpython/jit/metainterp/blackhole.py +++ b/rpython/jit/metainterp/blackhole.py @@ -944,6 +944,14 @@ class BlackholeInterpreter(object): pass @arguments("i") + def bhimpl_jit_enter_portal_frame(x): + pass + + @arguments() + def bhimpl_jit_leave_portal_frame(): + pass + + @arguments("i") def bhimpl_int_assert_green(x): pass @arguments("r") diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py index 2081651bbc..b196fe341e 100644 --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -28,7 +28,7 @@ def test_store_final_boxes_in_guard(): # setup rd data fi0 = resume.FrameInfo(None, FakeJitCode(), 11) snapshot0 = resume.Snapshot(None, [b0]) - op.rd_snapshot = resume.Snapshot(snapshot0, [b1]) + op.rd_snapshot = resume.TopSnapshot(snapshot0, [], [b1]) op.rd_frame_info_list = resume.FrameInfo(fi0, FakeJitCode(), 33) # opt.store_final_boxes_in_guard(op, []) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py index 8c190a2b9b..2ac0e98d1f 100644 --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -506,14 +506,15 @@ class BaseTest(object): index = 0 if op.is_guard(): - op.rd_snapshot = resume.Snapshot(None, op.getfailargs()) + op.rd_snapshot = resume.TopSnapshot( + resume.Snapshot(None, op.getfailargs()), [], []) op.rd_frame_info_list = resume.FrameInfo(None, FakeJitCode(), 11) def add_guard_future_condition(self, res): # invent a GUARD_FUTURE_CONDITION to not have to change all tests if res.operations[-1].getopnum() == rop.JUMP: guard = ResOperation(rop.GUARD_FUTURE_CONDITION, [], None) - guard.rd_snapshot = resume.Snapshot(None, []) + guard.rd_snapshot = resume.TopSnapshot(None, [], []) res.operations.insert(-1, guard) def assert_equal(self, optimized, expected, text_right=None): diff --git a/rpython/jit/metainterp/optimizeopt/util.py b/rpython/jit/metainterp/optimizeopt/util.py index c434fb63d5..78bccaf865 100644 --- a/rpython/jit/metainterp/optimizeopt/util.py +++ b/rpython/jit/metainterp/optimizeopt/util.py @@ -8,7 +8,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp import resoperation from rpython.rlib.debug import make_sure_not_resized from rpython.jit.metainterp.resoperation import rop -from rpython.jit.metainterp.resume import Snapshot, AccumInfo +from rpython.jit.metainterp.resume import AccumInfo # ____________________________________________________________ # Misc. utilities diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py index 8bfb259e18..fe3521e995 100644 --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1358,6 +1358,17 @@ class MIFrame(object): self.metainterp.attach_debug_info(op) @arguments("box") + def opimpl_jit_enter_portal_frame(self, uniqueidbox): + unique_id = uniqueidbox.getint() + jd_no = self.metainterp.jitdriver_sd.mainjitcode.index # fish + self.metainterp.enter_portal_frame(jd_no, unique_id) + + @arguments() + def opimpl_jit_leave_portal_frame(self): + jd_no = self.metainterp.jitdriver_sd.mainjitcode.index # fish + self.metainterp.leave_portal_frame(jd_no) + + @arguments("box") def _opimpl_assert_green(self, box): if not isinstance(box, Const): msg = "assert_green failed at %s:%d" % ( diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py index e563896a91..21b8b55f7a 100644 --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -27,6 +27,13 @@ class Snapshot(object): self.prev = prev self.boxes = boxes +class TopSnapshot(Snapshot): + __slots__ = ('vable_boxes',) + + def __init__(self, prev, boxes, vable_boxes): + Snapshot.__init__(self, prev, boxes) + self.vable_boxes = vable_boxes + def combine_uint(index1, index2): assert 0 <= index1 < 65536 assert 0 <= index2 < 65536 @@ -127,9 +134,11 @@ def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, snapshot_storage): n = len(framestack) - 1 if virtualizable_boxes is not None: - boxes = virtualref_boxes + virtualizable_boxes + virtualizable_boxes = ([virtualizable_boxes[-1]] + + virtualizable_boxes[:-1]) else: - boxes = virtualref_boxes[:] + virtualizable_boxes = [] + virtualref_boxes = virtualref_boxes[:] if n >= 0: top = framestack[n] _ensure_parent_resumedata(framestack, n) @@ -138,11 +147,12 @@ def capture_resumedata(framestack, virtualizable_boxes, virtualref_boxes, snapshot_storage.rd_frame_info_list = frame_info_list snapshot = Snapshot(top.parent_resumedata_snapshot, top.get_list_of_active_boxes(False)) - snapshot = Snapshot(snapshot, boxes) + snapshot = TopSnapshot(snapshot, virtualref_boxes, virtualizable_boxes) snapshot_storage.rd_snapshot = snapshot else: snapshot_storage.rd_frame_info_list = None - snapshot_storage.rd_snapshot = Snapshot(None, boxes) + snapshot_storage.rd_snapshot = TopSnapshot(None, virtualref_boxes, + virtualizable_boxes) PENDINGFIELDSTRUCT = lltype.Struct('PendingField', ('lldescr', OBJECTPTR), @@ -200,10 +210,12 @@ class NumberingState(object): self.v = 0 def count_boxes(self, lst): - c = 0 + snapshot = lst[0] + assert isinstance(snapshot, TopSnapshot) + c = len(snapshot.vable_boxes) for snapshot in lst: c += len(snapshot.boxes) - c += 2 * (len(lst) - 1) + c += 2 * (len(lst) - 1) + 1 + 1 return c def append(self, item): @@ -294,13 +306,11 @@ class ResumeDataLoopMemo(object): state.append(tagged) state.n = n state.v = v - state.position -= length + 2 - def number(self, optimizer, snapshot, frameinfo): + def number(self, optimizer, topsnapshot, frameinfo): # flatten the list - vref_snapshot = snapshot - cur = snapshot.prev - snapshot_list = [vref_snapshot] + cur = topsnapshot.prev + snapshot_list = [topsnapshot] framestack_list = [] while cur: framestack_list.append(frameinfo) @@ -311,19 +321,30 @@ class ResumeDataLoopMemo(object): # we want to number snapshots starting from the back, but ending # with a forward list - for i in range(len(snapshot_list) - 1, -1, -1): - state.position -= len(snapshot_list[i].boxes) - if i != 0: - frameinfo = framestack_list[i - 1] - jitcode_pos, pc = unpack_uint(frameinfo.packed_jitcode_pc) - state.position -= 2 - state.append(rffi.cast(rffi.SHORT, jitcode_pos)) - state.append(rffi.cast(rffi.SHORT, pc)) + for i in range(len(snapshot_list) - 1, 0, -1): + state.position -= len(snapshot_list[i].boxes) + 2 + frameinfo = framestack_list[i - 1] + jitcode_pos, pc = unpack_uint(frameinfo.packed_jitcode_pc) + state.append(rffi.cast(rffi.SHORT, jitcode_pos)) + state.append(rffi.cast(rffi.SHORT, pc)) self._number_boxes(snapshot_list[i].boxes, optimizer, state) - - numb = resumecode.create_numbering(state.current, - len(vref_snapshot.boxes)) - + state.position -= len(snapshot_list[i].boxes) + 2 + + assert isinstance(topsnapshot, TopSnapshot) + special_boxes_size = (1 + len(topsnapshot.vable_boxes) + + 1 + len(topsnapshot.boxes)) + assert state.position == special_boxes_size + + state.position = 0 + state.append(rffi.cast(rffi.SHORT, len(topsnapshot.vable_boxes))) + self._number_boxes(topsnapshot.vable_boxes, optimizer, state) + n = len(topsnapshot.boxes) + assert not (n & 1) + state.append(rffi.cast(rffi.SHORT, n >> 1)) + self._number_boxes(topsnapshot.boxes, optimizer, state) + assert state.position == special_boxes_size + + numb = resumecode.create_numbering(state.current) return numb, state.liveboxes, state.v def forget_numberings(self): @@ -1113,48 +1134,42 @@ class ResumeDataBoxReader(AbstractResumeDataReader): self.boxes_f = boxes_f self._prepare_next_section(info) - def consume_virtualizable_boxes(self, vinfo): + def consume_virtualizable_boxes(self, vinfo, index): # we have to ignore the initial part of 'nums' (containing vrefs), # find the virtualizable from nums[-1], and use it to know how many # boxes of which type we have to return. This does not write # anything into the virtualizable. numb = self.numb - first_snapshot_size = rffi.cast(lltype.Signed, numb.first_snapshot_size) - item, _ = resumecode.numb_next_item(numb, first_snapshot_size - 1) + item, index = resumecode.numb_next_item(numb, index) virtualizablebox = self.decode_ref(item) - index = first_snapshot_size - vinfo.get_total_size(virtualizablebox.getref_base()) - 1 virtualizable = vinfo.unwrap_virtualizable_box(virtualizablebox) return vinfo.load_list_of_boxes(virtualizable, self, virtualizablebox, numb, index) - def consume_virtualref_boxes(self, end): + def consume_virtualref_boxes(self, index): # Returns a list of boxes, assumed to be all BoxPtrs. # We leave up to the caller to call vrefinfo.continue_tracing(). - assert (end & 1) == 0 + size, index = resumecode.numb_next_item(self.numb, index) + if size == 0: + return [], index lst = [] - self.cur_index = 0 - for i in range(end): - item, self.cur_index = resumecode.numb_next_item(self.numb, - self.cur_index) + for i in range(size * 2): + item, index = resumecode.numb_next_item(self.numb, index) lst.append(self.decode_ref(item)) - return lst + return lst, index def consume_vref_and_vable_boxes(self, vinfo, ginfo): - first_snapshot_size = rffi.cast(lltype.Signed, - self.numb.first_snapshot_size) + vable_size, index = resumecode.numb_next_item(self.numb, 0) if vinfo is not None: - virtualizable_boxes = self.consume_virtualizable_boxes(vinfo) - end = first_snapshot_size - len(virtualizable_boxes) + virtualizable_boxes, index = self.consume_virtualizable_boxes(vinfo, + index) elif ginfo is not None: - item, self.cur_index = resumecode.numb_next_item(self.numb, - first_snapshot_size - 1) + item, index = resumecode.numb_next_item(self.numb, index) virtualizable_boxes = [self.decode_ref(item)] - end = first_snapshot_size - 1 else: - end = first_snapshot_size virtualizable_boxes = None - virtualref_boxes = self.consume_virtualref_boxes(end) - self.cur_index = rffi.cast(lltype.Signed, self.numb.first_snapshot_size) + virtualref_boxes, index = self.consume_virtualref_boxes(index) + self.cur_index = index return virtualizable_boxes, virtualref_boxes def allocate_with_vtable(self, descr=None): @@ -1429,39 +1444,36 @@ class ResumeDataDirectReader(AbstractResumeDataReader): info = blackholeinterp.get_current_position_info() self._prepare_next_section(info) - def consume_virtualref_info(self, vrefinfo, end): + def consume_virtualref_info(self, vrefinfo, index): # we have to decode a list of references containing pairs - # [..., virtual, vref, ...] stopping at 'end' - if vrefinfo is None: - assert end == 0 - return - assert (end & 1) == 0 - self.cur_index = 0 - for i in range(0, end, 2): - virtual_item, self.cur_index = resumecode.numb_next_item( - self.numb, self.cur_index) - vref_item, self.cur_index = resumecode.numb_next_item( - self.numb, self.cur_index) + # [..., virtual, vref, ...] and returns the index at the end + size, index = resumecode.numb_next_item(self.numb, index) + if vrefinfo is None or size == 0: + assert size == 0 + return index + for i in range(size): + virtual_item, index = resumecode.numb_next_item( + self.numb, index) + vref_item, index = resumecode.numb_next_item( + self.numb, index) virtual = self.decode_ref(virtual_item) vref = self.decode_ref(vref_item) # For each pair, we store the virtual inside the vref. vrefinfo.continue_tracing(vref, virtual) + return index - def consume_vable_info(self, vinfo): + def consume_vable_info(self, vinfo, index): # we have to ignore the initial part of 'nums' (containing vrefs), # find the virtualizable from nums[-1], load all other values # from the CPU stack, and copy them into the virtualizable numb = self.numb - first_snapshot_size = rffi.cast(lltype.Signed, numb.first_snapshot_size) - item, _ = resumecode.numb_next_item(self.numb, - first_snapshot_size - 1) + item, index = resumecode.numb_next_item(self.numb, index) virtualizable = self.decode_ref(item) - start_index = first_snapshot_size - 1 - vinfo.get_total_size(virtualizable) # just reset the token, we'll force it later vinfo.reset_token_gcref(virtualizable) - vinfo.write_from_resume_data_partial(virtualizable, self, start_index, - numb) - return start_index + index = vinfo.write_from_resume_data_partial(virtualizable, self, + index, numb) + return index def load_value_of_type(self, TYPE, tagged): from rpython.jit.metainterp.warmstate import specialize_value @@ -1478,14 +1490,18 @@ class ResumeDataDirectReader(AbstractResumeDataReader): load_value_of_type._annspecialcase_ = 'specialize:arg(1)' def consume_vref_and_vable(self, vrefinfo, vinfo, ginfo): + vable_size, index = resumecode.numb_next_item(self.numb, 0) if self.resume_after_guard_not_forced != 2: - end_vref = rffi.cast(lltype.Signed, self.numb.first_snapshot_size) if vinfo is not None: - end_vref = self.consume_vable_info(vinfo) + index = self.consume_vable_info(vinfo, index) if ginfo is not None: - end_vref -= 1 - self.consume_virtualref_info(vrefinfo, end_vref) - self.cur_index = rffi.cast(lltype.Signed, self.numb.first_snapshot_size) + _, index = resumecode.numb_next_item(self.numb, index) + index = self.consume_virtualref_info(vrefinfo, index) + else: + index = resumecode.numb_next_n_items(self.numb, vable_size, index) + vref_size, index = resumecode.numb_next_item(self.numb, index) + index = resumecode.numb_next_n_items(self.numb, vref_size * 2, index) + self.cur_index = index def allocate_with_vtable(self, descr=None): from rpython.jit.metainterp.executor import exec_new_with_vtable diff --git a/rpython/jit/metainterp/resumecode.py b/rpython/jit/metainterp/resumecode.py index 43f65a92a3..8c2104da83 100644 --- a/rpython/jit/metainterp/resumecode.py +++ b/rpython/jit/metainterp/resumecode.py @@ -1,96 +1,74 @@ """ Resume bytecode. It goes as following: -<numb> <numb> <pc> <jitcode> <numb> <numb> <numb> <pc> <jitcode> + [<length> <virtualizable object> <numb> <numb> <numb>] if vinfo is not None + -OR- + [1 <ginfo object>] if ginfo is not None + -OR- + [0] if both are None -until the length of the array. + [<length> <virtual> <vref> <virtual> <vref>] for virtualrefs -The interface is only create_numbering/numb_next_item, but! there is a trick -that uses first_snapshot_size + some knowledge about inside to decode -virtualref/virtualizable_fields/virtualizable in that order in resume.py. + [<pc> <jitcode> <numb> <numb> <numb>] the frames + [<pc> <jitcode> <numb> <numb>] + ... -If the algorithm changes, the part about how to find where virtualizable -and virtualrefs are to be found + until the length of the array. """ from rpython.rtyper.lltypesystem import rffi, lltype NUMBERINGP = lltype.Ptr(lltype.GcForwardReference()) NUMBERING = lltype.GcStruct('Numbering', -# ('prev', NUMBERINGP), -# ('prev_index', rffi.USHORT), - ('first_snapshot_size', rffi.USHORT), # ugh, ugly - ('code', lltype.Array(rffi.SHORT))) + ('code', lltype.Array(rffi.UCHAR))) NUMBERINGP.TO.become(NUMBERING) NULL_NUMBER = lltype.nullptr(NUMBERING) -# this is the actually used version - -def create_numbering(lst, first_snapshot_size): - numb = lltype.malloc(NUMBERING, len(lst)) - for i in range(len(lst)): - numb.code[i] = rffi.cast(rffi.SHORT, lst[i]) - numb.first_snapshot_size = rffi.cast(rffi.USHORT, first_snapshot_size) - return numb - -def numb_next_item(numb, index): - return rffi.cast(lltype.Signed, numb.code[index]), index + 1 - -# this is the version that can be potentially used - -def _create_numbering(lst, prev, prev_index, first_snapshot_size): - count = 0 +def create_numbering(lst): + result = [] for item in lst: + item = rffi.cast(lltype.Signed, item) + item *= 2 if item < 0: - if item < -63: - count += 1 - if item > 127: - count += 1 - count += 1 - numb = lltype.malloc(NUMBERING, count) - numb.prev = prev - numb.prev_index = rffi.cast(rffi.USHORT, prev_index) - numb.first_snapshot_size = rffi.cast(rffi.USHORT, first_snapshot_size) - index = 0 - for item in lst: - if 0 <= item <= 128: - numb.code[index] = rffi.cast(rffi.UCHAR, item) - index += 1 + item = -1 - item + + assert item >= 0 + if item < 2**7: + result.append(rffi.cast(rffi.UCHAR, item)) + elif item < 2**14: + result.append(rffi.cast(rffi.UCHAR, item | 0x80)) + result.append(rffi.cast(rffi.UCHAR, item >> 7)) else: - assert (item >> 8) <= 63 - if item < 0: - item = -item - if item <= 63: - numb.code[index] = rffi.cast(rffi.UCHAR, item | 0x40) - index += 1 - else: - numb.code[index] = rffi.cast(rffi.UCHAR, (item >> 8) | 0x80 | 0x40) - numb.code[index + 1] = rffi.cast(rffi.UCHAR, item & 0xff) - index += 2 - else: - numb.code[index] = rffi.cast(rffi.UCHAR, (item >> 8) | 0x80) - numb.code[index + 1] = rffi.cast(rffi.UCHAR, item & 0xff) - index += 2 + assert item < 2**16 + result.append(rffi.cast(rffi.UCHAR, item | 0x80)) + result.append(rffi.cast(rffi.UCHAR, (item >> 7) | 0x80)) + result.append(rffi.cast(rffi.UCHAR, item >> 14)) + + numb = lltype.malloc(NUMBERING, len(result)) + for i in range(len(result)): + numb.code[i] = result[i] return numb -def copy_from_list_to_numb(lst, numb, index): - i = 0 - while i < len(lst): - numb.code[i + index] = lst[i] - i += 1 +def numb_next_item(numb, index): + value = rffi.cast(lltype.Signed, numb.code[index]) + index += 1 + if value & (2**7): + value &= 2**7 - 1 + value |= rffi.cast(lltype.Signed, numb.code[index]) << 7 + index += 1 + if value & (2**14): + value &= 2**14 - 1 + value |= rffi.cast(lltype.Signed, numb.code[index]) << 14 + index += 1 + if value & 1: + value = -1 - value + value >>= 1 + return value, index -def _numb_next_item(numb, index): - one = rffi.cast(lltype.Signed, numb.code[index]) - if one & 0x40: - if one & 0x80: - two = rffi.cast(lltype.Signed, numb.code[index + 1]) - return -(((one & ~(0x80 | 0x40)) << 8) | two), index + 2 - else: - return -(one & (~0x40)), index + 1 - if one & 0x80: - two = rffi.cast(lltype.Signed, numb.code[index + 1]) - return ((one & 0x7f) << 8) | two, index + 2 - return one, index + 1 +def numb_next_n_items(numb, size, index): + for i in range(size): + _, index = numb_next_item(numb, index) + return index def unpack_numbering(numb): l = [] diff --git a/rpython/jit/metainterp/test/strategies.py b/rpython/jit/metainterp/test/strategies.py new file mode 100644 index 0000000000..e7cd79ec20 --- /dev/null +++ b/rpython/jit/metainterp/test/strategies.py @@ -0,0 +1,13 @@ + +import sys +from hypothesis import strategies +from rpython.jit.metainterp.resoperation import InputArgInt +from rpython.jit.metainterp.history import ConstInt + +machine_ints = strategies.integers(min_value=-sys.maxint - 1, + max_value=sys.maxint) +intboxes = strategies.builds(InputArgInt) +intconsts = strategies.builds(ConstInt, machine_ints) +boxes = intboxes | intconsts +boxlists = strategies.lists(boxes, min_size=1).flatmap( + lambda cis: strategies.lists(strategies.sampled_from(cis)))
\ No newline at end of file diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py index fc4a231aca..ecf82bcbf6 100644 --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -1199,6 +1199,31 @@ class BasicTests: (-sys.maxint-1) // (-6) + 100 * 8) + def test_overflow_fold_if_divisor_constant(self): + import sys + from rpython.rtyper.lltypesystem.lloperation import llop + myjitdriver = JitDriver(greens = [], reds = ['x', 'y', 'res']) + def f(x, y): + res = 0 + while y > 0: + myjitdriver.can_enter_jit(x=x, y=y, res=res) + myjitdriver.jit_merge_point(x=x, y=y, res=res) + try: + res += llop.int_floordiv_ovf(lltype.Signed, + x, 2) + res += llop.int_mod_ovf(lltype.Signed, + x, 2) + x += 5 + except OverflowError: + res += 100 + y -= 1 + return res + res = self.meta_interp(f, [-41, 8]) + # the guard_true are for the loop condition + # the guard_false needed to check whether an overflow can occur have + # been folded away + self.check_resops(guard_true=2, guard_false=0) + def test_isinstance(self): class A: pass diff --git a/rpython/jit/metainterp/test/test_jitdriver.py b/rpython/jit/metainterp/test/test_jitdriver.py index 56f26ffd85..cba873a956 100644 --- a/rpython/jit/metainterp/test/test_jitdriver.py +++ b/rpython/jit/metainterp/test/test_jitdriver.py @@ -213,6 +213,21 @@ class MultipleJitDriversTests(object): if op.getopname() == 'enter_portal_frame': assert op.getarg(0).getint() == 0 assert op.getarg(1).getint() == 1 - + + def test_manual_leave_enter_portal_frame(self): + from rpython.rlib import jit + driver = JitDriver(greens=[], reds='auto', is_recursive=True) + + def f(arg): + i = 0 + while i < 100: + driver.jit_merge_point() + jit.enter_portal_frame(42) + jit.leave_portal_frame() + i += 1 + + self.meta_interp(f, [0]) + self.check_simple_loop(enter_portal_frame=1, leave_portal_frame=1) + class TestLLtype(MultipleJitDriversTests, LLJitMixin): pass diff --git a/rpython/jit/metainterp/test/test_resume.py b/rpython/jit/metainterp/test/test_resume.py index ebbd602860..dd2a9f501d 100644 --- a/rpython/jit/metainterp/test/test_resume.py +++ b/rpython/jit/metainterp/test/test_resume.py @@ -10,7 +10,7 @@ from rpython.jit.metainterp.resume import ResumeDataVirtualAdder,\ VArrayInfoNotClear, VStrPlainInfo, VStrConcatInfo, VStrSliceInfo,\ VUniPlainInfo, VUniConcatInfo, VUniSliceInfo, Snapshot, FrameInfo,\ capture_resumedata, ResumeDataLoopMemo, UNASSIGNEDVIRTUAL, INT,\ - annlowlevel, PENDINGFIELDSP, unpack_uint, TAG_CONST_OFFSET + annlowlevel, PENDINGFIELDSP, unpack_uint, TAG_CONST_OFFSET, TopSnapshot from rpython.jit.metainterp.resumecode import unpack_numbering,\ create_numbering, NULL_NUMBER @@ -22,8 +22,12 @@ from rpython.jit.metainterp import executor from rpython.jit.codewriter import heaptracker, longlong from rpython.jit.metainterp.resoperation import ResOperation, InputArgInt,\ InputArgRef, rop +from rpython.jit.metainterp.test.strategies import boxlists from rpython.rlib.debug import debug_start, debug_stop, debug_print,\ have_debug_prints +from rpython.jit.metainterp import resumecode + +from hypothesis import given class Storage: rd_frame_info_list = None @@ -278,9 +282,7 @@ def _next_section(reader, *expected): assert bh.written_f == expected_f -def Numbering(nums): - numb = create_numbering(nums, 0) - return numb +Numbering = create_numbering def tagconst(i): return tag(i + TAG_CONST_OFFSET, TAGCONST) @@ -610,7 +612,8 @@ def test_capture_resumedata(): assert unpack_uint(frame_info_list.packed_jitcode_pc) == (2, 15) snapshot = storage.rd_snapshot - assert snapshot.boxes == vrs + vbs # in the same list + assert snapshot.boxes == vrs + assert snapshot.vable_boxes == [b2, b1] snapshot = snapshot.prev assert snapshot.prev is fs[2].parent_resumedata_snapshot @@ -904,9 +907,9 @@ def test_ResumeDataLoopMemo_number(): env = [b1, c1, b2, b1, c2] snap = Snapshot(None, env) env1 = [c3, b3, b1, c1] - snap1 = Snapshot(snap, env1) + snap1 = TopSnapshot(snap, env1, []) env2 = [c3, b3, b1, c3] - snap2 = Snapshot(snap, env2) + snap2 = TopSnapshot(snap, env2, []) memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) frameinfo = FrameInfo(None, FakeJitCode("jitcode", 0), 0) @@ -916,10 +919,11 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} - base = [tag(0, TAGBOX), tag(1, TAGINT), tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)] + base = [0, 0, tag(0, TAGBOX), tag(1, TAGINT), + tag(1, TAGBOX), tag(0, TAGBOX), tag(2, TAGINT)] - assert unpack_numbering(numb) == [ - tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(1, TAGINT), 0, 0] + base + assert unpack_numbering(numb) == [0, 2, tag(3, TAGINT), tag(2, TAGBOX), + tag(0, TAGBOX), tag(1, TAGINT)] + base numb2, liveboxes2, v = memo.number(FakeOptimizer(), snap2, frameinfo) assert v == 0 @@ -927,11 +931,11 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes2 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b3: tag(2, TAGBOX)} assert liveboxes2 is not liveboxes - assert unpack_numbering(numb2) == [ - tag(3, TAGINT), tag(2, TAGBOX), tag(0, TAGBOX), tag(3, TAGINT), 0, 0] + base + assert unpack_numbering(numb2) == [0, 2, tag(3, TAGINT), tag(2, TAGBOX), + tag(0, TAGBOX), tag(3, TAGINT)] + base env3 = [c3, b3, b1, c3] - snap3 = Snapshot(snap, env3) + snap3 = TopSnapshot(snap, env3, []) class FakeVirtualInfo(info.AbstractInfo): def __init__(self, virt): @@ -946,13 +950,12 @@ def test_ResumeDataLoopMemo_number(): assert v == 0 assert liveboxes3 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX)} - assert unpack_numbering(numb3) == [tag(3, TAGINT), tag(4, TAGINT), - tag(0, TAGBOX), - tag(3, TAGINT), 0, 0] + base + assert unpack_numbering(numb3) == [0, 2, tag(3, TAGINT), tag(4, TAGINT), + tag(0, TAGBOX), tag(3, TAGINT)] + base # virtual env4 = [c3, b4, b1, c3] - snap4 = Snapshot(snap, env4) + snap4 = TopSnapshot(snap, env4, []) b4.set_forwarded(FakeVirtualInfo(True)) numb4, liveboxes4, v = memo.number(FakeOptimizer(), snap4, frameinfo) @@ -960,11 +963,11 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes4 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL)} - assert unpack_numbering(numb4) == [tag(3, TAGINT), tag(0, TAGVIRTUAL), - tag(0, TAGBOX), tag(3, TAGINT), 0, 0] + base + assert unpack_numbering(numb4) == [0, 2, tag(3, TAGINT), tag(0, TAGVIRTUAL), + tag(0, TAGBOX), tag(3, TAGINT)] + base env5 = [b1, b4, b5] - snap5 = Snapshot(snap4, env5) + snap5 = TopSnapshot(snap4, [], env5) b4.set_forwarded(FakeVirtualInfo(True)) b5.set_forwarded(FakeVirtualInfo(True)) @@ -974,9 +977,30 @@ def test_ResumeDataLoopMemo_number(): assert liveboxes5 == {b1: tag(0, TAGBOX), b2: tag(1, TAGBOX), b4: tag(0, TAGVIRTUAL), b5: tag(1, TAGVIRTUAL)} - assert unpack_numbering(numb5) == [tag(0, TAGBOX), tag(0, TAGVIRTUAL), - tag(1, TAGVIRTUAL), 2, 1] + unpack_numbering(numb4) - + assert unpack_numbering(numb5) == [ + 3, tag(0, TAGBOX), tag(0, TAGVIRTUAL), tag(1, TAGVIRTUAL), + 0, + 2, 1, tag(3, TAGINT), tag(0, TAGVIRTUAL), tag(0, TAGBOX), tag(3, TAGINT) + ] + base + +@given(boxlists) +def test_ResumeDataLoopMemo_random(lst): + s = TopSnapshot(None, [], lst) + frameinfo = FrameInfo(None, FakeJitCode("foo", 0), 0) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + num, liveboxes, v = memo.number(FakeOptimizer(), s, frameinfo) + l = unpack_numbering(num) + assert l[-1] == 0 + assert l[0] == len(lst) + for i, item in enumerate(lst): + v, tag = untag(l[i + 1]) + if tag == TAGBOX: + assert l[i + 1] == liveboxes[item] + elif tag == TAGCONST: + assert memo.consts[v].getint() == item.getint() + elif tag == TAGINT: + assert v == item.getint() + def test_ResumeDataLoopMemo_number_boxes(): memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) b1, b2 = [InputArgInt(), InputArgInt()] @@ -1060,10 +1084,11 @@ def make_storage(b1, b2, b3): storage = Storage() snapshot = Snapshot(None, [b1, ConstInt(1), b1, b2]) snapshot = Snapshot(snapshot, [ConstInt(2), ConstInt(3)]) - snapshot = Snapshot(snapshot, [b1, b2, b3]) - frameinfo = FrameInfo(FrameInfo(None, FakeJitCode("code1", 21), 22), - FakeJitCode("code2", 31), 32) - storage.rd_snapshot = snapshot + snapshot = Snapshot(snapshot, [b1, b2, b3]) + top_snapshot = TopSnapshot(snapshot, [], []) + frameinfo = FrameInfo(FrameInfo(FrameInfo(None, FakeJitCode("code1", 21), 22), + FakeJitCode("code2", 31), 32), FakeJitCode("code3", 41), 42) + storage.rd_snapshot = top_snapshot storage.rd_frame_info_list = frameinfo return storage @@ -1076,6 +1101,8 @@ def test_virtual_adder_int_constants(): assert storage.rd_snapshot is None cpu = MyCPU([]) reader = ResumeDataDirectReader(MyMetaInterp(cpu), storage, "deadframe") + reader.consume_vref_and_vable(None, None, None) + reader.cur_index += 2 # framestack _next_section(reader, sys.maxint, 2**16, -65) reader.cur_index += 2 # framestack _next_section(reader, 2, 3) @@ -1116,7 +1143,8 @@ class ResumeDataFakeReader(ResumeDataBoxReader): class MyInfo: @staticmethod def enumerate_vars(callback_i, callback_r, callback_f, _, index): - for tagged in self.numb.code: + while index < len(self.numb.code): + tagged, _ = resumecode.numb_next_item(self.numb, index) _, tag = untag(tagged) if tag == TAGVIRTUAL: kind = REF @@ -1131,6 +1159,13 @@ class ResumeDataFakeReader(ResumeDataBoxReader): index = callback_f(index, index) else: assert 0 + size, self.cur_index = resumecode.numb_next_item(self.numb, 0) + assert size == 0 + size, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index) + assert size == 0 + pc, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index) + jitcode_pos, self.cur_index = resumecode.numb_next_item(self.numb, self.cur_index) + self._prepare_next_section(MyInfo()) return self.lst diff --git a/rpython/jit/metainterp/test/test_resumecode.py b/rpython/jit/metainterp/test/test_resumecode.py index 848a4f1009..f65fae7d07 100644 --- a/rpython/jit/metainterp/test/test_resumecode.py +++ b/rpython/jit/metainterp/test/test_resumecode.py @@ -1,9 +1,12 @@ from rpython.jit.metainterp.resumecode import NUMBERING, NULL_NUMBER from rpython.jit.metainterp.resumecode import create_numbering,\ - unpack_numbering, copy_from_list_to_numb + unpack_numbering from rpython.rtyper.lltypesystem import lltype +from hypothesis import strategies, given + + def test_pack_unpack(): examples = [ [1, 2, 3, 4, 257, 10000, 13, 15], @@ -12,5 +15,15 @@ def test_pack_unpack(): [13000, 12000, 10000, 256, 255, 254, 257, -3, -1000] ] for l in examples: - n = create_numbering(l, 0) + n = create_numbering(l) assert unpack_numbering(n) == l + +@given(strategies.lists(strategies.integers(-2**15, 2**15-1))) +def test_roundtrip(l): + n = create_numbering(l) + assert unpack_numbering(n) == l + +@given(strategies.lists(strategies.integers(-2**15, 2**15-1))) +def test_compressing(l): + n = create_numbering(l) + assert len(n.code) <= len(l) * 3 diff --git a/rpython/jit/metainterp/test/test_tlc.py b/rpython/jit/metainterp/test/test_tlc.py index ee88035bf2..5c1396d533 100644 --- a/rpython/jit/metainterp/test/test_tlc.py +++ b/rpython/jit/metainterp/test/test_tlc.py @@ -1,5 +1,4 @@ import py -from rpython.rtyper.module.support import LLSupport from rpython.jit.tl import tlc diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py index 08f2ec2059..3278ff0e84 100644 --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -142,6 +142,7 @@ class VirtualizableInfo(object): item, index = numb_next_item(numb, index) x = reader.load_value_of_type(ARRAYITEMTYPE, item) setarrayitem(lst, j, x) + return index def load_list_of_boxes(virtualizable, reader, vable_box, numb, index): virtualizable = cast_gcref_to_vtype(virtualizable) @@ -161,7 +162,7 @@ class VirtualizableInfo(object): box = reader.decode_box_of_type(ARRAYITEMTYPE, item) boxes.append(box) boxes.append(vable_box) - return boxes + return boxes, index def check_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py index 6c04f077c6..7e9999fb6d 100644 --- a/rpython/memory/gc/incminimark.py +++ b/rpython/memory/gc/incminimark.py @@ -706,6 +706,7 @@ class IncrementalMiniMarkGC(MovingGCBase): self.major_collection_step() else: self.minor_and_major_collection() + self.rrc_invoke_callback() def collect_and_reserve(self, totalsize): @@ -783,12 +784,15 @@ class IncrementalMiniMarkGC(MovingGCBase): self.threshold_reached()): # ^^but only if still self.minor_collection() # the same collection self.major_collection_step() - # - # The nursery might not be empty now, because of - # execute_finalizers(). If it is almost full again, - # we need to fix it with another call to minor_collection(). - if self.nursery_free + totalsize > self.nursery_top: - self.minor_collection() + # + self.rrc_invoke_callback() + # + # The nursery might not be empty now, because of + # execute_finalizers() or rrc_invoke_callback(). + # If it is almost full again, + # we need to fix it with another call to minor_collection(). + if self.nursery_free + totalsize > self.nursery_top: + self.minor_collection() # else: ll_assert(minor_collection_count == 2, @@ -861,6 +865,7 @@ class IncrementalMiniMarkGC(MovingGCBase): if self.threshold_reached(raw_malloc_usage(totalsize) + self.nursery_size // 2): self.major_collection_step(raw_malloc_usage(totalsize)) + self.rrc_invoke_callback() # note that this loop should not be infinite: when the # last step of a major collection is done but # threshold_reached(totalsize) is still true, then @@ -1080,35 +1085,19 @@ class IncrementalMiniMarkGC(MovingGCBase): "odd-valued (i.e. tagged) pointer unexpected here") return self.nursery <= addr < self.nursery + self.nursery_size - def appears_to_be_young(self, addr): - # "is a valid addr to a young object?" - # but it's ok to occasionally return True accidentally. - # Maybe the best implementation would be a bloom filter - # of some kind instead of the dictionary lookup that is - # sometimes done below. But the expected common answer - # is "Yes" because addr points to the nursery, so it may - # not be useful to optimize the other case too much. - # - # First, if 'addr' appears to be a pointer to some place within - # the nursery, return True - if not self.translated_to_c: - # When non-translated, filter out tagged pointers explicitly. - # When translated, it may occasionally give a wrong answer - # of True if 'addr' is a tagged pointer with just the wrong value. - if not self.is_valid_gc_object(addr): - return False - + def is_young_object(self, addr): + # Check if the object at 'addr' is young. + if not self.is_valid_gc_object(addr): + return False # filter out tagged pointers explicitly. if self.nursery <= addr < self.nursery_top: return True # addr is in the nursery - # # Else, it may be in the set 'young_rawmalloced_objects' return (bool(self.young_rawmalloced_objects) and self.young_rawmalloced_objects.contains(addr)) - appears_to_be_young._always_inline_ = True def debug_is_old_object(self, addr): return (self.is_valid_gc_object(addr) - and not self.appears_to_be_young(addr)) + and not self.is_young_object(addr)) def is_forwarded(self, obj): """Returns True if the nursery obj is marked as forwarded. @@ -1618,6 +1607,10 @@ class IncrementalMiniMarkGC(MovingGCBase): self._visit_old_objects_pointing_to_pinned, None) current_old_objects_pointing_to_pinned.delete() # + # visit the P list from rawrefcount, if enabled. + if self.rrc_enabled: + self.rrc_minor_collection_trace() + # while True: # If we are using card marking, do a partial trace of the arrays # that are flagged with GCFLAG_CARDS_SET. @@ -1666,6 +1659,10 @@ class IncrementalMiniMarkGC(MovingGCBase): if self.young_rawmalloced_objects: self.free_young_rawmalloced_objects() # + # visit the P and O lists from rawrefcount, if enabled. + if self.rrc_enabled: + self.rrc_minor_collection_free() + # # All live nursery objects are out of the nursery or pinned inside # the nursery. Create nursery barriers to protect the pinned objects, # fill the rest of the nursery with zeros and reset the current nursery @@ -2178,9 +2175,13 @@ class IncrementalMiniMarkGC(MovingGCBase): # finalizers/weak references are rare and short which means that # they do not need a separate state and do not need to be # made incremental. + # For now, the same applies to rawrefcount'ed objects. if (not self.objects_to_trace.non_empty() and not self.more_objects_to_trace.non_empty()): # + if self.rrc_enabled: + self.rrc_major_collection_trace() + # if self.objects_with_finalizers.non_empty(): self.deal_with_objects_with_finalizers() elif self.old_objects_with_weakrefs.non_empty(): @@ -2215,6 +2216,10 @@ class IncrementalMiniMarkGC(MovingGCBase): self.old_objects_pointing_to_pinned = \ new_old_objects_pointing_to_pinned self.updated_old_objects_pointing_to_pinned = True + # + if self.rrc_enabled: + self.rrc_major_collection_free() + # self.gc_state = STATE_SWEEPING #END MARKING elif self.gc_state == STATE_SWEEPING: @@ -2745,3 +2750,238 @@ class IncrementalMiniMarkGC(MovingGCBase): (obj + offset).address[0] = llmemory.NULL self.old_objects_with_weakrefs.delete() self.old_objects_with_weakrefs = new_with_weakref + + + # ---------- + # RawRefCount + + rrc_enabled = False + + _ADDRARRAY = lltype.Array(llmemory.Address, hints={'nolength': True}) + PYOBJ_HDR = lltype.Struct('GCHdr_PyObject', + ('ob_refcnt', lltype.Signed), + ('ob_pypy_link', lltype.Signed)) + PYOBJ_HDR_PTR = lltype.Ptr(PYOBJ_HDR) + RAWREFCOUNT_DEALLOC_TRIGGER = lltype.Ptr(lltype.FuncType([], lltype.Void)) + + def _pyobj(self, pyobjaddr): + return llmemory.cast_adr_to_ptr(pyobjaddr, self.PYOBJ_HDR_PTR) + + def rawrefcount_init(self, dealloc_trigger_callback): + # see pypy/doc/discussion/rawrefcount.rst + if not self.rrc_enabled: + self.rrc_p_list_young = self.AddressStack() + self.rrc_p_list_old = self.AddressStack() + self.rrc_o_list_young = self.AddressStack() + self.rrc_o_list_old = self.AddressStack() + self.rrc_p_dict = self.AddressDict() # non-nursery keys only + self.rrc_p_dict_nurs = self.AddressDict() # nursery keys only + p = lltype.malloc(self._ADDRARRAY, 1, flavor='raw', + track_allocation=False) + self.rrc_singleaddr = llmemory.cast_ptr_to_adr(p) + self.rrc_dealloc_trigger_callback = dealloc_trigger_callback + self.rrc_dealloc_pending = self.AddressStack() + self.rrc_enabled = True + + def check_no_more_rawrefcount_state(self): + "NOT_RPYTHON: for tests" + assert self.rrc_p_list_young.length() == 0 + assert self.rrc_p_list_old .length() == 0 + assert self.rrc_o_list_young.length() == 0 + assert self.rrc_o_list_old .length() == 0 + def check_value_is_null(key, value, ignore): + assert value == llmemory.NULL + self.rrc_p_dict.foreach(check_value_is_null, None) + self.rrc_p_dict_nurs.foreach(check_value_is_null, None) + + def rawrefcount_create_link_pypy(self, gcobj, pyobject): + ll_assert(self.rrc_enabled, "rawrefcount.init not called") + obj = llmemory.cast_ptr_to_adr(gcobj) + objint = llmemory.cast_adr_to_int(obj, "symbolic") + self._pyobj(pyobject).ob_pypy_link = objint + # + lst = self.rrc_p_list_young + if self.is_in_nursery(obj): + dct = self.rrc_p_dict_nurs + else: + dct = self.rrc_p_dict + if not self.is_young_object(obj): + lst = self.rrc_p_list_old + lst.append(pyobject) + dct.setitem(obj, pyobject) + + def rawrefcount_create_link_pyobj(self, gcobj, pyobject): + ll_assert(self.rrc_enabled, "rawrefcount.init not called") + obj = llmemory.cast_ptr_to_adr(gcobj) + if self.is_young_object(obj): + self.rrc_o_list_young.append(pyobject) + else: + self.rrc_o_list_old.append(pyobject) + objint = llmemory.cast_adr_to_int(obj, "symbolic") + self._pyobj(pyobject).ob_pypy_link = objint + # there is no rrc_o_dict + + def rawrefcount_from_obj(self, gcobj): + obj = llmemory.cast_ptr_to_adr(gcobj) + if self.is_in_nursery(obj): + dct = self.rrc_p_dict_nurs + else: + dct = self.rrc_p_dict + return dct.get(obj) + + def rawrefcount_to_obj(self, pyobject): + obj = llmemory.cast_int_to_adr(self._pyobj(pyobject).ob_pypy_link) + return llmemory.cast_adr_to_ptr(obj, llmemory.GCREF) + + def rawrefcount_next_dead(self): + if self.rrc_dealloc_pending.non_empty(): + return self.rrc_dealloc_pending.pop() + return llmemory.NULL + + + def rrc_invoke_callback(self): + if self.rrc_enabled and self.rrc_dealloc_pending.non_empty(): + self.rrc_dealloc_trigger_callback() + + def rrc_minor_collection_trace(self): + length_estimate = self.rrc_p_dict_nurs.length() + self.rrc_p_dict_nurs.delete() + self.rrc_p_dict_nurs = self.AddressDict(length_estimate) + self.rrc_p_list_young.foreach(self._rrc_minor_trace, + self.rrc_singleaddr) + + def _rrc_minor_trace(self, pyobject, singleaddr): + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT + # + rc = self._pyobj(pyobject).ob_refcnt + if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT: + pass # the corresponding object may die + else: + # force the corresponding object to be alive + intobj = self._pyobj(pyobject).ob_pypy_link + singleaddr.address[0] = llmemory.cast_int_to_adr(intobj) + self._trace_drag_out(singleaddr, llmemory.NULL) + + def rrc_minor_collection_free(self): + ll_assert(self.rrc_p_dict_nurs.length() == 0, "p_dict_nurs not empty 1") + lst = self.rrc_p_list_young + while lst.non_empty(): + self._rrc_minor_free(lst.pop(), self.rrc_p_list_old, + self.rrc_p_dict) + lst = self.rrc_o_list_young + no_o_dict = self.null_address_dict() + while lst.non_empty(): + self._rrc_minor_free(lst.pop(), self.rrc_o_list_old, + no_o_dict) + + def _rrc_minor_free(self, pyobject, surviving_list, surviving_dict): + intobj = self._pyobj(pyobject).ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + if self.is_in_nursery(obj): + if self.is_forwarded(obj): + # Common case: survives and moves + obj = self.get_forwarding_address(obj) + intobj = llmemory.cast_adr_to_int(obj, "symbolic") + self._pyobj(pyobject).ob_pypy_link = intobj + surviving = True + if surviving_dict: + # Surviving nursery object: was originally in + # rrc_p_dict_nurs and now must be put into rrc_p_dict + surviving_dict.setitem(obj, pyobject) + else: + surviving = False + elif (bool(self.young_rawmalloced_objects) and + self.young_rawmalloced_objects.contains(obj)): + # young weakref to a young raw-malloced object + if self.header(obj).tid & GCFLAG_VISITED_RMY: + surviving = True # survives, but does not move + else: + surviving = False + if surviving_dict: + # Dying young large object: was in rrc_p_dict, + # must be deleted + surviving_dict.setitem(obj, llmemory.NULL) + else: + ll_assert(False, "rrc_X_list_young contains non-young obj") + return + # + if surviving: + surviving_list.append(pyobject) + else: + self._rrc_free(pyobject) + + def _rrc_free(self, pyobject): + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT + # + rc = self._pyobj(pyobject).ob_refcnt + if rc >= REFCNT_FROM_PYPY_LIGHT: + rc -= REFCNT_FROM_PYPY_LIGHT + if rc == 0: + lltype.free(self._pyobj(pyobject), flavor='raw') + else: + # can only occur if LIGHT is used in create_link_pyobj() + self._pyobj(pyobject).ob_refcnt = rc + self._pyobj(pyobject).ob_pypy_link = 0 + else: + ll_assert(rc >= REFCNT_FROM_PYPY, "refcount underflow?") + ll_assert(rc < int(REFCNT_FROM_PYPY_LIGHT * 0.99), + "refcount underflow from REFCNT_FROM_PYPY_LIGHT?") + rc -= REFCNT_FROM_PYPY + self._pyobj(pyobject).ob_refcnt = rc + self._pyobj(pyobject).ob_pypy_link = 0 + if rc == 0: + self.rrc_dealloc_pending.append(pyobject) + _rrc_free._always_inline_ = True + + def rrc_major_collection_trace(self): + self.rrc_p_list_old.foreach(self._rrc_major_trace, None) + + def _rrc_major_trace(self, pyobject, ignore): + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY + from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT + # + rc = self._pyobj(pyobject).ob_refcnt + if rc == REFCNT_FROM_PYPY or rc == REFCNT_FROM_PYPY_LIGHT: + pass # the corresponding object may die + else: + # force the corresponding object to be alive + intobj = self._pyobj(pyobject).ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + self.objects_to_trace.append(obj) + self.visit_all_objects() + + def rrc_major_collection_free(self): + ll_assert(self.rrc_p_dict_nurs.length() == 0, "p_dict_nurs not empty 2") + length_estimate = self.rrc_p_dict.length() + self.rrc_p_dict.delete() + self.rrc_p_dict = new_p_dict = self.AddressDict(length_estimate) + new_p_list = self.AddressStack() + while self.rrc_p_list_old.non_empty(): + self._rrc_major_free(self.rrc_p_list_old.pop(), new_p_list, + new_p_dict) + self.rrc_p_list_old.delete() + self.rrc_p_list_old = new_p_list + # + new_o_list = self.AddressStack() + no_o_dict = self.null_address_dict() + while self.rrc_o_list_old.non_empty(): + self._rrc_major_free(self.rrc_o_list_old.pop(), new_o_list, + no_o_dict) + self.rrc_o_list_old.delete() + self.rrc_o_list_old = new_o_list + + def _rrc_major_free(self, pyobject, surviving_list, surviving_dict): + # The pyobject survives if the corresponding obj survives. + # This is true if the obj has one of the following two flags: + # * GCFLAG_VISITED: was seen during tracing + # * GCFLAG_NO_HEAP_PTRS: immortal object never traced (so far) + intobj = self._pyobj(pyobject).ob_pypy_link + obj = llmemory.cast_int_to_adr(intobj) + if self.header(obj).tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS): + surviving_list.append(pyobject) + if surviving_dict: + surviving_dict.insertclean(obj, pyobject) + else: + self._rrc_free(pyobject) diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py new file mode 100644 index 0000000000..aca61015b8 --- /dev/null +++ b/rpython/memory/gc/test/test_rawrefcount.py @@ -0,0 +1,282 @@ +import py +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.memory.gc.incminimark import IncrementalMiniMarkGC +from rpython.memory.gc.test.test_direct import BaseDirectGCTest +from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY +from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY_LIGHT + +PYOBJ_HDR = IncrementalMiniMarkGC.PYOBJ_HDR +PYOBJ_HDR_PTR = IncrementalMiniMarkGC.PYOBJ_HDR_PTR + +S = lltype.GcForwardReference() +S.become(lltype.GcStruct('S', + ('x', lltype.Signed), + ('prev', lltype.Ptr(S)), + ('next', lltype.Ptr(S)))) + + +class TestRawRefCount(BaseDirectGCTest): + GCClass = IncrementalMiniMarkGC + + def _collect(self, major, expected_trigger=0): + if major: + self.gc.collect() + else: + self.gc.minor_collection() + count1 = len(self.trigger) + self.gc.rrc_invoke_callback() + count2 = len(self.trigger) + assert count2 - count1 == expected_trigger + + def _rawrefcount_pair(self, intval, is_light=False, is_pyobj=False, + create_old=False, create_immortal=False): + if is_light: + rc = REFCNT_FROM_PYPY_LIGHT + else: + rc = REFCNT_FROM_PYPY + self.trigger = [] + self.gc.rawrefcount_init(lambda: self.trigger.append(1)) + # + if create_immortal: + p1 = lltype.malloc(S, immortal=True) + else: + p1 = self.malloc(S) + p1.x = intval + if create_immortal: + self.consider_constant(p1) + elif create_old: + self.stackroots.append(p1) + self._collect(major=False) + p1 = self.stackroots.pop() + p1ref = lltype.cast_opaque_ptr(llmemory.GCREF, p1) + r1 = lltype.malloc(PYOBJ_HDR, flavor='raw', immortal=create_immortal) + r1.ob_refcnt = rc + r1.ob_pypy_link = 0 + r1addr = llmemory.cast_ptr_to_adr(r1) + if is_pyobj: + assert not is_light + self.gc.rawrefcount_create_link_pyobj(p1ref, r1addr) + else: + self.gc.rawrefcount_create_link_pypy(p1ref, r1addr) + assert r1.ob_refcnt == rc + assert r1.ob_pypy_link != 0 + + def check_alive(extra_refcount): + assert r1.ob_refcnt == rc + extra_refcount + assert r1.ob_pypy_link != 0 + p1ref = self.gc.rawrefcount_to_obj(r1addr) + p1 = lltype.cast_opaque_ptr(lltype.Ptr(S), p1ref) + assert p1.x == intval + if not is_pyobj: + assert self.gc.rawrefcount_from_obj(p1ref) == r1addr + else: + assert self.gc.rawrefcount_from_obj(p1ref) == llmemory.NULL + return p1 + return p1, p1ref, r1, r1addr, check_alive + + def test_rawrefcount_objects_basic(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_light=True, create_old=old)) + p2 = self.malloc(S) + p2.x = 84 + p2ref = lltype.cast_opaque_ptr(llmemory.GCREF, p2) + r2 = lltype.malloc(PYOBJ_HDR, flavor='raw') + r2.ob_refcnt = 1 + r2.ob_pypy_link = 0 + r2addr = llmemory.cast_ptr_to_adr(r2) + # p2 and r2 are not linked + assert r1.ob_pypy_link != 0 + assert r2.ob_pypy_link == 0 + assert self.gc.rawrefcount_from_obj(p1ref) == r1addr + assert self.gc.rawrefcount_from_obj(p2ref) == llmemory.NULL + assert self.gc.rawrefcount_to_obj(r1addr) == p1ref + assert self.gc.rawrefcount_to_obj(r2addr) == lltype.nullptr( + llmemory.GCREF.TO) + lltype.free(r1, flavor='raw') + lltype.free(r2, flavor='raw') + + def test_rawrefcount_objects_collection_survives_from_raw(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_light=True, create_old=old)) + check_alive(0) + r1.ob_refcnt += 1 + self._collect(major=False) + check_alive(+1) + self._collect(major=True) + check_alive(+1) + r1.ob_refcnt -= 1 + self._collect(major=False) + p1 = check_alive(0) + self._collect(major=True) + py.test.raises(RuntimeError, "r1.ob_refcnt") # dead + py.test.raises(RuntimeError, "p1.x") # dead + self.gc.check_no_more_rawrefcount_state() + assert self.trigger == [] + assert self.gc.rawrefcount_next_dead() == llmemory.NULL + + def test_rawrefcount_dies_quickly(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_light=True, create_old=old)) + check_alive(0) + self._collect(major=False) + if old: + check_alive(0) + self._collect(major=True) + py.test.raises(RuntimeError, "r1.ob_refcnt") # dead + py.test.raises(RuntimeError, "p1.x") # dead + self.gc.check_no_more_rawrefcount_state() + + def test_rawrefcount_objects_collection_survives_from_obj(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_light=True, create_old=old)) + check_alive(0) + self.stackroots.append(p1) + self._collect(major=False) + check_alive(0) + self._collect(major=True) + check_alive(0) + p1 = self.stackroots.pop() + self._collect(major=False) + check_alive(0) + assert p1.x == 42 + self._collect(major=True) + py.test.raises(RuntimeError, "r1.ob_refcnt") # dead + py.test.raises(RuntimeError, "p1.x") # dead + self.gc.check_no_more_rawrefcount_state() + + def test_rawrefcount_objects_basic_old(self): + self.test_rawrefcount_objects_basic(old=True) + def test_rawrefcount_objects_collection_survives_from_raw_old(self): + self.test_rawrefcount_objects_collection_survives_from_raw(old=True) + def test_rawrefcount_dies_quickly_old(self): + self.test_rawrefcount_dies_quickly(old=True) + def test_rawrefcount_objects_collection_survives_from_obj_old(self): + self.test_rawrefcount_objects_collection_survives_from_obj(old=True) + + def test_pypy_nonlight_survives_from_raw(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_light=False, create_old=old)) + check_alive(0) + r1.ob_refcnt += 1 + self._collect(major=False) + check_alive(+1) + self._collect(major=True) + check_alive(+1) + r1.ob_refcnt -= 1 + self._collect(major=False) + p1 = check_alive(0) + self._collect(major=True, expected_trigger=1) + py.test.raises(RuntimeError, "p1.x") # dead + assert r1.ob_refcnt == 0 + assert r1.ob_pypy_link == 0 + assert self.gc.rawrefcount_next_dead() == r1addr + assert self.gc.rawrefcount_next_dead() == llmemory.NULL + assert self.gc.rawrefcount_next_dead() == llmemory.NULL + self.gc.check_no_more_rawrefcount_state() + lltype.free(r1, flavor='raw') + + def test_pypy_nonlight_survives_from_obj(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_light=False, create_old=old)) + check_alive(0) + self.stackroots.append(p1) + self._collect(major=False) + check_alive(0) + self._collect(major=True) + check_alive(0) + p1 = self.stackroots.pop() + self._collect(major=False) + check_alive(0) + assert p1.x == 42 + self._collect(major=True, expected_trigger=1) + py.test.raises(RuntimeError, "p1.x") # dead + assert r1.ob_refcnt == 0 + assert r1.ob_pypy_link == 0 + assert self.gc.rawrefcount_next_dead() == r1addr + self.gc.check_no_more_rawrefcount_state() + lltype.free(r1, flavor='raw') + + def test_pypy_nonlight_dies_quickly(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_light=False, create_old=old)) + check_alive(0) + if old: + self._collect(major=False) + check_alive(0) + self._collect(major=True, expected_trigger=1) + else: + self._collect(major=False, expected_trigger=1) + py.test.raises(RuntimeError, "p1.x") # dead + assert r1.ob_refcnt == 0 + assert r1.ob_pypy_link == 0 + assert self.gc.rawrefcount_next_dead() == r1addr + self.gc.check_no_more_rawrefcount_state() + lltype.free(r1, flavor='raw') + + def test_pypy_nonlight_survives_from_raw_old(self): + self.test_pypy_nonlight_survives_from_raw(old=True) + def test_pypy_nonlight_survives_from_obj_old(self): + self.test_pypy_nonlight_survives_from_obj(old=True) + def test_pypy_nonlight_dies_quickly_old(self): + self.test_pypy_nonlight_dies_quickly(old=True) + + def test_pyobject_pypy_link_dies_on_minor_collection(self): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_pyobj=True)) + check_alive(0) + r1.ob_refcnt += 1 # the pyobject is kept alive + self._collect(major=False) + assert r1.ob_refcnt == 1 # refcnt dropped to 1 + assert r1.ob_pypy_link == 0 # detached + self.gc.check_no_more_rawrefcount_state() + lltype.free(r1, flavor='raw') + + def test_pyobject_dies(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_pyobj=True, create_old=old)) + check_alive(0) + if old: + self._collect(major=False) + check_alive(0) + self._collect(major=True, expected_trigger=1) + else: + self._collect(major=False, expected_trigger=1) + assert r1.ob_refcnt == 0 # refcnt dropped to 0 + assert r1.ob_pypy_link == 0 # detached + assert self.gc.rawrefcount_next_dead() == r1addr + self.gc.check_no_more_rawrefcount_state() + lltype.free(r1, flavor='raw') + + def test_pyobject_survives_from_obj(self, old=False): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, is_pyobj=True, create_old=old)) + check_alive(0) + self.stackroots.append(p1) + self._collect(major=False) + check_alive(0) + self._collect(major=True) + check_alive(0) + p1 = self.stackroots.pop() + self._collect(major=False) + check_alive(0) + assert p1.x == 42 + assert self.trigger == [] + self._collect(major=True, expected_trigger=1) + py.test.raises(RuntimeError, "p1.x") # dead + assert r1.ob_refcnt == 0 + assert r1.ob_pypy_link == 0 + assert self.gc.rawrefcount_next_dead() == r1addr + self.gc.check_no_more_rawrefcount_state() + lltype.free(r1, flavor='raw') + + def test_pyobject_dies_old(self): + self.test_pyobject_dies(old=True) + def test_pyobject_survives_from_obj_old(self): + self.test_pyobject_survives_from_obj(old=True) + + def test_pyobject_attached_to_prebuilt_obj(self): + p1, p1ref, r1, r1addr, check_alive = ( + self._rawrefcount_pair(42, create_immortal=True)) + check_alive(0) + self._collect(major=True) + check_alive(0) diff --git a/rpython/memory/gctransform/boehm.py b/rpython/memory/gctransform/boehm.py index 5144714329..25f72d32f6 100644 --- a/rpython/memory/gctransform/boehm.py +++ b/rpython/memory/gctransform/boehm.py @@ -156,9 +156,9 @@ class BoehmGCTransformer(GCTransformer): resulttype = lltype.Signed) hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result) - def gcheader_initdata(self, defnode): + def gcheader_initdata(self, obj): hdr = lltype.malloc(self.HDR, immortal=True) - hdr.hash = lltype.identityhash_nocache(defnode.obj._as_ptr()) + hdr.hash = lltype.identityhash_nocache(obj._as_ptr()) return hdr._obj diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py index 140e3cab11..8a7c0f06ad 100644 --- a/rpython/memory/gctransform/framework.py +++ b/rpython/memory/gctransform/framework.py @@ -153,6 +153,7 @@ class BaseFrameworkGCTransformer(GCTransformer): else: # for regular translation: pick the GC from the config GCClass, GC_PARAMS = choose_gc_from_config(translator.config) + self.GCClass = GCClass if hasattr(translator, '_jit2gc'): self.layoutbuilder = translator._jit2gc['layoutbuilder'] @@ -482,6 +483,29 @@ class BaseFrameworkGCTransformer(GCTransformer): annmodel.SomeInteger(nonneg=True)], annmodel.s_None) + if hasattr(GCClass, 'rawrefcount_init'): + self.rawrefcount_init_ptr = getfn( + GCClass.rawrefcount_init, + [s_gc, SomePtr(GCClass.RAWREFCOUNT_DEALLOC_TRIGGER)], + annmodel.s_None) + self.rawrefcount_create_link_pypy_ptr = getfn( + GCClass.rawrefcount_create_link_pypy, + [s_gc, s_gcref, SomeAddress()], + annmodel.s_None) + self.rawrefcount_create_link_pyobj_ptr = getfn( + GCClass.rawrefcount_create_link_pyobj, + [s_gc, s_gcref, SomeAddress()], + annmodel.s_None) + self.rawrefcount_from_obj_ptr = getfn( + GCClass.rawrefcount_from_obj, [s_gc, s_gcref], SomeAddress(), + inline = True) + self.rawrefcount_to_obj_ptr = getfn( + GCClass.rawrefcount_to_obj, [s_gc, SomeAddress()], s_gcref, + inline = True) + self.rawrefcount_next_dead_ptr = getfn( + GCClass.rawrefcount_next_dead, [s_gc], SomeAddress(), + inline = True) + if GCClass.can_usually_pin_objects: self.pin_ptr = getfn(GCClass.pin, [s_gc, SomeAddress()], @@ -1227,6 +1251,50 @@ class BaseFrameworkGCTransformer(GCTransformer): resultvar=hop.spaceop.result) self.pop_roots(hop, livevars) + def gct_gc_rawrefcount_init(self, hop): + [v_fnptr] = hop.spaceop.args + assert v_fnptr.concretetype == self.GCClass.RAWREFCOUNT_DEALLOC_TRIGGER + hop.genop("direct_call", + [self.rawrefcount_init_ptr, self.c_const_gc, v_fnptr]) + + def gct_gc_rawrefcount_create_link_pypy(self, hop): + [v_gcobj, v_pyobject] = hop.spaceop.args + assert v_gcobj.concretetype == llmemory.GCREF + assert v_pyobject.concretetype == llmemory.Address + hop.genop("direct_call", + [self.rawrefcount_create_link_pypy_ptr, self.c_const_gc, + v_gcobj, v_pyobject]) + + def gct_gc_rawrefcount_create_link_pyobj(self, hop): + [v_gcobj, v_pyobject] = hop.spaceop.args + assert v_gcobj.concretetype == llmemory.GCREF + assert v_pyobject.concretetype == llmemory.Address + hop.genop("direct_call", + [self.rawrefcount_create_link_pyobj_ptr, self.c_const_gc, + v_gcobj, v_pyobject]) + + def gct_gc_rawrefcount_from_obj(self, hop): + [v_gcobj] = hop.spaceop.args + assert v_gcobj.concretetype == llmemory.GCREF + assert hop.spaceop.result.concretetype == llmemory.Address + hop.genop("direct_call", + [self.rawrefcount_from_obj_ptr, self.c_const_gc, v_gcobj], + resultvar=hop.spaceop.result) + + def gct_gc_rawrefcount_to_obj(self, hop): + [v_pyobject] = hop.spaceop.args + assert v_pyobject.concretetype == llmemory.Address + assert hop.spaceop.result.concretetype == llmemory.GCREF + hop.genop("direct_call", + [self.rawrefcount_to_obj_ptr, self.c_const_gc, v_pyobject], + resultvar=hop.spaceop.result) + + def gct_gc_rawrefcount_next_dead(self, hop): + assert hop.spaceop.result.concretetype == llmemory.Address + hop.genop("direct_call", + [self.rawrefcount_next_dead_ptr, self.c_const_gc], + resultvar=hop.spaceop.result) + def _set_into_gc_array_part(self, op): if op.opname == 'setarrayitem': return op.args[1] @@ -1411,8 +1479,8 @@ class BaseFrameworkGCTransformer(GCTransformer): resulttype=llmemory.Address) llops.genop('raw_memclear', [v_adr, v_totalsize]) - def gcheader_initdata(self, defnode): - o = lltype.top_container(defnode.obj) + def gcheader_initdata(self, obj): + o = lltype.top_container(obj) needs_hash = self.get_prebuilt_hash(o) is not None hdr = self.gc_header_for(o, needs_hash) return hdr._obj diff --git a/rpython/memory/gctransform/refcounting.py b/rpython/memory/gctransform/refcounting.py index ae6f64b4ee..a496db590c 100644 --- a/rpython/memory/gctransform/refcounting.py +++ b/rpython/memory/gctransform/refcounting.py @@ -286,6 +286,6 @@ def ll_deallocator(addr): hop.genop("direct_call", [self.identityhash_ptr, v_adr], resultvar=hop.spaceop.result) - def gcheader_initdata(self, defnode): - top = lltype.top_container(defnode.obj) + def gcheader_initdata(self, obj): + top = lltype.top_container(obj) return self.gcheaderbuilder.header_of_object(top)._obj diff --git a/rpython/memory/gctransform/test/test_transform.py b/rpython/memory/gctransform/test/test_transform.py index 03343618db..9cd4382c5d 100644 --- a/rpython/memory/gctransform/test/test_transform.py +++ b/rpython/memory/gctransform/test/test_transform.py @@ -5,6 +5,7 @@ from rpython.translator.translator import TranslationContext, graphof from rpython.translator.exceptiontransform import ExceptionTransformer from rpython.rtyper.lltypesystem import lltype from rpython.conftest import option +from rpython.rtyper.rtyper import llinterp_backend class LLInterpedTranformerTests: @@ -131,8 +132,10 @@ def checkblock(block, is_borrowed, is_start_block): def rtype(func, inputtypes, specialize=True): t = TranslationContext() t.buildannotator().build_types(func, inputtypes) + rtyper = t.buildrtyper() + rtyper.backend = llinterp_backend if specialize: - t.buildrtyper().specialize() + rtyper.specialize() if option.view: t.view() return t diff --git a/rpython/memory/test/test_transformed_gc.py b/rpython/memory/test/test_transformed_gc.py index 81b77451a2..8feeb9198b 100644 --- a/rpython/memory/test/test_transformed_gc.py +++ b/rpython/memory/test/test_transformed_gc.py @@ -14,6 +14,7 @@ from rpython.rlib import rgc from rpython.conftest import option from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import LONG_BIT +from rpython.rtyper.rtyper import llinterp_backend WORD = LONG_BIT // 8 @@ -29,9 +30,11 @@ def rtype(func, inputtypes, specialize=True, gcname='ref', t.config.set(**extraconfigopts) ann = t.buildannotator() ann.build_types(func, inputtypes) + rtyper = t.buildrtyper() + rtyper.backend = llinterp_backend if specialize: - t.buildrtyper().specialize() + rtyper.specialize() if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) diff --git a/rpython/rtyper/module/support.py b/rpython/rlib/_os_support.py index fac520788b..0365efdb08 100644 --- a/rpython/rtyper/module/support.py +++ b/rpython/rlib/_os_support.py @@ -1,53 +1,20 @@ import sys -from rpython.annotator import model as annmodel -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.objectmodel import specialize +from rpython.annotator.model import s_Str0, s_Unicode0 from rpython.rlib import rstring +from rpython.rlib.objectmodel import specialize +from rpython.rtyper.lltypesystem import rffi + +_CYGWIN = sys.platform == 'cygwin' _WIN32 = sys.platform.startswith('win') UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' - -# utility conversion functions -class LLSupport: - _mixin_ = True - - def to_rstr(s): - from rpython.rtyper.lltypesystem.rstr import STR, mallocstr - if s is None: - return lltype.nullptr(STR) - p = mallocstr(len(s)) - for i in range(len(s)): - p.chars[i] = s[i] - return p - to_rstr = staticmethod(to_rstr) - - def to_runicode(s): - from rpython.rtyper.lltypesystem.rstr import UNICODE, mallocunicode - if s is None: - return lltype.nullptr(UNICODE) - p = mallocunicode(len(s)) - for i in range(len(s)): - p.chars[i] = s[i] - return p - to_runicode = staticmethod(to_runicode) - - def from_rstr(rs): - if not rs: # null pointer - return None - else: - return ''.join([rs.chars[i] for i in range(len(rs.chars))]) - from_rstr = staticmethod(from_rstr) - - def from_rstr_nonnull(rs): - assert rs - return ''.join([rs.chars[i] for i in range(len(rs.chars))]) - from_rstr_nonnull = staticmethod(from_rstr_nonnull) +_MACRO_ON_POSIX = True if not _WIN32 else None -class StringTraits: +class StringTraits(object): str = str - str0 = annmodel.s_Str0 + str0 = s_Str0 CHAR = rffi.CHAR CCHARP = rffi.CCHARP charp2str = staticmethod(rffi.charp2str) @@ -58,14 +25,6 @@ class StringTraits: scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_buffer) @staticmethod - def posix_function_name(name): - return UNDERSCORE_ON_WIN32 + name - - @staticmethod - def ll_os_name(name): - return 'll_os.ll_os_' + name - - @staticmethod @specialize.argtype(0) def as_str(path): assert path is not None @@ -77,7 +36,7 @@ class StringTraits: # We implement python2 behavior: silently convert to ascii. return path.encode('ascii') else: - return path.as_bytes() + return path.as_bytes() @staticmethod @specialize.argtype(0) @@ -86,9 +45,10 @@ class StringTraits: rstring.check_str0(res) return res -class UnicodeTraits: + +class UnicodeTraits(object): str = unicode - str0 = annmodel.s_Unicode0 + str0 = s_Unicode0 CHAR = rffi.WCHAR_T CCHARP = rffi.CWCHARP charp2str = staticmethod(rffi.wcharp2unicode) @@ -99,15 +59,6 @@ class UnicodeTraits: scoped_alloc_buffer = staticmethod(rffi.scoped_alloc_unicodebuffer) @staticmethod - def posix_function_name(name): - return UNDERSCORE_ON_WIN32 + 'w' + name - - @staticmethod - @specialize.argtype(0) - def ll_os_name(name): - return 'll_os.ll_os_w' + name - - @staticmethod @specialize.argtype(0) def as_str(path): assert path is not None @@ -115,7 +66,7 @@ class UnicodeTraits: return path else: return path.as_unicode() - + @staticmethod @specialize.argtype(0) def as_str0(path): @@ -123,17 +74,36 @@ class UnicodeTraits: rstring.check_str0(res) return res -def ll_strcpy(dst_s, src_s, n): - dstchars = dst_s.chars - srcchars = src_s.chars - i = 0 - while i < n: - dstchars[i] = srcchars[i] - i += 1 - -def _ll_strfill(dst_s, srcchars, n): - dstchars = dst_s.chars - i = 0 - while i < n: - dstchars[i] = srcchars[i] - i += 1 + +string_traits = StringTraits() +unicode_traits = UnicodeTraits() + + +# Returns True when the unicode function should be called: +# - on Windows +# - if the path is Unicode. +if _WIN32: + @specialize.argtype(0) + def _prefer_unicode(path): + assert path is not None + if isinstance(path, str): + return False + elif isinstance(path, unicode): + return True + else: + return path.is_unicode + + @specialize.argtype(0) + def _preferred_traits(path): + if _prefer_unicode(path): + return unicode_traits + else: + return string_traits +else: + @specialize.argtype(0) + def _prefer_unicode(path): + return False + + @specialize.argtype(0) + def _preferred_traits(path): + return string_traits diff --git a/rpython/rlib/exports.py b/rpython/rlib/exports.py index 1ae9ae4778..391678e1c4 100644 --- a/rpython/rlib/exports.py +++ b/rpython/rlib/exports.py @@ -1,5 +1,7 @@ from rpython.rtyper.lltypesystem.lltype import typeOf, ContainerType +# XXX kill me + def export_struct(name, struct): assert name not in EXPORTS_names, "Duplicate export " + name assert isinstance(typeOf(struct), ContainerType) diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py index d17e919e62..7017795baf 100644 --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -284,7 +284,7 @@ isvirtual._annspecialcase_ = "specialize:call_location" def loop_unrolling_heuristic(lst, size, cutoff=2): """ In which cases iterating over items of lst can be unrolled """ - return isvirtual(lst) or (isconstant(size) and size <= cutoff) + return size == 0 or isvirtual(lst) or (isconstant(size) and size <= cutoff) class Entry(ExtRegistryEntry): _about_ = hint @@ -1168,6 +1168,24 @@ class ConditionalCallEntry(ExtRegistryEntry): hop.exception_is_here() return hop.genop('jit_conditional_call', args_v) +def enter_portal_frame(unique_id): + """call this when starting to interpret a function. calling this is not + necessary for almost all interpreters. The only exception is stackless + interpreters where the portal never calls itself. + """ + from rpython.rtyper.lltypesystem import lltype + from rpython.rtyper.lltypesystem.lloperation import llop + llop.jit_enter_portal_frame(lltype.Void, unique_id) + +def leave_portal_frame(): + """call this after the end of executing a function. calling this is not + necessary for almost all interpreters. The only exception is stackless + interpreters where the portal never calls itself. + """ + from rpython.rtyper.lltypesystem import lltype + from rpython.rtyper.lltypesystem.lloperation import llop + llop.jit_leave_portal_frame(lltype.Void) + class Counters(object): counters=""" TRACING diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py index 464afa1ee5..aa2a2d78c0 100644 --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -275,8 +275,6 @@ class CDefinedIntSymbolic(Symbolic): return lltype.Signed malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0) -running_on_llinterp = CDefinedIntSymbolic('RUNNING_ON_LLINTERP', default=1) -# running_on_llinterp is meant to have the value 0 in all backends # ____________________________________________________________ diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py new file mode 100644 index 0000000000..cb19b18c47 --- /dev/null +++ b/rpython/rlib/rawrefcount.py @@ -0,0 +1,265 @@ +# +# See documentation in pypy/doc/discussion/rawrefcount.rst +# +# This is meant for pypy's cpyext module, but is a generally +# useful interface over our GC. XXX "pypy" should be removed here +# +import sys, weakref +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rlib.objectmodel import we_are_translated, specialize +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.rlib import rgc + + +REFCNT_FROM_PYPY = sys.maxint // 4 + 1 +REFCNT_FROM_PYPY_LIGHT = REFCNT_FROM_PYPY + (sys.maxint // 2 + 1) + +RAWREFCOUNT_DEALLOC_TRIGGER = lltype.Ptr(lltype.FuncType([], lltype.Void)) + + +def _build_pypy_link(p): + res = len(_adr2pypy) + _adr2pypy.append(p) + return res + + +def init(dealloc_trigger_callback=None): + """NOT_RPYTHON: set up rawrefcount with the GC. This is only used + for tests; it should not be called at all during translation. + """ + global _p_list, _o_list, _adr2pypy, _pypy2ob + global _d_list, _dealloc_trigger_callback + _p_list = [] + _o_list = [] + _adr2pypy = [None] + _pypy2ob = {} + _d_list = [] + _dealloc_trigger_callback = dealloc_trigger_callback + +def create_link_pypy(p, ob): + "NOT_RPYTHON: a link where the PyPy object contains some or all the data" + #print 'create_link_pypy\n\t%s\n\t%s' % (p, ob) + assert p not in _pypy2ob + #assert not ob.c_ob_pypy_link + ob.c_ob_pypy_link = _build_pypy_link(p) + _pypy2ob[p] = ob + _p_list.append(ob) + +def create_link_pyobj(p, ob): + """NOT_RPYTHON: a link where the PyObject contains all the data. + from_obj() will not work on this 'p'.""" + #print 'create_link_pyobj\n\t%s\n\t%s' % (p, ob) + assert p not in _pypy2ob + #assert not ob.c_ob_pypy_link + ob.c_ob_pypy_link = _build_pypy_link(p) + _o_list.append(ob) + +def from_obj(OB_PTR_TYPE, p): + "NOT_RPYTHON" + ob = _pypy2ob.get(p) + if ob is None: + return lltype.nullptr(OB_PTR_TYPE.TO) + assert lltype.typeOf(ob) == OB_PTR_TYPE + return ob + +def to_obj(Class, ob): + "NOT_RPYTHON" + link = ob.c_ob_pypy_link + if link == 0: + return None + p = _adr2pypy[link] + assert isinstance(p, Class) + return p + +def next_dead(OB_PTR_TYPE): + if len(_d_list) == 0: + return lltype.nullptr(OB_PTR_TYPE.TO) + ob = _d_list.pop() + assert lltype.typeOf(ob) == OB_PTR_TYPE + return ob + +def _collect(track_allocation=True): + """NOT_RPYTHON: for tests only. Emulates a GC collection. + Will invoke dealloc_trigger_callback() once if there are objects + whose _Py_Dealloc() should be called. + """ + def detach(ob, wr_list): + assert ob.c_ob_refcnt >= REFCNT_FROM_PYPY + assert ob.c_ob_pypy_link + p = _adr2pypy[ob.c_ob_pypy_link] + assert p is not None + _adr2pypy[ob.c_ob_pypy_link] = None + wr_list.append((ob, weakref.ref(p))) + return p + + global _p_list, _o_list + wr_p_list = [] + new_p_list = [] + for ob in reversed(_p_list): + if ob.c_ob_refcnt not in (REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT): + new_p_list.append(ob) + else: + p = detach(ob, wr_p_list) + del _pypy2ob[p] + del p + ob = None + _p_list = Ellipsis + + wr_o_list = [] + for ob in reversed(_o_list): + detach(ob, wr_o_list) + _o_list = Ellipsis + + rgc.collect() # forces the cycles to be resolved and the weakrefs to die + rgc.collect() + rgc.collect() + + def attach(ob, wr, final_list): + assert ob.c_ob_refcnt >= REFCNT_FROM_PYPY + p = wr() + if p is not None: + assert ob.c_ob_pypy_link + _adr2pypy[ob.c_ob_pypy_link] = p + final_list.append(ob) + return p + else: + ob.c_ob_pypy_link = 0 + if ob.c_ob_refcnt >= REFCNT_FROM_PYPY_LIGHT: + ob.c_ob_refcnt -= REFCNT_FROM_PYPY_LIGHT + ob.c_ob_pypy_link = 0 + if ob.c_ob_refcnt == 0: + lltype.free(ob, flavor='raw', + track_allocation=track_allocation) + else: + assert ob.c_ob_refcnt >= REFCNT_FROM_PYPY + assert ob.c_ob_refcnt < int(REFCNT_FROM_PYPY_LIGHT * 0.99) + ob.c_ob_refcnt -= REFCNT_FROM_PYPY + ob.c_ob_pypy_link = 0 + if ob.c_ob_refcnt == 0: + _d_list.append(ob) + return None + + _p_list = new_p_list + for ob, wr in wr_p_list: + p = attach(ob, wr, _p_list) + if p is not None: + _pypy2ob[p] = ob + _o_list = [] + for ob, wr in wr_o_list: + attach(ob, wr, _o_list) + + if _d_list: + res = _dealloc_trigger_callback() + if res == "RETRY": + _collect(track_allocation=track_allocation) + +_keepalive_forever = set() +def _dont_free_any_more(): + "Make sure that any object still referenced won't be freed any more." + for ob in _p_list + _o_list: + _keepalive_forever.add(to_obj(object, ob)) + del _d_list[:] + +# ____________________________________________________________ + + +def _unspec_p(hop, v_p): + assert isinstance(v_p.concretetype, lltype.Ptr) + assert v_p.concretetype.TO._gckind == 'gc' + return hop.genop('cast_opaque_ptr', [v_p], resulttype=llmemory.GCREF) + +def _unspec_ob(hop, v_ob): + assert isinstance(v_ob.concretetype, lltype.Ptr) + assert v_ob.concretetype.TO._gckind == 'raw' + return hop.genop('cast_ptr_to_adr', [v_ob], resulttype=llmemory.Address) + +def _spec_p(hop, v_p): + assert v_p.concretetype == llmemory.GCREF + return hop.genop('cast_opaque_ptr', [v_p], + resulttype=hop.r_result.lowleveltype) + +def _spec_ob(hop, v_ob): + assert v_ob.concretetype == llmemory.Address + return hop.genop('cast_adr_to_ptr', [v_ob], + resulttype=hop.r_result.lowleveltype) + + +class Entry(ExtRegistryEntry): + _about_ = init + + def compute_result_annotation(self, s_dealloc_callback): + from rpython.rtyper.llannotation import SomePtr + assert isinstance(s_dealloc_callback, SomePtr) # ll-ptr-to-function + + def specialize_call(self, hop): + hop.exception_cannot_occur() + [v_dealloc_callback] = hop.inputargs(hop.args_r[0]) + hop.genop('gc_rawrefcount_init', [v_dealloc_callback]) + + +class Entry(ExtRegistryEntry): + _about_ = (create_link_pypy, create_link_pyobj) + + def compute_result_annotation(self, s_p, s_ob): + pass + + def specialize_call(self, hop): + if self.instance is create_link_pypy: + name = 'gc_rawrefcount_create_link_pypy' + elif self.instance is create_link_pyobj: + name = 'gc_rawrefcount_create_link_pyobj' + v_p, v_ob = hop.inputargs(*hop.args_r) + hop.exception_cannot_occur() + hop.genop(name, [_unspec_p(hop, v_p), _unspec_ob(hop, v_ob)]) + + +class Entry(ExtRegistryEntry): + _about_ = from_obj + + def compute_result_annotation(self, s_OB_PTR_TYPE, s_p): + from rpython.annotator import model as annmodel + from rpython.rtyper.llannotation import lltype_to_annotation + assert (isinstance(s_p, annmodel.SomeInstance) or + annmodel.s_None.contains(s_p)) + assert s_OB_PTR_TYPE.is_constant() + return lltype_to_annotation(s_OB_PTR_TYPE.const) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + v_p = hop.inputarg(hop.args_r[1], arg=1) + v_ob = hop.genop('gc_rawrefcount_from_obj', [_unspec_p(hop, v_p)], + resulttype = llmemory.Address) + return _spec_ob(hop, v_ob) + +class Entry(ExtRegistryEntry): + _about_ = to_obj + + def compute_result_annotation(self, s_Class, s_ob): + from rpython.annotator import model as annmodel + from rpython.rtyper.llannotation import SomePtr + assert isinstance(s_ob, SomePtr) + assert s_Class.is_constant() + classdef = self.bookkeeper.getuniqueclassdef(s_Class.const) + return annmodel.SomeInstance(classdef, can_be_None=True) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + v_ob = hop.inputarg(hop.args_r[1], arg=1) + v_p = hop.genop('gc_rawrefcount_to_obj', [_unspec_ob(hop, v_ob)], + resulttype = llmemory.GCREF) + return _spec_p(hop, v_p) + +class Entry(ExtRegistryEntry): + _about_ = next_dead + + def compute_result_annotation(self, s_OB_PTR_TYPE): + from rpython.annotator import model as annmodel + from rpython.rtyper.llannotation import lltype_to_annotation + assert s_OB_PTR_TYPE.is_constant() + return lltype_to_annotation(s_OB_PTR_TYPE.const) + + def specialize_call(self, hop): + hop.exception_cannot_occur() + v_ob = hop.genop('gc_rawrefcount_next_dead', [], + resulttype = llmemory.Address) + return _spec_ob(hop, v_ob) diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py index e87849d393..99be096eef 100644 --- a/rpython/rlib/rgc.py +++ b/rpython/rlib/rgc.py @@ -487,6 +487,7 @@ NULL_GCREF = lltype.nullptr(llmemory.GCREF.TO) class _GcRef(object): # implementation-specific: there should not be any after translation __slots__ = ['_x', '_handle'] + _TYPE = llmemory.GCREF def __init__(self, x): self._x = x def __hash__(self): diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py index 7ef8da67e9..b2d29d345f 100644 --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -1,27 +1,22 @@ import os import sys import errno +from rpython.annotator.model import s_Str0 from rpython.rtyper.lltypesystem.rffi import CConstant, CExternVariable, INT from rpython.rtyper.lltypesystem import lltype, ll2ctypes, rffi -from rpython.rtyper.module.support import StringTraits, UnicodeTraits from rpython.rtyper.tool import rffi_platform -from rpython.tool.sourcetools import func_renamer -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rlib.rarithmetic import intmask, widen +from rpython.rlib import debug, jit, rstring, rthread, types +from rpython.rlib._os_support import ( + _CYGWIN, _MACRO_ON_POSIX, UNDERSCORE_ON_WIN32, _WIN32, + _prefer_unicode, _preferred_traits) from rpython.rlib.objectmodel import ( specialize, enforceargs, register_replacement_for, NOT_CONSTANT) +from rpython.rlib.rarithmetic import intmask, widen from rpython.rlib.signature import signature -from rpython.rlib import types -from rpython.annotator.model import s_Str0 -from rpython.rlib import jit +from rpython.tool.sourcetools import func_renamer from rpython.translator.platform import platform -from rpython.rlib import rstring -from rpython.rlib import debug, rthread +from rpython.translator.tool.cbuild import ExternalCompilationInfo -_WIN32 = sys.platform.startswith('win') -_CYGWIN = sys.platform == 'cygwin' -UNDERSCORE_ON_WIN32 = '_' if _WIN32 else '' -_MACRO_ON_POSIX = True if not _WIN32 else None if _WIN32: from rpython.rlib import rwin32 @@ -119,7 +114,6 @@ def get_saved_errno(): with the flag llexternal(..., save_err=rffi.RFFI_SAVE_ERRNO). Functions without that flag don't change the saved errno. """ - from rpython.rlib import rthread return intmask(rthread.tlfield_rpy_errno.getraw()) def set_saved_errno(errno): @@ -130,7 +124,6 @@ def set_saved_errno(errno): zero; for that case, use llexternal(..., save_err=RFFI_ZERO_ERRNO_BEFORE) and then you don't need set_saved_errno(0). """ - from rpython.rlib import rthread rthread.tlfield_rpy_errno.setraw(rffi.cast(INT, errno)) def get_saved_alterrno(): @@ -139,7 +132,6 @@ def get_saved_alterrno(): with the flag llexternal(..., save_err=rffi.RFFI_SAVE_ERRNO | rffl.RFFI_ALT_ERRNO). Functions without that flag don't change the saved errno. """ - from rpython.rlib import rthread return intmask(rthread.tlfield_alt_errno.getraw()) def set_saved_alterrno(errno): @@ -150,7 +142,6 @@ def set_saved_alterrno(errno): zero; for that case, use llexternal(..., save_err=RFFI_ZERO_ERRNO_BEFORE) and then you don't need set_saved_errno(0). """ - from rpython.rlib import rthread rthread.tlfield_alt_errno.setraw(rffi.cast(INT, errno)) @@ -158,7 +149,6 @@ def set_saved_alterrno(errno): @specialize.call_location() def _errno_before(save_err): if save_err & rffi.RFFI_READSAVED_ERRNO: - from rpython.rlib import rthread if save_err & rffi.RFFI_ALT_ERRNO: _set_errno(rthread.tlfield_alt_errno.getraw()) else: @@ -166,7 +156,6 @@ def _errno_before(save_err): elif save_err & rffi.RFFI_ZERO_ERRNO_BEFORE: _set_errno(rffi.cast(rffi.INT, 0)) if _WIN32 and (save_err & rffi.RFFI_READSAVED_LASTERROR): - from rpython.rlib import rthread, rwin32 if save_err & rffi.RFFI_ALT_ERRNO: err = rthread.tlfield_alt_lasterror.getraw() else: @@ -180,7 +169,6 @@ def _errno_before(save_err): def _errno_after(save_err): if _WIN32: if save_err & rffi.RFFI_SAVE_LASTERROR: - from rpython.rlib import rthread, rwin32 err = rwin32._GetLastError() # careful, setraw() overwrites GetLastError. # We must read it first, before the errno handling. @@ -189,14 +177,13 @@ def _errno_after(save_err): else: rthread.tlfield_rpy_lasterror.setraw(err) elif save_err & rffi.RFFI_SAVE_WSALASTERROR: - from rpython.rlib import rthread, _rsocket_rffi + from rpython.rlib import _rsocket_rffi err = _rsocket_rffi._WSAGetLastError() if save_err & rffi.RFFI_ALT_ERRNO: rthread.tlfield_alt_lasterror.setraw(err) else: rthread.tlfield_rpy_lasterror.setraw(err) if save_err & rffi.RFFI_SAVE_ERRNO: - from rpython.rlib import rthread if save_err & rffi.RFFI_ALT_ERRNO: rthread.tlfield_alt_errno.setraw(_get_errno()) else: @@ -342,37 +329,6 @@ def _as_unicode0(path): rstring.check_str0(res) return res -# Returns True when the unicode function should be called: -# - on Windows -# - if the path is Unicode. -unicode_traits = UnicodeTraits() -string_traits = StringTraits() -if _WIN32: - @specialize.argtype(0) - def _prefer_unicode(path): - assert path is not None - if isinstance(path, str): - return False - elif isinstance(path, unicode): - return True - else: - return path.is_unicode - - @specialize.argtype(0) - def _preferred_traits(path): - if _prefer_unicode(path): - return unicode_traits - else: - return string_traits -else: - @specialize.argtype(0) - def _prefer_unicode(path): - return False - - @specialize.argtype(0) - def _preferred_traits(path): - return string_traits - @specialize.argtype(0, 1) def putenv(name, value): os.environ[_as_bytes(name)] = _as_bytes(value) diff --git a/rpython/rlib/rposix_environ.py b/rpython/rlib/rposix_environ.py index 9906ceb0d3..b419b6eaeb 100644 --- a/rpython/rlib/rposix_environ.py +++ b/rpython/rlib/rposix_environ.py @@ -1,11 +1,12 @@ import os import sys from rpython.annotator import model as annmodel +from rpython.rlib._os_support import _WIN32, StringTraits, UnicodeTraits from rpython.rlib.objectmodel import enforceargs +# importing rposix here creates a cycle on Windows from rpython.rtyper.controllerentry import Controller from rpython.rtyper.extfunc import register_external from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.module.support import _WIN32, StringTraits, UnicodeTraits from rpython.translator.tool.cbuild import ExternalCompilationInfo str0 = annmodel.s_Str0 diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py index b6c20b591c..1b66547e50 100644 --- a/rpython/rlib/rposix_stat.py +++ b/rpython/rlib/rposix_stat.py @@ -16,13 +16,13 @@ from rpython.rtyper.rmodel import Repr from rpython.rtyper.rint import IntegerRepr from rpython.rtyper.error import TyperError +from rpython.rlib._os_support import _preferred_traits, string_traits from rpython.rlib.objectmodel import specialize from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.rarithmetic import intmask from rpython.rlib.rposix import ( - replace_os_function, handle_posix_error, _as_bytes0, - _preferred_traits, string_traits) + replace_os_function, handle_posix_error, _as_bytes0) _WIN32 = sys.platform.startswith('win') _LINUX = sys.platform.startswith('linux') diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py index 9f2247e05b..ff6b9b4257 100644 --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -516,6 +516,10 @@ class RSocket(object): """RPython-level socket object. """ fd = _c.INVALID_SOCKET + family = 0 + type = 0 + proto = 0 + timeout = -1.0 def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fd=_c.INVALID_SOCKET): @@ -531,6 +535,11 @@ class RSocket(object): self.proto = proto self.timeout = defaults.timeout + @staticmethod + def empty_rsocket(): + rsocket = instantiate(RSocket) + return rsocket + @rgc.must_be_light_finalizer def __del__(self): fd = self.fd diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py index 154c06bcbd..80276a0975 100644 --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -7,8 +7,6 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform as platform from rpython.rlib import rthread -from rpython.jit.backend import detect_cpu - class VMProfPlatformUnsupported(Exception): pass diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py index 95b4c8987c..86abed36ba 100644 --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -4,7 +4,8 @@ from rpython.conftest import option from rpython.annotator.model import UnionError from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote, JitHintError, oopspec, isconstant, conditional_call, - elidable, unroll_safe, dont_look_inside) + elidable, unroll_safe, dont_look_inside, + enter_portal_frame, leave_portal_frame) from rpython.rlib.rarithmetic import r_uint from rpython.rtyper.test.tool import BaseRtypingTest from rpython.rtyper.lltypesystem import lltype @@ -300,3 +301,11 @@ class TestJIT(BaseRtypingTest): mix = MixLevelHelperAnnotator(t.rtyper) mix.getgraph(later, [annmodel.s_Bool], annmodel.s_None) mix.finish() + + def test_enter_leave_portal_frame(self): + from rpython.translator.interactive import Translation + def g(): + enter_portal_frame(1) + leave_portal_frame() + t = Translation(g, []) + t.compile_c() # does not crash diff --git a/rpython/rlib/test/test_posix.py b/rpython/rlib/test/test_posix.py new file mode 100644 index 0000000000..6f4b2bce62 --- /dev/null +++ b/rpython/rlib/test/test_posix.py @@ -0,0 +1,304 @@ +import py.test +from rpython.rtyper.test.tool import BaseRtypingTest +from rpython.rtyper.annlowlevel import hlstr +from rpython.tool.udir import udir +from rpython.rlib.rarithmetic import is_valid_int + +import os +exec 'import %s as posix' % os.name + +def setup_module(module): + testf = udir.join('test.txt') + module.path = testf.strpath + +class TestPosix(BaseRtypingTest): + + def setup_method(self, meth): + # prepare/restore the file before each test + testfile = open(path, 'wb') + testfile.write('This is a test') + testfile.close() + + def test_open(self): + def f(): + ff = posix.open(path, posix.O_RDONLY, 0777) + return ff + func = self.interpret(f, []) + assert is_valid_int(func) + + def test_fstat(self): + def fo(fi): + g = posix.fstat(fi) + return g + fi = os.open(path,os.O_RDONLY,0777) + func = self.interpret(fo,[fi]) + stat = os.fstat(fi) + for i in range(len(stat)): + assert long(getattr(func, 'item%d' % i)) == stat[i] + + + def test_stat(self): + def fo(): + g = posix.stat(path) + return g + func = self.interpret(fo,[]) + stat = os.stat(path) + for i in range(len(stat)): + assert long(getattr(func, 'item%d' % i)) == stat[i] + + def test_stat_exception(self): + def fo(): + try: + posix.stat('I/do/not/exist') + except OSError: + return True + else: + return False + res = self.interpret(fo,[]) + assert res + + def test_times(self): + py.test.skip("llinterp does not like tuple returns") + from rpython.rtyper.test.test_llinterp import interpret + times = interpret(lambda: posix.times(), ()) + assert isinstance(times, tuple) + assert len(times) == 5 + for value in times: + assert is_valid_int(value) + + + def test_lseek(self): + def f(fi, pos): + posix.lseek(fi, pos, 0) + fi = os.open(path, os.O_RDONLY, 0777) + func = self.interpret(f, [fi, 5]) + res = os.read(fi, 2) + assert res =='is' + + def test_isatty(self): + def f(fi): + posix.isatty(fi) + fi = os.open(path, os.O_RDONLY, 0777) + func = self.interpret(f, [fi]) + assert not func + os.close(fi) + func = self.interpret(f, [fi]) + assert not func + + def test_getcwd(self): + def f(): + return posix.getcwd() + res = self.interpret(f,[]) + cwd = os.getcwd() + #print res.chars,cwd + assert self.ll_to_string(res) == cwd + + def test_write(self): + def f(fi): + if fi > 0: + text = 'This is a test' + else: + text = '333' + return posix.write(fi,text) + fi = os.open(path,os.O_WRONLY,0777) + text = 'This is a test' + func = self.interpret(f,[fi]) + os.close(fi) + fi = os.open(path,os.O_RDONLY,0777) + res = os.read(fi,20) + assert res == text + + def test_read(self): + def f(fi,len): + return posix.read(fi,len) + fi = os.open(path,os.O_WRONLY,0777) + text = 'This is a test' + os.write(fi,text) + os.close(fi) + fi = os.open(path,os.O_RDONLY,0777) + res = self.interpret(f,[fi,20]) + assert self.ll_to_string(res) == text + + @py.test.mark.skipif("not hasattr(os, 'chown')") + def test_chown(self): + f = open(path, "w") + f.write("xyz") + f.close() + def f(): + try: + posix.chown(path, os.getuid(), os.getgid()) + return 1 + except OSError: + return 2 + + assert self.interpret(f, []) == 1 + os.unlink(path) + assert self.interpret(f, []) == 2 + + def test_close(self): + def f(fi): + return posix.close(fi) + fi = os.open(path,os.O_WRONLY,0777) + text = 'This is a test' + os.write(fi,text) + res = self.interpret(f,[fi]) + py.test.raises( OSError, os.fstat, fi) + + @py.test.mark.skipif("not hasattr(os, 'ftruncate')") + def test_ftruncate(self): + def f(fi,len): + os.ftruncate(fi,len) + fi = os.open(path,os.O_RDWR,0777) + func = self.interpret(f,[fi,6]) + assert os.fstat(fi).st_size == 6 + + @py.test.mark.skipif("not hasattr(os, 'getuid')") + def test_getuid(self): + def f(): + return os.getuid() + assert self.interpret(f, []) == f() + + @py.test.mark.skipif("not hasattr(os, 'getgid')") + def test_getgid(self): + def f(): + return os.getgid() + assert self.interpret(f, []) == f() + + @py.test.mark.skipif("not hasattr(os, 'setuid')") + def test_os_setuid(self): + def f(): + os.setuid(os.getuid()) + return os.getuid() + assert self.interpret(f, []) == f() + + @py.test.mark.skipif("not hasattr(os, 'sysconf')") + def test_os_sysconf(self): + def f(i): + return os.sysconf(i) + assert self.interpret(f, [13]) == f(13) + + @py.test.mark.skipif("not hasattr(os, 'confstr')") + def test_os_confstr(self): + def f(i): + try: + return os.confstr(i) + except OSError: + return "oooops!!" + some_value = os.confstr_names.values()[-1] + res = self.interpret(f, [some_value]) + assert hlstr(res) == f(some_value) + res = self.interpret(f, [94781413]) + assert hlstr(res) == "oooops!!" + + @py.test.mark.skipif("not hasattr(os, 'pathconf')") + def test_os_pathconf(self): + def f(i): + return os.pathconf("/tmp", i) + i = os.pathconf_names["PC_NAME_MAX"] + some_value = self.interpret(f, [i]) + assert some_value >= 31 + + @py.test.mark.skipif("not hasattr(os, 'chroot')") + def test_os_chroot(self): + def f(): + try: + os.chroot('!@$#!#%$#^#@!#!$$#^') + except OSError: + return 1 + return 0 + + assert self.interpret(f, []) == 1 + + def test_os_wstar(self): + from rpython.rlib import rposix + for name in rposix.WAIT_MACROS: + if not hasattr(os, name): + continue + def fun(s): + return getattr(os, name)(s) + + for value in [0, 1, 127, 128, 255]: + res = self.interpret(fun, [value]) + assert res == fun(value) + + @py.test.mark.skipif("not hasattr(os, 'getgroups')") + def test_getgroups(self): + def f(): + return os.getgroups() + ll_a = self.interpret(f, []) + assert self.ll_to_list(ll_a) == f() + + @py.test.mark.skipif("not hasattr(os, 'setgroups')") + def test_setgroups(self): + def f(): + try: + os.setgroups(os.getgroups()) + except OSError: + pass + self.interpret(f, []) + + @py.test.mark.skipif("not hasattr(os, 'initgroups')") + def test_initgroups(self): + def f(): + try: + os.initgroups('sUJJeumz', 4321) + except OSError: + return 1 + return 0 + res = self.interpret(f, []) + assert res == 1 + + @py.test.mark.skipif("not hasattr(os, 'tcgetpgrp')") + def test_tcgetpgrp(self): + def f(fd): + try: + return os.tcgetpgrp(fd) + except OSError: + return 42 + res = self.interpret(f, [9999]) + assert res == 42 + + @py.test.mark.skipif("not hasattr(os, 'tcsetpgrp')") + def test_tcsetpgrp(self): + def f(fd, pgrp): + try: + os.tcsetpgrp(fd, pgrp) + except OSError: + return 1 + return 0 + res = self.interpret(f, [9999, 1]) + assert res == 1 + + @py.test.mark.skipif("not hasattr(os, 'getresuid')") + def test_getresuid(self): + def f(): + a, b, c = os.getresuid() + return a + b * 37 + c * 1291 + res = self.interpret(f, []) + a, b, c = os.getresuid() + assert res == a + b * 37 + c * 1291 + + @py.test.mark.skipif("not hasattr(os, 'getresgid')") + def test_getresgid(self): + def f(): + a, b, c = os.getresgid() + return a + b * 37 + c * 1291 + res = self.interpret(f, []) + a, b, c = os.getresgid() + assert res == a + b * 37 + c * 1291 + + @py.test.mark.skipif("not hasattr(os, 'setresuid')") + def test_setresuid(self): + def f(): + a, b, c = os.getresuid() + a = (a + 1) - 1 + os.setresuid(a, b, c) + self.interpret(f, []) + + @py.test.mark.skipif("not hasattr(os, 'setresgid')") + def test_setresgid(self): + def f(): + a, b, c = os.getresgid() + a = (a + 1) - 1 + os.setresgid(a, b, c) + self.interpret(f, []) diff --git a/rpython/rlib/test/test_rawrefcount.py b/rpython/rlib/test/test_rawrefcount.py new file mode 100644 index 0000000000..6f095b9ae7 --- /dev/null +++ b/rpython/rlib/test/test_rawrefcount.py @@ -0,0 +1,268 @@ +import weakref +from rpython.rlib import rawrefcount, objectmodel, rgc +from rpython.rlib.rawrefcount import REFCNT_FROM_PYPY, REFCNT_FROM_PYPY_LIGHT +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.annlowlevel import llhelper +from rpython.translator.c.test.test_standalone import StandaloneTests +from rpython.config.translationoption import get_combined_translation_config + + +class W_Root(object): + def __init__(self, intval=0): + self.intval = intval + def __nonzero__(self): + raise Exception("you cannot do that, you must use space.is_true()") + +PyObjectS = lltype.Struct('PyObjectS', + ('c_ob_refcnt', lltype.Signed), + ('c_ob_pypy_link', lltype.Signed)) +PyObject = lltype.Ptr(PyObjectS) + + +class TestRawRefCount: + + def setup_method(self, meth): + rawrefcount.init() + + def test_create_link_pypy(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + assert rawrefcount.from_obj(PyObject, p) == lltype.nullptr(PyObjectS) + assert rawrefcount.to_obj(W_Root, ob) == None + rawrefcount.create_link_pypy(p, ob) + assert ob.c_ob_refcnt == 0 + ob.c_ob_refcnt += REFCNT_FROM_PYPY_LIGHT + assert rawrefcount.from_obj(PyObject, p) == ob + assert rawrefcount.to_obj(W_Root, ob) == p + lltype.free(ob, flavor='raw') + + def test_create_link_pyobj(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + assert rawrefcount.from_obj(PyObject, p) == lltype.nullptr(PyObjectS) + assert rawrefcount.to_obj(W_Root, ob) == None + rawrefcount.create_link_pyobj(p, ob) + assert ob.c_ob_refcnt == 0 + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount.from_obj(PyObject, p) == lltype.nullptr(PyObjectS) + assert rawrefcount.to_obj(W_Root, ob) == p + lltype.free(ob, flavor='raw') + + def test_collect_p_dies(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + rawrefcount.create_link_pypy(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY_LIGHT + assert rawrefcount._p_list == [ob] + wr_ob = weakref.ref(ob) + wr_p = weakref.ref(p) + del ob, p + rawrefcount._collect() + assert rawrefcount._p_list == [] + assert wr_ob() is None + assert wr_p() is None + + def test_collect_p_keepalive_pyobject(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + rawrefcount.create_link_pypy(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY_LIGHT + assert rawrefcount._p_list == [ob] + wr_ob = weakref.ref(ob) + wr_p = weakref.ref(p) + ob.c_ob_refcnt += 1 # <= + del ob, p + rawrefcount._collect() + ob = wr_ob() + p = wr_p() + assert ob is not None and p is not None + assert rawrefcount._p_list == [ob] + assert rawrefcount.to_obj(W_Root, ob) == p + assert rawrefcount.from_obj(PyObject, p) == ob + lltype.free(ob, flavor='raw') + + def test_collect_p_keepalive_w_root(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + rawrefcount.create_link_pypy(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY_LIGHT + assert rawrefcount._p_list == [ob] + wr_ob = weakref.ref(ob) + del ob # p remains + rawrefcount._collect() + ob = wr_ob() + assert ob is not None + assert rawrefcount._p_list == [ob] + assert rawrefcount.to_obj(W_Root, ob) == p + assert rawrefcount.from_obj(PyObject, p) == ob + lltype.free(ob, flavor='raw') + + def test_collect_o_dies(self): + trigger = []; rawrefcount.init(lambda: trigger.append(1)) + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + rawrefcount.create_link_pyobj(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount._o_list == [ob] + wr_ob = weakref.ref(ob) + wr_p = weakref.ref(p) + del ob, p + rawrefcount._collect() + ob = wr_ob() + assert ob is not None + assert trigger == [1] + assert rawrefcount.next_dead(PyObject) == ob + assert rawrefcount.next_dead(PyObject) == lltype.nullptr(PyObjectS) + assert rawrefcount.next_dead(PyObject) == lltype.nullptr(PyObjectS) + assert rawrefcount._o_list == [] + assert wr_p() is None + assert ob.c_ob_refcnt == 0 + assert ob.c_ob_pypy_link == 0 + lltype.free(ob, flavor='raw') + + def test_collect_o_keepalive_pyobject(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + p.pyobj = ob + rawrefcount.create_link_pyobj(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount._o_list == [ob] + wr_ob = weakref.ref(ob) + wr_p = weakref.ref(p) + ob.c_ob_refcnt += 1 # <= + del p + rawrefcount._collect() + p = wr_p() + assert p is None # was unlinked + assert ob.c_ob_refcnt == 1 # != REFCNT_FROM_PYPY_OBJECT + 1 + assert rawrefcount._o_list == [] + assert rawrefcount.to_obj(W_Root, ob) == None + lltype.free(ob, flavor='raw') + + def test_collect_o_keepalive_w_root(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + p.pyobj = ob + rawrefcount.create_link_pyobj(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount._o_list == [ob] + wr_ob = weakref.ref(ob) + del ob # p remains + rawrefcount._collect() + ob = wr_ob() + assert ob is not None + assert rawrefcount._o_list == [ob] + assert rawrefcount.to_obj(W_Root, ob) == p + assert p.pyobj == ob + lltype.free(ob, flavor='raw') + + def test_collect_s_dies(self): + trigger = []; rawrefcount.init(lambda: trigger.append(1)) + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + rawrefcount.create_link_pypy(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount._p_list == [ob] + wr_ob = weakref.ref(ob) + wr_p = weakref.ref(p) + del ob, p + rawrefcount._collect() + ob = wr_ob() + assert ob is not None + assert trigger == [1] + assert rawrefcount._d_list == [ob] + assert rawrefcount._p_list == [] + assert wr_p() is None + assert ob.c_ob_refcnt == 0 + assert ob.c_ob_pypy_link == 0 + lltype.free(ob, flavor='raw') + + def test_collect_s_keepalive_pyobject(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + p.pyobj = ob + rawrefcount.create_link_pypy(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount._p_list == [ob] + wr_ob = weakref.ref(ob) + wr_p = weakref.ref(p) + ob.c_ob_refcnt += 1 # <= + del ob, p + rawrefcount._collect() + ob = wr_ob() + p = wr_p() + assert ob is not None and p is not None + assert rawrefcount._p_list == [ob] + assert rawrefcount.to_obj(W_Root, ob) == p + lltype.free(ob, flavor='raw') + + def test_collect_s_keepalive_w_root(self): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + p.pyobj = ob + rawrefcount.create_link_pypy(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount._p_list == [ob] + wr_ob = weakref.ref(ob) + del ob # p remains + rawrefcount._collect() + ob = wr_ob() + assert ob is not None + assert rawrefcount._p_list == [ob] + assert rawrefcount.to_obj(W_Root, ob) == p + lltype.free(ob, flavor='raw') + + +class TestTranslated(StandaloneTests): + + def test_full_translation(self): + class State: + pass + state = State() + state.seen = [] + def dealloc_trigger(): + state.seen.append(1) + + def make_p(): + p = W_Root(42) + ob = lltype.malloc(PyObjectS, flavor='raw', zero=True) + rawrefcount.create_link_pypy(p, ob) + ob.c_ob_refcnt += REFCNT_FROM_PYPY + assert rawrefcount.from_obj(PyObject, p) == ob + assert rawrefcount.to_obj(W_Root, ob) == p + return ob, p + + FTYPE = rawrefcount.RAWREFCOUNT_DEALLOC_TRIGGER + + def entry_point(argv): + ll_dealloc_trigger_callback = llhelper(FTYPE, dealloc_trigger) + rawrefcount.init(ll_dealloc_trigger_callback) + ob, p = make_p() + if state.seen != []: + print "OB COLLECTED REALLY TOO SOON" + return 1 + rgc.collect() + if state.seen != []: + print "OB COLLECTED TOO SOON" + return 1 + objectmodel.keepalive_until_here(p) + p = None + rgc.collect() + if state.seen != [1]: + print "OB NOT COLLECTED" + return 1 + if rawrefcount.next_dead(PyObject) != ob: + print "NEXT_DEAD != OB" + return 1 + if rawrefcount.next_dead(PyObject) != lltype.nullptr(PyObjectS): + print "NEXT_DEAD second time != NULL" + return 1 + print "OK!" + lltype.free(ob, flavor='raw') + return 0 + + self.config = get_combined_translation_config(translating=True) + self.config.translation.gc = "incminimark" + t, cbuilder = self.compile(entry_point) + data = cbuilder.cmdexec('hi there') + assert data.startswith('OK!\n') diff --git a/rpython/rlib/test/test_rerased.py b/rpython/rlib/test/test_rerased.py index 9cb4924cfc..989a62bede 100644 --- a/rpython/rlib/test/test_rerased.py +++ b/rpython/rlib/test/test_rerased.py @@ -192,7 +192,7 @@ class TestRErased(BaseRtypingTest): def interpret(self, *args, **kwargs): kwargs["taggedpointers"] = True - return BaseRtypingTest.interpret(self, *args, **kwargs) + return BaseRtypingTest.interpret(*args, **kwargs) def test_rtype_1(self): def f(): return eraseX(X()) diff --git a/rpython/rtyper/extfunc.py b/rpython/rtyper/extfunc.py index 30d61ec4ee..0b926c9ce1 100644 --- a/rpython/rtyper/extfunc.py +++ b/rpython/rtyper/extfunc.py @@ -1,99 +1,105 @@ -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.rtyper.lltypesystem.lltype import typeOf, FuncType, functionptr -from rpython.annotator.model import unionof +from rpython.annotator.model import unionof, SomeObject from rpython.annotator.signature import annotation, SignatureError +from rpython.rtyper.extregistry import ExtRegistryEntry, lookup +from rpython.rtyper.lltypesystem.lltype import ( + typeOf, FuncType, functionptr, _ptr, Void) +from rpython.rtyper.error import TyperError +from rpython.rtyper.rmodel import Repr -import py +class SomeExternalFunction(SomeObject): + def __init__(self, name, args_s, s_result): + self.name = name + self.args_s = args_s + self.s_result = s_result -class ExtFuncEntry(ExtRegistryEntry): - safe_not_sandboxed = False + def check_args(self, callspec): + params_s = self.args_s + args_s, kwargs = callspec.unpack() + if kwargs: + raise SignatureError( + "External functions cannot be called with keyword arguments") + if len(args_s) != len(params_s): + raise SignatureError("Argument number mismatch") + for i, s_param in enumerate(params_s): + arg = unionof(args_s[i], s_param) + if not s_param.contains(arg): + raise SignatureError( + "In call to external function %r:\n" + "arg %d must be %s,\n" + " got %s" % ( + self.name, i + 1, s_param, args_s[i])) + + def call(self, callspec): + self.check_args(callspec) + return self.s_result + + def rtyper_makerepr(self, rtyper): + if not self.is_constant(): + raise TyperError("Non-constant external function!") + entry = lookup(self.const) + impl = getattr(entry, 'lltypeimpl', None) + fakeimpl = getattr(entry, 'lltypefakeimpl', None) + return ExternalFunctionRepr(self, impl, fakeimpl) + + def rtyper_makekey(self): + return self.__class__, self - # common case: args is a list of annotation or types - def normalize_args(self, *args_s): - args = self.signature_args - signature_args = [annotation(arg, None) for arg in args] - assert len(args_s) == len(signature_args),\ - "Argument number mismatch" - - for i, expected in enumerate(signature_args): - arg = unionof(args_s[i], expected) - if not expected.contains(arg): - name = getattr(self, 'name', None) - if not name: - try: - name = self.instance.__name__ - except AttributeError: - name = '?' - raise SignatureError("In call to external function %r:\n" - "arg %d must be %s,\n" - " got %s" % ( - name, i+1, expected, args_s[i])) - return signature_args - - def compute_result_annotation(self, *args_s): - self.normalize_args(*args_s) # check arguments - return self.signature_result - - def specialize_call(self, hop): +class ExternalFunctionRepr(Repr): + lowleveltype = Void + + def __init__(self, s_func, impl, fakeimpl): + self.s_func = s_func + self.impl = impl + self.fakeimpl = fakeimpl + + def rtype_simple_call(self, hop): rtyper = hop.rtyper - signature_args = self.normalize_args(*hop.args_s) - args_r = [rtyper.getrepr(s_arg) for s_arg in signature_args] + args_r = [rtyper.getrepr(s_arg) for s_arg in self.s_func.args_s] + r_result = rtyper.getrepr(self.s_func.s_result) + obj = self.get_funcptr(rtyper, args_r, r_result) + hop2 = hop.copy() + hop2.r_s_popfirstarg() + vlist = [hop2.inputconst(typeOf(obj), obj)] + hop2.inputargs(*args_r) + hop2.exception_is_here() + return hop2.genop('direct_call', vlist, r_result) + + def get_funcptr(self, rtyper, args_r, r_result): + from rpython.rtyper.rtyper import llinterp_backend args_ll = [r_arg.lowleveltype for r_arg in args_r] - s_result = hop.s_result - r_result = rtyper.getrepr(s_result) ll_result = r_result.lowleveltype - name = getattr(self, 'name', None) or self.instance.__name__ - impl = getattr(self, 'lltypeimpl', None) - fakeimpl = getattr(self, 'lltypefakeimpl', self.instance) - if impl: - if (rtyper.annotator.translator.config.translation.sandbox - and not self.safe_not_sandboxed): - from rpython.translator.sandbox.rsandbox import ( - make_sandbox_trampoline) - impl = make_sandbox_trampoline( - self.name, signature_args, s_result) - if hasattr(self, 'lltypefakeimpl'): - # If we have both an llimpl and an llfakeimpl, - # we need a wrapper that selects the proper one and calls it - from rpython.tool.sourcetools import func_with_new_name - # Using '*args' is delicate because this wrapper is also - # created for init-time functions like llarena.arena_malloc - # which are called before the GC is fully initialized - args = ', '.join(['arg%d' % i for i in range(len(args_ll))]) - d = {'original_impl': impl, - 's_result': s_result, - 'fakeimpl': fakeimpl, - '__name__': __name__, - } - exec py.code.compile(""" - from rpython.rlib.objectmodel import running_on_llinterp - from rpython.rlib.debug import llinterpcall - from rpython.rlib.jit import dont_look_inside - # note: we say 'dont_look_inside' mostly because the - # JIT does not support 'running_on_llinterp', but in - # theory it is probably right to stop jitting anyway. - @dont_look_inside - def ll_wrapper(%s): - if running_on_llinterp: - return llinterpcall(s_result, fakeimpl, %s) - else: - return original_impl(%s) - """ % (args, args, args)) in d - impl = func_with_new_name(d['ll_wrapper'], name + '_wrapper') - # store some attributes to the 'impl' function, where - # the eventual call to rtyper.getcallable() will find them - # and transfer them to the final lltype.functionptr(). - impl._llfnobjattrs_ = {'_name': self.name} - obj = rtyper.getannmixlevel().delayedfunction( - impl, signature_args, hop.s_result) + name = self.s_func.name + if self.fakeimpl and rtyper.backend is llinterp_backend: + FT = FuncType(args_ll, ll_result) + return functionptr( + FT, name, _external_name=name, _callable=self.fakeimpl) + elif self.impl: + if isinstance(self.impl, _ptr): + return self.impl + else: + # store some attributes to the 'impl' function, where + # the eventual call to rtyper.getcallable() will find them + # and transfer them to the final lltype.functionptr(). + self.impl._llfnobjattrs_ = {'_name': name} + return rtyper.getannmixlevel().delayedfunction( + self.impl, self.s_func.args_s, self.s_func.s_result) else: + fakeimpl = self.fakeimpl or self.s_func.const FT = FuncType(args_ll, ll_result) - obj = functionptr(FT, name, _external_name=self.name, - _callable=fakeimpl, - _safe_not_sandboxed=self.safe_not_sandboxed) - vlist = [hop.inputconst(typeOf(obj), obj)] + hop.inputargs(*args_r) - hop.exception_is_here() - return hop.genop('direct_call', vlist, r_result) + return functionptr( + FT, name, _external_name=name, _callable=fakeimpl) + + +class ExtFuncEntry(ExtRegistryEntry): + safe_not_sandboxed = False + + def compute_annotation(self): + s_result = SomeExternalFunction( + self.name, self.signature_args, self.signature_result) + if (self.bookkeeper.annotator.translator.config.translation.sandbox + and not self.safe_not_sandboxed): + s_result.needs_sandboxing = True + return s_result + def register_external(function, args, result=None, export_name=None, llimpl=None, llfakeimpl=None, sandboxsafe=False): @@ -109,32 +115,20 @@ def register_external(function, args, result=None, export_name=None, if export_name is None: export_name = function.__name__ + params_s = [annotation(arg) for arg in args] + s_result = annotation(result) class FunEntry(ExtFuncEntry): _about_ = function safe_not_sandboxed = sandboxsafe - - if args is None: - def normalize_args(self, *args_s): - return args_s # accept any argument unmodified - elif callable(args): - # custom annotation normalizer (see e.g. os.utime()) - normalize_args = staticmethod(args) - else: # use common case behavior - signature_args = args - - signature_result = annotation(result, None) + signature_args = params_s + signature_result = s_result name = export_name if llimpl: lltypeimpl = staticmethod(llimpl) if llfakeimpl: lltypefakeimpl = staticmethod(llfakeimpl) - if export_name: - FunEntry.__name__ = export_name - else: - FunEntry.__name__ = function.func_name - def is_external(func): if hasattr(func, 'value'): func = func.value diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py index caeea15e6f..8f29a3208e 100644 --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -925,6 +925,21 @@ class LLFrame(object): def op_gc_gcflag_extra(self, subopnum, *args): return self.heap.gcflag_extra(subopnum, *args) + def op_gc_rawrefcount_init(self, *args): + raise NotImplementedError("gc_rawrefcount_init") + + def op_gc_rawrefcount_to_obj(self, *args): + raise NotImplementedError("gc_rawrefcount_to_obj") + + def op_gc_rawrefcount_from_obj(self, *args): + raise NotImplementedError("gc_rawrefcount_from_obj") + + def op_gc_rawrefcount_create_link_pyobj(self, *args): + raise NotImplementedError("gc_rawrefcount_create_link_pyobj") + + def op_gc_rawrefcount_create_link_pypy(self, *args): + raise NotImplementedError("gc_rawrefcount_create_link_pypy") + def op_do_malloc_fixedsize(self): raise NotImplementedError("do_malloc_fixedsize") def op_do_malloc_fixedsize_clear(self): diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py index 43a59f82cf..664be37524 100644 --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -426,7 +426,12 @@ def convert_struct(container, cstruct=None, delayed_converters=None): else: n = None cstruct = cls._malloc(n) - add_storage(container, _struct_mixin, ctypes.pointer(cstruct)) + + if isinstance(container, lltype._fixedsizearray): + cls_mixin = _fixedsizedarray_mixin + else: + cls_mixin = _struct_mixin + add_storage(container, cls_mixin, ctypes.pointer(cstruct)) if delayed_converters is None: delayed_converters_was_None = True @@ -463,6 +468,9 @@ def convert_struct(container, cstruct=None, delayed_converters=None): def remove_regular_struct_content(container): STRUCT = container._TYPE + if isinstance(STRUCT, lltype.FixedSizeArray): + del container._items + return for field_name in STRUCT._names: FIELDTYPE = getattr(STRUCT, field_name) if not isinstance(FIELDTYPE, lltype.ContainerType): @@ -503,7 +511,11 @@ def remove_regular_array_content(container): def struct_use_ctypes_storage(container, ctypes_storage): STRUCT = container._TYPE assert isinstance(STRUCT, lltype.Struct) - add_storage(container, _struct_mixin, ctypes_storage) + if isinstance(container, lltype._fixedsizearray): + cls_mixin = _fixedsizedarray_mixin + else: + cls_mixin = _struct_mixin + add_storage(container, cls_mixin, ctypes_storage) remove_regular_struct_content(container) for field_name in STRUCT._names: FIELDTYPE = getattr(STRUCT, field_name) @@ -515,8 +527,10 @@ def struct_use_ctypes_storage(container, ctypes_storage): struct_use_ctypes_storage(struct_container, struct_storage) struct_container._setparentstructure(container, field_name) elif isinstance(FIELDTYPE, lltype.Array): - assert FIELDTYPE._hints.get('nolength', False) == False - arraycontainer = _array_of_known_length(FIELDTYPE) + if FIELDTYPE._hints.get('nolength', False): + arraycontainer = _array_of_unknown_length(FIELDTYPE) + else: + arraycontainer = _array_of_known_length(FIELDTYPE) arraycontainer._storage = ctypes.pointer( getattr(ctypes_storage.contents, field_name)) arraycontainer._setparentstructure(container, field_name) @@ -528,6 +542,7 @@ def struct_use_ctypes_storage(container, ctypes_storage): # Ctypes-aware subclasses of the _parentable classes ALLOCATED = {} # mapping {address: _container} +DEBUG_ALLOCATED = False def get_common_subclass(cls1, cls2, cache={}): """Return a unique subclass with (cls1, cls2) as bases.""" @@ -567,6 +582,8 @@ class _parentable_mixin(object): raise Exception("internal ll2ctypes error - " "double conversion from lltype to ctypes?") # XXX don't store here immortal structures + if DEBUG_ALLOCATED: + print >> sys.stderr, "LL2CTYPES:", hex(addr) ALLOCATED[addr] = self def _addressof_storage(self): @@ -579,6 +596,8 @@ class _parentable_mixin(object): self._check() # no double-frees # allow the ctypes object to go away now addr = ctypes.cast(self._storage, ctypes.c_void_p).value + if DEBUG_ALLOCATED: + print >> sys.stderr, "LL2C FREE:", hex(addr) try: del ALLOCATED[addr] except KeyError: @@ -613,11 +632,14 @@ class _parentable_mixin(object): return object.__hash__(self) def __repr__(self): + if '__str__' in self._TYPE._adtmeths: + r = self._TYPE._adtmeths['__str__'](self) + else: + r = 'C object %s' % (self._TYPE,) if self._storage is None: - return '<freed C object %s>' % (self._TYPE,) + return '<freed %s>' % (r,) else: - return '<C object %s at 0x%x>' % (self._TYPE, - fixid(self._addressof_storage())) + return '<%s at 0x%x>' % (r, fixid(self._addressof_storage())) def __str__(self): return repr(self) @@ -642,6 +664,45 @@ class _struct_mixin(_parentable_mixin): cobj = lltype2ctypes(value) setattr(self._storage.contents, field_name, cobj) +class _fixedsizedarray_mixin(_parentable_mixin): + """Mixin added to _fixedsizearray containers when they become ctypes-based.""" + __slots__ = () + + def __getattr__(self, field_name): + if hasattr(self, '_items'): + obj = lltype._fixedsizearray.__getattr__.im_func(self, field_name) + return obj + else: + cobj = getattr(self._storage.contents, field_name) + T = getattr(self._TYPE, field_name) + return ctypes2lltype(T, cobj) + + def __setattr__(self, field_name, value): + if field_name.startswith('_'): + object.__setattr__(self, field_name, value) # '_xxx' attributes + else: + cobj = lltype2ctypes(value) + if hasattr(self, '_items'): + lltype._fixedsizearray.__setattr__.im_func(self, field_name, cobj) + else: + setattr(self._storage.contents, field_name, cobj) + + + def getitem(self, index, uninitialized_ok=False): + if hasattr(self, '_items'): + obj = lltype._fixedsizearray.getitem.im_func(self, + index, uninitialized_ok=uninitialized_ok) + return obj + else: + return getattr(self, 'item%d' % index) + + def setitem(self, index, value): + cobj = lltype2ctypes(value) + if hasattr(self, '_items'): + lltype._fixedsizearray.setitem.im_func(self, index, value) + else: + setattr(self, 'item%d' % index, cobj) + class _array_mixin(_parentable_mixin): """Mixin added to _array containers when they become ctypes-based.""" __slots__ = () @@ -942,7 +1003,8 @@ def ctypes2lltype(T, cobj): REAL_TYPE = T.TO if T.TO._arrayfld is not None: carray = getattr(cobj.contents, T.TO._arrayfld) - container = lltype._struct(T.TO, carray.length) + length = getattr(carray, 'length', 9999) # XXX + container = lltype._struct(T.TO, length) else: # special treatment of 'OBJECT' subclasses if get_rtyper() and lltype._castdepth(REAL_TYPE, OBJECT) >= 0: diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py index 87256ac849..9368164578 100644 --- a/rpython/rtyper/lltypesystem/lloperation.py +++ b/rpython/rtyper/lltypesystem/lloperation.py @@ -453,6 +453,8 @@ LL_OPERATIONS = { 'jit_record_exact_class' : LLOp(canrun=True), 'jit_ffi_save_result': LLOp(canrun=True), 'jit_conditional_call': LLOp(), + 'jit_enter_portal_frame': LLOp(canrun=True), + 'jit_leave_portal_frame': LLOp(canrun=True), 'get_exception_addr': LLOp(), 'get_exc_value_addr': LLOp(), 'do_malloc_fixedsize':LLOp(canmallocgc=True), @@ -503,6 +505,12 @@ LL_OPERATIONS = { 'gc_gcflag_extra' : LLOp(), 'gc_add_memory_pressure': LLOp(), + 'gc_rawrefcount_init': LLOp(), + 'gc_rawrefcount_create_link_pypy': LLOp(), + 'gc_rawrefcount_create_link_pyobj': LLOp(), + 'gc_rawrefcount_from_obj': LLOp(sideeffects=False), + 'gc_rawrefcount_to_obj': LLOp(sideeffects=False), + # ------- JIT & GC interaction, only for some GCs ---------- 'gc_adr_of_nursery_free' : LLOp(), diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py index ac95f57e2d..677170a553 100644 --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -1761,7 +1761,10 @@ class _struct(_parentable): def __new__(self, TYPE, n=None, initialization=None, parent=None, parentindex=None): - my_variety = _struct_variety(TYPE._names) + if isinstance(TYPE, FixedSizeArray): + my_variety = _fixedsizearray + else: + my_variety = _struct_variety(TYPE._names) return object.__new__(my_variety) def __init__(self, TYPE, n=None, initialization=None, parent=None, @@ -1771,7 +1774,6 @@ class _struct(_parentable): raise TypeError("%r is not variable-sized" % (TYPE,)) if n is None and TYPE._arrayfld is not None: raise TypeError("%r is variable-sized" % (TYPE,)) - first, FIRSTTYPE = TYPE._first_struct() for fld, typ in TYPE._flds.items(): if fld == TYPE._arrayfld: value = _array(typ, n, initialization=initialization, @@ -1814,23 +1816,48 @@ class _struct(_parentable): raise UninitializedMemoryAccess("%r.%s"%(self, field_name)) return r - # for FixedSizeArray kind of structs: + +class _fixedsizearray(_struct): + def __init__(self, TYPE, n=None, initialization=None, parent=None, + parentindex=None): + _parentable.__init__(self, TYPE) + if n is not None: + raise TypeError("%r is not variable-sized" % (TYPE,)) + typ = TYPE.OF + storage = [] + for i, fld in enumerate(TYPE._names): + value = typ._allocate(initialization=initialization, + parent=self, parentindex=fld) + storage.append(value) + self._items = storage + if parent is not None: + self._setparentstructure(parent, parentindex) def getlength(self): - assert isinstance(self._TYPE, FixedSizeArray) return self._TYPE.length def getbounds(self): return 0, self.getlength() def getitem(self, index, uninitialized_ok=False): - assert isinstance(self._TYPE, FixedSizeArray) - return self._getattr('item%d' % index, uninitialized_ok) + assert 0 <= index < self.getlength() + return self._items[index] def setitem(self, index, value): - assert isinstance(self._TYPE, FixedSizeArray) - setattr(self, 'item%d' % index, value) + assert 0 <= index < self.getlength() + self._items[index] = value + def __getattr__(self, name): + # obscure + if name.startswith("item"): + return self.getitem(int(name[len('item'):])) + return _struct.__getattr__(self, name) + + def __setattr__(self, name, value): + if name.startswith("item"): + self.setitem(int(name[len('item'):]), value) + return + _struct.__setattr__(self, name, value) class _array(_parentable): _kind = "array" diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py index 259b709d69..f3bbe4b033 100644 --- a/rpython/rtyper/lltypesystem/module/ll_math.py +++ b/rpython/rtyper/lltypesystem/module/ll_math.py @@ -6,8 +6,8 @@ import sys from rpython.translator import cdir from rpython.rlib import jit, rposix from rpython.rlib.rfloat import INFINITY, NAN, isfinite, isinf, isnan +from rpython.rlib.rposix import UNDERSCORE_ON_WIN32 from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.module.support import UNDERSCORE_ON_WIN32 from rpython.tool.sourcetools import func_with_new_name from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py index dd702f73af..ae7db848a3 100644 --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -624,6 +624,12 @@ def op_jit_record_exact_class(x, y): def op_jit_ffi_save_result(*args): pass +def op_jit_enter_portal_frame(x): + pass + +def op_jit_leave_portal_frame(): + pass + def op_get_group_member(TYPE, grpptr, memberoffset): from rpython.rtyper.lltypesystem import llgroup assert isinstance(memberoffset, llgroup.GroupMemberOffset) diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py index 0b5a124b06..ec0811d5a3 100644 --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -631,7 +631,8 @@ def COpaquePtr(*args, **kwds): def CExternVariable(TYPE, name, eci, _CConstantClass=CConstant, sandboxsafe=False, _nowrapper=False, - c_type=None, getter_only=False): + c_type=None, getter_only=False, + declare_as_extern=(sys.platform != 'win32')): """Return a pair of functions - a getter and a setter - to access the given global C variable. """ @@ -661,7 +662,7 @@ def CExternVariable(TYPE, name, eci, _CConstantClass=CConstant, c_setter = "void %(setter_name)s (%(c_type)s v) { %(name)s = v; }" % locals() lines = ["#include <%s>" % i for i in eci.includes] - if sys.platform != 'win32': + if declare_as_extern: lines.append('extern %s %s;' % (c_type, name)) lines.append(c_getter) if not getter_only: @@ -790,6 +791,12 @@ def make_string_mappings(strtype): return length str2chararray._annenforceargs_ = [strtype, None, int] + # s[start:start+length] -> already-existing char[], + # all characters including zeros + def str2rawmem(s, array, start, length): + ll_s = llstrtype(s) + copy_string_to_raw(ll_s, array, start, length) + # char* -> str # doesn't free char* def charp2str(cp): @@ -940,19 +947,19 @@ def make_string_mappings(strtype): return (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, str2chararray, + charp2strn, charpsize2str, str2chararray, str2rawmem, ) (str2charp, free_charp, charp2str, get_nonmovingbuffer, free_nonmovingbuffer, alloc_buffer, str_from_buffer, keep_buffer_alive_until_here, - charp2strn, charpsize2str, str2chararray, + charp2strn, charpsize2str, str2chararray, str2rawmem, ) = make_string_mappings(str) (unicode2wcharp, free_wcharp, wcharp2unicode, get_nonmoving_unicodebuffer, free_nonmoving_unicodebuffer, alloc_unicodebuffer, unicode_from_buffer, keep_unicodebuffer_alive_until_here, - wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, + wcharp2unicoden, wcharpsize2unicode, unicode2wchararray, unicode2rawmem, ) = make_string_mappings(unicode) # char** diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py index b99c607e49..6e76ab4526 100644 --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -717,10 +717,7 @@ class LLHelpers(AbstractLLHelpers): return cls.ll_count_char(s1, s2.chars[0], start, end) res = cls.ll_search(s1, s2, start, end, FAST_COUNT) - # For a few cases ll_search can return -1 to indicate an "impossible" - # condition for a string match, count just returns 0 in these cases. - if res < 0: - res = 0 + assert res >= 0 return res @staticmethod @@ -741,6 +738,8 @@ class LLHelpers(AbstractLLHelpers): w = n - m if w < 0: + if mode == FAST_COUNT: + return 0 return -1 mlast = m - 1 diff --git a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py index 4b1e59c884..3a9535925c 100644 --- a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py @@ -11,12 +11,12 @@ from rpython.rtyper.lltypesystem.ll2ctypes import cast_adr_to_int, get_ctypes_ty from rpython.rtyper.lltypesystem.ll2ctypes import _llgcopaque from rpython.rtyper.annlowlevel import llhelper from rpython.rlib import rposix +from rpython.rlib.rposix import UNDERSCORE_ON_WIN32 from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator import cdir from rpython.tool.udir import udir from rpython.rtyper.test.test_llinterp import interpret from rpython.annotator.annrpython import RPythonAnnotator -from rpython.rtyper.module.support import UNDERSCORE_ON_WIN32 from rpython.rtyper.rtyper import RPythonTyper from rpython.rlib.rarithmetic import r_uint, get_long_pattern, is_emulated_long from rpython.rlib.rarithmetic import is_valid_int @@ -1466,3 +1466,20 @@ class TestPlatform(object): assert a[3].a == 17 #lltype.free(a, flavor='raw') py.test.skip("free() not working correctly here...") + + def test_fixedsizedarray_to_ctypes(self): + T = lltype.Ptr(rffi.CFixedArray(rffi.INT, 1)) + inst = lltype.malloc(T.TO, flavor='raw') + inst[0] = rffi.cast(rffi.INT, 42) + assert inst[0] == 42 + cinst = lltype2ctypes(inst) + assert rffi.cast(lltype.Signed, inst[0]) == 42 + assert cinst.contents.item0 == 42 + lltype.free(inst, flavor='raw') + + def test_fixedsizedarray_to_ctypes(self): + T = lltype.Ptr(rffi.CFixedArray(rffi.CHAR, 123)) + inst = lltype.malloc(T.TO, flavor='raw', zero=True) + cinst = lltype2ctypes(inst) + assert cinst.contents.item0 == 0 + lltype.free(inst, flavor='raw') diff --git a/rpython/rtyper/module/__init__.py b/rpython/rtyper/module/__init__.py deleted file mode 100644 index 792d600548..0000000000 --- a/rpython/rtyper/module/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/rpython/rtyper/module/test/__init__.py b/rpython/rtyper/module/test/__init__.py deleted file mode 100644 index 792d600548..0000000000 --- a/rpython/rtyper/module/test/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# diff --git a/rpython/rtyper/module/test/test_ll_strtod.py b/rpython/rtyper/module/test/test_ll_strtod.py deleted file mode 100644 index 4e1788eb69..0000000000 --- a/rpython/rtyper/module/test/test_ll_strtod.py +++ /dev/null @@ -1,13 +0,0 @@ -import py - -from rpython.rtyper.test.tool import BaseRtypingTest -from rpython.rlib import rfloat - -class TestStrtod(BaseRtypingTest): - def test_formatd(self): - for flags in [0, - rfloat.DTSF_ADD_DOT_0]: - def f(y): - return rfloat.formatd(y, 'g', 2, flags) - - assert self.ll_to_string(self.interpret(f, [3.0])) == f(3.0) diff --git a/rpython/rtyper/module/test/test_posix.py b/rpython/rtyper/module/test/test_posix.py deleted file mode 100644 index df091ed1c4..0000000000 --- a/rpython/rtyper/module/test/test_posix.py +++ /dev/null @@ -1,304 +0,0 @@ -import py -from rpython.rtyper.test.tool import BaseRtypingTest -from rpython.rtyper.annlowlevel import hlstr -from rpython.tool.udir import udir -from rpython.rlib.rarithmetic import is_valid_int - -import os -exec 'import %s as posix' % os.name - -def setup_module(module): - testf = udir.join('test.txt') - module.path = testf.strpath - -class TestPosix(BaseRtypingTest): - - def setup_method(self, meth): - # prepare/restore the file before each test - testfile = open(path, 'wb') - testfile.write('This is a test') - testfile.close() - - def test_open(self): - def f(): - ff = posix.open(path, posix.O_RDONLY, 0777) - return ff - func = self.interpret(f, []) - assert is_valid_int(func) - - def test_fstat(self): - def fo(fi): - g = posix.fstat(fi) - return g - fi = os.open(path,os.O_RDONLY,0777) - func = self.interpret(fo,[fi]) - stat = os.fstat(fi) - for i in range(len(stat)): - assert long(getattr(func, 'item%d' % i)) == stat[i] - - - def test_stat(self): - def fo(): - g = posix.stat(path) - return g - func = self.interpret(fo,[]) - stat = os.stat(path) - for i in range(len(stat)): - assert long(getattr(func, 'item%d' % i)) == stat[i] - - def test_stat_exception(self): - def fo(): - try: - posix.stat('I/do/not/exist') - except OSError: - return True - else: - return False - res = self.interpret(fo,[]) - assert res - - def test_times(self): - import py; py.test.skip("llinterp does not like tuple returns") - from rpython.rtyper.test.test_llinterp import interpret - times = interpret(lambda: posix.times(), ()) - assert isinstance(times, tuple) - assert len(times) == 5 - for value in times: - assert is_valid_int(value) - - - def test_lseek(self): - def f(fi, pos): - posix.lseek(fi, pos, 0) - fi = os.open(path, os.O_RDONLY, 0777) - func = self.interpret(f, [fi, 5]) - res = os.read(fi, 2) - assert res =='is' - - def test_isatty(self): - def f(fi): - posix.isatty(fi) - fi = os.open(path, os.O_RDONLY, 0777) - func = self.interpret(f, [fi]) - assert not func - os.close(fi) - func = self.interpret(f, [fi]) - assert not func - - def test_getcwd(self): - def f(): - return posix.getcwd() - res = self.interpret(f,[]) - cwd = os.getcwd() - #print res.chars,cwd - assert self.ll_to_string(res) == cwd - - def test_write(self): - def f(fi): - if fi > 0: - text = 'This is a test' - else: - text = '333' - return posix.write(fi,text) - fi = os.open(path,os.O_WRONLY,0777) - text = 'This is a test' - func = self.interpret(f,[fi]) - os.close(fi) - fi = os.open(path,os.O_RDONLY,0777) - res = os.read(fi,20) - assert res == text - - def test_read(self): - def f(fi,len): - return posix.read(fi,len) - fi = os.open(path,os.O_WRONLY,0777) - text = 'This is a test' - os.write(fi,text) - os.close(fi) - fi = os.open(path,os.O_RDONLY,0777) - res = self.interpret(f,[fi,20]) - assert self.ll_to_string(res) == text - - if hasattr(os, 'chown'): - def test_chown(self): - f = open(path, "w") - f.write("xyz") - f.close() - def f(): - try: - posix.chown(path, os.getuid(), os.getgid()) - return 1 - except OSError: - return 2 - - assert self.interpret(f, []) == 1 - os.unlink(path) - assert self.interpret(f, []) == 2 - - def test_close(self): - def f(fi): - return posix.close(fi) - fi = os.open(path,os.O_WRONLY,0777) - text = 'This is a test' - os.write(fi,text) - res = self.interpret(f,[fi]) - py.test.raises( OSError, os.fstat, fi) - - if hasattr(os, 'ftruncate'): - def test_ftruncate(self): - def f(fi,len): - os.ftruncate(fi,len) - fi = os.open(path,os.O_RDWR,0777) - func = self.interpret(f,[fi,6]) - assert os.fstat(fi).st_size == 6 - - if hasattr(os, 'getuid'): - def test_getuid(self): - def f(): - return os.getuid() - assert self.interpret(f, []) == f() - - if hasattr(os, 'getgid'): - def test_getgid(self): - def f(): - return os.getgid() - assert self.interpret(f, []) == f() - - if hasattr(os, 'setuid'): - def test_os_setuid(self): - def f(): - os.setuid(os.getuid()) - return os.getuid() - assert self.interpret(f, []) == f() - - if hasattr(os, 'sysconf'): - def test_os_sysconf(self): - def f(i): - return os.sysconf(i) - assert self.interpret(f, [13]) == f(13) - - if hasattr(os, 'confstr'): - def test_os_confstr(self): - def f(i): - try: - return os.confstr(i) - except OSError: - return "oooops!!" - some_value = os.confstr_names.values()[-1] - res = self.interpret(f, [some_value]) - assert hlstr(res) == f(some_value) - res = self.interpret(f, [94781413]) - assert hlstr(res) == "oooops!!" - - if hasattr(os, 'pathconf'): - def test_os_pathconf(self): - def f(i): - return os.pathconf("/tmp", i) - i = os.pathconf_names["PC_NAME_MAX"] - some_value = self.interpret(f, [i]) - assert some_value >= 31 - - if hasattr(os, 'chroot'): - def test_os_chroot(self): - def f(): - try: - os.chroot('!@$#!#%$#^#@!#!$$#^') - except OSError: - return 1 - return 0 - - assert self.interpret(f, []) == 1 - - def test_os_wstar(self): - from rpython.rlib import rposix - for name in rposix.WAIT_MACROS: - if not hasattr(os, name): - continue - def fun(s): - return getattr(os, name)(s) - - for value in [0, 1, 127, 128, 255]: - res = self.interpret(fun, [value]) - assert res == fun(value) - - if hasattr(os, 'getgroups'): - def test_getgroups(self): - def f(): - return os.getgroups() - ll_a = self.interpret(f, []) - assert self.ll_to_list(ll_a) == f() - - if hasattr(os, 'setgroups'): - def test_setgroups(self): - def f(): - try: - os.setgroups(os.getgroups()) - except OSError: - pass - self.interpret(f, []) - - if hasattr(os, 'initgroups'): - def test_initgroups(self): - def f(): - try: - os.initgroups('sUJJeumz', 4321) - except OSError: - return 1 - return 0 - res = self.interpret(f, []) - assert res == 1 - - if hasattr(os, 'tcgetpgrp'): - def test_tcgetpgrp(self): - def f(fd): - try: - return os.tcgetpgrp(fd) - except OSError: - return 42 - res = self.interpret(f, [9999]) - assert res == 42 - - if hasattr(os, 'tcsetpgrp'): - def test_tcsetpgrp(self): - def f(fd, pgrp): - try: - os.tcsetpgrp(fd, pgrp) - except OSError: - return 1 - return 0 - res = self.interpret(f, [9999, 1]) - assert res == 1 - - if hasattr(os, 'getresuid'): - def test_getresuid(self): - def f(): - a, b, c = os.getresuid() - return a + b * 37 + c * 1291 - res = self.interpret(f, []) - a, b, c = os.getresuid() - assert res == a + b * 37 + c * 1291 - - if hasattr(os, 'getresgid'): - def test_getresgid(self): - def f(): - a, b, c = os.getresgid() - return a + b * 37 + c * 1291 - res = self.interpret(f, []) - a, b, c = os.getresgid() - assert res == a + b * 37 + c * 1291 - - if hasattr(os, 'setresuid'): - def test_setresuid(self): - def f(): - a, b, c = os.getresuid() - a = (a + 1) - 1 - os.setresuid(a, b, c) - self.interpret(f, []) - - if hasattr(os, 'setresgid'): - def test_setresgid(self): - def f(): - a, b, c = os.getresgid() - a = (a + 1) - 1 - os.setresgid(a, b, c) - self.interpret(f, []) diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py index 7734f74554..dfe867ad00 100644 --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -32,11 +32,24 @@ from rpython.translator.unsimplify import insert_empty_block from rpython.translator.sandbox.rsandbox import make_sandbox_trampoline +class RTyperBackend(object): + pass + +class GenCBackend(RTyperBackend): + pass +genc_backend = GenCBackend() + +class LLInterpBackend(RTyperBackend): + pass +llinterp_backend = LLInterpBackend() + + class RPythonTyper(object): from rpython.rtyper.rmodel import log - def __init__(self, annotator): + def __init__(self, annotator, backend=genc_backend): self.annotator = annotator + self.backend = backend self.lowlevel_ann_policy = LowLevelAnnotatorPolicy(self) self.reprs = {} self._reprs_must_call_setup = [] diff --git a/rpython/rtyper/test/test_extfunc.py b/rpython/rtyper/test/test_extfunc.py index c6b3948a9a..778be158f7 100644 --- a/rpython/rtyper/test/test_extfunc.py +++ b/rpython/rtyper/test/test_extfunc.py @@ -1,7 +1,6 @@ import py -from rpython.rtyper.extfunc import ExtFuncEntry, register_external,\ - is_external +from rpython.rtyper.extfunc import register_external from rpython.annotator.model import SomeInteger, SomeString, AnnotatorError from rpython.annotator.annrpython import RPythonAnnotator from rpython.annotator.policy import AnnotatorPolicy @@ -19,11 +18,7 @@ class TestExtFuncEntry: "NOT_RPYTHON" return eval("x+40") - class BTestFuncEntry(ExtFuncEntry): - _about_ = b - name = 'b' - signature_args = [SomeInteger()] - signature_result = SomeInteger() + register_external(b, [int], result=int) def f(): return b(2) @@ -43,15 +38,11 @@ class TestExtFuncEntry: def c(y, x): yyy - class CTestFuncEntry(ExtFuncEntry): - _about_ = c - name = 'ccc' - signature_args = [SomeInteger()] * 2 - signature_result = SomeInteger() + def llimpl(y, x): + return y + x - def lltypeimpl(y, x): - return y + x - lltypeimpl = staticmethod(lltypeimpl) + register_external(c, [int, int], result=int, llimpl=llimpl, + export_name='ccc') def f(): return c(3, 4) @@ -59,22 +50,6 @@ class TestExtFuncEntry: res = interpret(f, []) assert res == 7 - def test_register_external_signature(self): - """ - Test the standard interface for external functions. - """ - def dd(): - pass - register_external(dd, [int], int) - - def f(): - return dd(3) - - policy = AnnotatorPolicy() - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, SomeInteger) - def test_register_external_tuple_args(self): """ Verify the annotation of a registered external function which takes a @@ -121,23 +96,6 @@ class TestExtFuncEntry: s = a.build_types(f, []) assert isinstance(s, SomeInteger) - def test_register_external_specialcase(self): - """ - When args=None, the external function accepts any arguments unmodified. - """ - def function_withspecialcase(arg): - return repr(arg) - register_external(function_withspecialcase, args=None, result=str) - - def f(): - x = function_withspecialcase - return x(33) + x("aaa") + x([]) + "\n" - - policy = AnnotatorPolicy() - a = RPythonAnnotator(policy=policy) - s = a.build_types(f, []) - assert isinstance(s, SomeString) - def test_str0(self): str0 = SomeString(no_nul=True) def os_open(s): @@ -182,3 +140,22 @@ class TestExtFuncEntry: # fails with TooLateForChange a.build_types(g, [[str]]) a.build_types(g, [[str0]]) # Does not raise + + def test_register_external_llfakeimpl(self): + def a(i): + return i + def a_llimpl(i): + return i * 2 + def a_llfakeimpl(i): + return i * 3 + register_external(a, [int], int, llimpl=a_llimpl, + llfakeimpl=a_llfakeimpl) + def f(i): + return a(i) + + res = interpret(f, [7]) + assert res == 21 + + from rpython.translator.c.test.test_genc import compile + fc = compile(f, [int]) + assert fc(7) == 14 diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py index 9c8796ac6c..479a1de0a9 100644 --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -13,7 +13,7 @@ from rpython.rtyper.llannotation import lltype_to_annotation from rpython.rlib.rarithmetic import r_uint, ovfcheck from rpython.tool import leakfinder from rpython.conftest import option - +from rpython.rtyper.rtyper import llinterp_backend # switch on logging of interp to show more info on failing tests @@ -39,6 +39,7 @@ def gengraph(func, argtypes=[], viewbefore='auto', policy=None, t.view() global typer # we need it for find_exception typer = t.buildrtyper() + typer.backend = llinterp_backend typer.specialize() #t.view() t.checkgraphs() diff --git a/rpython/rtyper/test/test_rbuiltin.py b/rpython/rtyper/test/test_rbuiltin.py index 5531dcaedc..767fd1d7ed 100644 --- a/rpython/rtyper/test/test_rbuiltin.py +++ b/rpython/rtyper/test/test_rbuiltin.py @@ -3,8 +3,7 @@ import os import py -from rpython.rlib.debug import llinterpcall -from rpython.rlib.objectmodel import instantiate, running_on_llinterp, compute_unique_id, current_object_addr_as_int +from rpython.rlib.objectmodel import instantiate, compute_unique_id, current_object_addr_as_int from rpython.rlib.rarithmetic import (intmask, longlongmask, r_int64, is_valid_int, r_int, r_uint, r_longlong, r_ulonglong) from rpython.rlib.rstring import StringBuilder, UnicodeBuilder @@ -456,26 +455,6 @@ class TestRbuiltin(BaseRtypingTest): res = self.interpret(fn, [3.25]) assert res == 7.25 - def test_debug_llinterpcall(self): - S = lltype.Struct('S', ('m', lltype.Signed)) - SPTR = lltype.Ptr(S) - def foo(n): - "NOT_RPYTHON" - s = lltype.malloc(S, immortal=True) - s.m = eval("n*6", locals()) - return s - def fn(n): - if running_on_llinterp: - return llinterpcall(SPTR, foo, n).m - else: - return 321 - res = self.interpret(fn, [7]) - assert res == 42 - from rpython.translator.c.test.test_genc import compile - f = compile(fn, [int]) - res = f(7) - assert res == 321 - def test_id(self): class A: pass diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py index f71aaf8174..e9a8b1c718 100644 --- a/rpython/rtyper/test/test_rdict.py +++ b/rpython/rtyper/test/test_rdict.py @@ -1,13 +1,63 @@ +import sys +from contextlib import contextmanager +import signal + from rpython.translator.translator import TranslationContext +from rpython.annotator.model import ( + SomeInteger, SomeString, SomeChar, SomeUnicodeString, SomeUnicodeCodePoint) +from rpython.annotator.dictdef import DictKey, DictValue from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper import rint -from rpython.rtyper.lltypesystem import rdict, rstr +from rpython.rtyper.lltypesystem import rdict from rpython.rtyper.test.tool import BaseRtypingTest from rpython.rlib.objectmodel import r_dict from rpython.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong import py -py.log.setconsumer("rtyper", py.log.STDOUT) +from hypothesis import settings +from hypothesis.strategies import ( + builds, sampled_from, binary, just, integers, text, characters, tuples) +from hypothesis.stateful import GenericStateMachine, run_state_machine_as_test + +def ann2strategy(s_value): + if isinstance(s_value, SomeChar): + return builds(chr, integers(min_value=0, max_value=255)) + elif isinstance(s_value, SomeString): + if s_value.can_be_None: + return binary() | just(None) + else: + return binary() + elif isinstance(s_value, SomeUnicodeCodePoint): + return characters() + elif isinstance(s_value, SomeUnicodeString): + if s_value.can_be_None: + return text() | just(None) + else: + return text() + elif isinstance(s_value, SomeInteger): + return integers(min_value=~sys.maxint, max_value=sys.maxint) + else: + raise TypeError("Cannot convert annotation %s to a strategy" % s_value) + + +if hasattr(signal, 'alarm'): + @contextmanager + def signal_timeout(n): + """A flaky context manager that throws an exception if the body of the + `with` block runs for longer than `n` seconds. + """ + def handler(signum, frame): + raise RuntimeError('timeout') + signal.signal(signal.SIGALRM, handler) + signal.alarm(n) + try: + yield + finally: + signal.alarm(0) +else: + @contextmanager + def signal_timeout(n): + yield + def not_really_random(): """A random-ish generator, which also generates nice patterns from time to time. @@ -199,9 +249,8 @@ class BaseTestRDict(BaseRtypingTest): def test_dict_copy(self): def func(): - # XXX this does not work if we use chars, only! dic = self.newdict() - dic['ab'] = 1 + dic['a'] = 1 dic['b'] = 2 d2 = dic.copy() ok = 1 @@ -1004,28 +1053,6 @@ class TestRDict(BaseTestRDict): assert r_AB_dic.lowleveltype == r_BA_dic.lowleveltype - def test_dict_resize(self): - py.test.skip("test written for non-ordered dicts, update or kill") - # XXX we no longer automatically resize on 'del'. We need to - # hack a bit in this test to trigger a resize by continuing to - # fill the dict's table while keeping the actual size very low - # in order to force a resize to shrink the table back - def func(want_empty): - d = self.newdict() - for i in range(rdict.DICT_INITSIZE << 1): - d[chr(ord('a') + i)] = i - if want_empty: - for i in range(rdict.DICT_INITSIZE << 1): - del d[chr(ord('a') + i)] - for i in range(rdict.DICT_INITSIZE << 3): - d[chr(ord('A') - i)] = i - del d[chr(ord('A') - i)] - return d - res = self.interpret(func, [0]) - assert len(res.entries) > rdict.DICT_INITSIZE - res = self.interpret(func, [1]) - assert len(res.entries) == rdict.DICT_INITSIZE - def test_opt_dummykeymarker(self): def f(): d = {"hello": None} @@ -1117,183 +1144,111 @@ class TestRDict(BaseTestRDict): DICT = lltype.typeOf(llres.item1) assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value'] - def test_deleted_entry_reusage_with_colliding_hashes(self): - py.test.skip("test written for non-ordered dicts, update or kill") - def lowlevelhash(value): - p = rstr.mallocstr(len(value)) - for i in range(len(value)): - p.chars[i] = value[i] - return rstr.LLHelpers.ll_strhash(p) - - def func(c1, c2): - c1 = chr(c1) - c2 = chr(c2) - d = self.newdict() - d[c1] = 1 - d[c2] = 2 - del d[c1] - return d[c2] - - char_by_hash = {} - base = rdict.DICT_INITSIZE - for y in range(0, 256): - y = chr(y) - y_hash = lowlevelhash(y) % base - char_by_hash.setdefault(y_hash, []).append(y) - - x, y = char_by_hash[0][:2] # find a collision - - res = self.interpret(func, [ord(x), ord(y)]) - assert res == 2 - - def func2(c1, c2): - c1 = chr(c1) - c2 = chr(c2) - d = self.newdict() - d[c1] = 1 - d[c2] = 2 - del d[c1] - d[c1] = 3 - return d - - res = self.interpret(func2, [ord(x), ord(y)]) - for i in range(len(res.entries)): - assert not (res.entries.everused(i) and not res.entries.valid(i)) - def func3(c0, c1, c2, c3, c4, c5, c6, c7): - d = self.newdict() - c0 = chr(c0) ; d[c0] = 1; del d[c0] - c1 = chr(c1) ; d[c1] = 1; del d[c1] - c2 = chr(c2) ; d[c2] = 1; del d[c2] - c3 = chr(c3) ; d[c3] = 1; del d[c3] - c4 = chr(c4) ; d[c4] = 1; del d[c4] - c5 = chr(c5) ; d[c5] = 1; del d[c5] - c6 = chr(c6) ; d[c6] = 1; del d[c6] - c7 = chr(c7) ; d[c7] = 1; del d[c7] - return d - - if rdict.DICT_INITSIZE != 8: - py.test.skip("make dict tests more indepdent from initsize") - res = self.interpret(func3, [ord(char_by_hash[i][0]) - for i in range(rdict.DICT_INITSIZE)]) - count_frees = 0 - for i in range(len(res.entries)): - if not res.entries.everused(i): - count_frees += 1 - assert count_frees >= 3 - -class TestStress: - - def test_stress(self): - from rpython.annotator.dictdef import DictKey, DictValue - from rpython.annotator import model as annmodel - dictrepr = rdict.DictRepr(None, rint.signed_repr, rint.signed_repr, - DictKey(None, annmodel.SomeInteger()), - DictValue(None, annmodel.SomeInteger())) +class Action(object): + def __init__(self, method, args): + self.method = method + self.args = args + + def execute(self, space): + getattr(space, self.method)(*self.args) + + def __repr__(self): + return "space.%s(%s)" % (self.method, ', '.join(map(repr, self.args))) + +class PseudoRTyper: + cache_dummy_values = {} + +# XXX: None keys crash the test, but translation sort-of allows it +keytypes_s = [ + SomeString(), SomeInteger(), SomeChar(), + SomeUnicodeString(), SomeUnicodeCodePoint()] +st_keys = sampled_from(keytypes_s) +st_values = sampled_from(keytypes_s + [SomeString(can_be_None=True)]) + +class Space(object): + def __init__(self, s_key, s_value): + self.s_key = s_key + self.s_value = s_value + rtyper = PseudoRTyper() + r_key = s_key.rtyper_makerepr(rtyper) + r_value = s_value.rtyper_makerepr(rtyper) + dictrepr = rdict.DictRepr(rtyper, r_key, r_value, + DictKey(None, s_key), + DictValue(None, s_value)) dictrepr.setup() - l_dict = rdict.ll_newdict(dictrepr.DICT) - referencetable = [None] * 400 - referencelength = 0 - value = 0 - - def complete_check(): - for n, refvalue in zip(range(len(referencetable)), referencetable): - try: - gotvalue = rdict.ll_dict_getitem(l_dict, n) - except KeyError: - assert refvalue is None - else: - assert gotvalue == refvalue - - for x in not_really_random(): - n = int(x*100.0) # 0 <= x < 400 - op = repr(x)[-1] - if op <= '2' and referencetable[n] is not None: - rdict.ll_dict_delitem(l_dict, n) - referencetable[n] = None - referencelength -= 1 - elif op <= '6': - rdict.ll_dict_setitem(l_dict, n, value) - if referencetable[n] is None: - referencelength += 1 - referencetable[n] = value - value += 1 - else: - try: - gotvalue = rdict.ll_dict_getitem(l_dict, n) - except KeyError: - assert referencetable[n] is None - else: - assert gotvalue == referencetable[n] - if 1.38 <= x <= 1.39: - complete_check() - print 'current dict length:', referencelength - assert l_dict.num_items == referencelength - complete_check() - - def test_stress_2(self): - yield self.stress_combination, True, False - yield self.stress_combination, False, True - yield self.stress_combination, False, False - yield self.stress_combination, True, True - - def stress_combination(self, key_can_be_none, value_can_be_none): - from rpython.rtyper.lltypesystem.rstr import string_repr - from rpython.annotator.dictdef import DictKey, DictValue - from rpython.annotator import model as annmodel - - print - print "Testing combination with can_be_None: keys %s, values %s" % ( - key_can_be_none, value_can_be_none) - - class PseudoRTyper: - cache_dummy_values = {} - dictrepr = rdict.DictRepr(PseudoRTyper(), string_repr, string_repr, - DictKey(None, annmodel.SomeString(key_can_be_none)), - DictValue(None, annmodel.SomeString(value_can_be_none))) - dictrepr.setup() - print dictrepr.lowleveltype - for key, value in dictrepr.DICTENTRY._adtmeths.items(): - print ' %s = %s' % (key, value) - l_dict = rdict.ll_newdict(dictrepr.DICT) - referencetable = [None] * 400 - referencelength = 0 - values = not_really_random() - keytable = [string_repr.convert_const("foo%d" % n) - for n in range(len(referencetable))] - - def complete_check(): - for n, refvalue in zip(range(len(referencetable)), referencetable): - try: - gotvalue = rdict.ll_dict_getitem(l_dict, keytable[n]) - except KeyError: - assert refvalue is None - else: - assert gotvalue == refvalue - - for x in not_really_random(): - n = int(x*100.0) # 0 <= x < 400 - op = repr(x)[-1] - if op <= '2' and referencetable[n] is not None: - rdict.ll_dict_delitem(l_dict, keytable[n]) - referencetable[n] = None - referencelength -= 1 - elif op <= '6': - ll_value = string_repr.convert_const(str(values.next())) - rdict.ll_dict_setitem(l_dict, keytable[n], ll_value) - if referencetable[n] is None: - referencelength += 1 - referencetable[n] = ll_value - else: - try: - gotvalue = rdict.ll_dict_getitem(l_dict, keytable[n]) - except KeyError: - assert referencetable[n] is None - else: - assert gotvalue == referencetable[n] - if 1.38 <= x <= 1.39: - complete_check() - print 'current dict length:', referencelength - assert l_dict.num_items == referencelength - complete_check() - + self.l_dict = rdict.ll_newdict(dictrepr.DICT) + self.reference = {} + self.ll_key = r_key.convert_const + self.ll_value = r_value.convert_const + + def setitem(self, key, value): + ll_key = self.ll_key(key) + ll_value = self.ll_value(value) + rdict.ll_dict_setitem(self.l_dict, ll_key, ll_value) + self.reference[key] = value + assert rdict.ll_contains(self.l_dict, ll_key) + + def delitem(self, key): + ll_key = self.ll_key(key) + rdict.ll_dict_delitem(self.l_dict, ll_key) + del self.reference[key] + assert not rdict.ll_contains(self.l_dict, ll_key) + + def copydict(self): + self.l_dict = rdict.ll_copy(self.l_dict) + + def cleardict(self): + rdict.ll_clear(self.l_dict) + self.reference.clear() + assert rdict.ll_dict_len(self.l_dict) == 0 + + def fullcheck(self): + assert rdict.ll_dict_len(self.l_dict) == len(self.reference) + for key, value in self.reference.iteritems(): + assert (rdict.ll_dict_getitem(self.l_dict, self.ll_key(key)) == + self.ll_value(value)) + +class StressTest(GenericStateMachine): + def __init__(self): + self.space = None + + def st_setitem(self): + return builds(Action, + just('setitem'), tuples(self.st_keys, self.st_values)) + + def st_updateitem(self): + return builds(Action, + just('setitem'), + tuples(sampled_from(self.space.reference), self.st_values)) + + def st_delitem(self): + return builds(Action, + just('delitem'), tuples(sampled_from(self.space.reference))) + + def steps(self): + if not self.space: + return builds(Action, just('setup'), tuples(st_keys, st_values)) + global_actions = [Action('copydict', ()), Action('cleardict', ())] + if self.space.reference: + return ( + self.st_setitem() | sampled_from(global_actions) | + self.st_updateitem() | self.st_delitem()) + else: + return (self.st_setitem() | sampled_from(global_actions)) + + def execute_step(self, action): + if action.method == 'setup': + self.space = Space(*action.args) + self.st_keys = ann2strategy(self.space.s_key) + self.st_values = ann2strategy(self.space.s_value) + return + with signal_timeout(1): # catches infinite loops + action.execute(self.space) + + def teardown(self): + if self.space: + self.space.fullcheck() + +def test_hypothesis(): + run_state_machine_as_test(StressTest, settings(max_examples=500, stateful_step_count=100)) diff --git a/rpython/rtyper/test/test_rfloat.py b/rpython/rtyper/test/test_rfloat.py index e563d797c4..c87f3a4599 100644 --- a/rpython/rtyper/test/test_rfloat.py +++ b/rpython/rtyper/test/test_rfloat.py @@ -204,6 +204,14 @@ class TestRfloat(BaseRtypingTest): res = self.ll_to_string(self.interpret(f, [10/3.0])) assert res == '3.33' + def test_formatd_g(self): + from rpython.rlib import rfloat + for flags in [0, rfloat.DTSF_ADD_DOT_0]: + def f(y): + return rfloat.formatd(y, 'g', 2, flags) + + assert self.ll_to_string(self.interpret(f, [3.0])) == f(3.0) + def test_formatd_repr(self): from rpython.rlib.rfloat import formatd def f(x): diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py index 0c80338eb5..0123ca3ef8 100644 --- a/rpython/rtyper/test/test_rpbc.py +++ b/rpython/rtyper/test/test_rpbc.py @@ -1,7 +1,7 @@ import py from rpython.annotator import model as annmodel -from rpython.annotator import policy, specialize +from rpython.annotator import specialize from rpython.rtyper.lltypesystem.lltype import typeOf from rpython.rtyper.test.tool import BaseRtypingTest from rpython.rtyper.llannotation import SomePtr, lltype_to_annotation @@ -1690,59 +1690,6 @@ class TestRPBC(BaseRtypingTest): # ____________________________________________________________ -class TestRPBCExtra(BaseRtypingTest): - - def test_folding_specialize_support(self): - - class S(object): - - def w(s, x): - if isinstance(x, int): - return x - if isinstance(x, str): - return len(x) - return -1 - w._annspecialcase_ = "specialize:w" - - def _freeze_(self): - return True - - s = S() - - def f(i, n): - w = s.w - if i == 0: - return w(0) - elif i == 1: - return w("abc") - elif i == 2: - return w(3*n) - elif i == 3: - return w(str(n)) - return -1 - - class P(policy.AnnotatorPolicy): - def specialize__w(pol, funcdesc, args_s): - typ = args_s[1].knowntype - if args_s[0].is_constant() and args_s[1].is_constant(): - x = args_s[1].const - v = s.w(x) - builder = specialize.make_constgraphbuilder(2, v) - return funcdesc.cachedgraph(x, builder=builder) - return funcdesc.cachedgraph(typ) - - p = P() - - res = self.interpret(f, [0, 66], policy=p) - assert res == 0 - res = self.interpret(f, [1, 66], policy=p) - assert res == 3 - res = self.interpret(f, [2, 4], policy=p) - assert res == 12 - res = self.interpret(f, [3, 5555], policy=p) - assert res == 4 - - def test_hlinvoke_simple(): def f(a,b): return a + b @@ -1998,7 +1945,7 @@ class TestSmallFuncSets(TestRPBC): def interpret(self, fn, args, **kwds): kwds['config'] = self.config - return TestRPBC.interpret(self, fn, args, **kwds) + return TestRPBC.interpret(fn, args, **kwds) def test_smallfuncsets_basic(): from rpython.translator.translator import TranslationContext, graphof diff --git a/rpython/rtyper/test/tool.py b/rpython/rtyper/test/tool.py index d565f1ecc7..079abaa9c7 100644 --- a/rpython/rtyper/test/tool.py +++ b/rpython/rtyper/test/tool.py @@ -5,22 +5,27 @@ from rpython.rtyper.test.test_llinterp import gengraph, interpret, interpret_rai class BaseRtypingTest(object): FLOAT_PRECISION = 8 - def gengraph(self, func, argtypes=[], viewbefore='auto', policy=None, + @staticmethod + def gengraph(func, argtypes=[], viewbefore='auto', policy=None, backendopt=False, config=None): return gengraph(func, argtypes, viewbefore, policy, backendopt=backendopt, config=config) - def interpret(self, fn, args, **kwds): + @staticmethod + def interpret(fn, args, **kwds): return interpret(fn, args, **kwds) - def interpret_raises(self, exc, fn, args, **kwds): + @staticmethod + def interpret_raises(exc, fn, args, **kwds): return interpret_raises(exc, fn, args, **kwds) - def float_eq(self, x, y): + @staticmethod + def float_eq(x, y): return x == y - def float_eq_approx(self, x, y): - maxError = 10**-self.FLOAT_PRECISION + @classmethod + def float_eq_approx(cls, x, y): + maxError = 10**-cls.FLOAT_PRECISION if abs(x-y) < maxError: return True @@ -31,45 +36,66 @@ class BaseRtypingTest(object): return relativeError < maxError - def is_of_type(self, x, type_): + @staticmethod + def is_of_type(x, type_): return type(x) is type_ - def _skip_llinterpreter(self, reason): + @staticmethod + def _skip_llinterpreter(reason): py.test.skip("lltypesystem doesn't support %s, yet" % reason) - def ll_to_string(self, s): + @staticmethod + def ll_to_string(s): if not s: return None return ''.join(s.chars) - def ll_to_unicode(self, s): + @staticmethod + def ll_to_unicode(s): return u''.join(s.chars) - def string_to_ll(self, s): - from rpython.rtyper.module.support import LLSupport - return LLSupport.to_rstr(s) - - def unicode_to_ll(self, s): - from rpython.rtyper.module.support import LLSupport - return LLSupport.to_runicode(s) - - def ll_to_list(self, l): + @staticmethod + def string_to_ll(s): + from rpython.rtyper.lltypesystem.rstr import STR, mallocstr + if s is None: + return lltype.nullptr(STR) + p = mallocstr(len(s)) + for i in range(len(s)): + p.chars[i] = s[i] + return p + + @staticmethod + def unicode_to_ll(s): + from rpython.rtyper.lltypesystem.rstr import UNICODE, mallocunicode + if s is None: + return lltype.nullptr(UNICODE) + p = mallocunicode(len(s)) + for i in range(len(s)): + p.chars[i] = s[i] + return p + + @staticmethod + def ll_to_list(l): r = [] items = l.ll_items() for i in range(l.ll_length()): r.append(items[i]) return r - def ll_unpack_tuple(self, t, length): + @staticmethod + def ll_unpack_tuple(t, length): return tuple([getattr(t, 'item%d' % i) for i in range(length)]) - def get_callable(self, fnptr): + @staticmethod + def get_callable(fnptr): return fnptr._obj._callable - def class_name(self, value): + @staticmethod + def class_name(value): return ''.join(value.super.typeptr.name.chars) - def read_attr(self, value, attr_name): + @staticmethod + def read_attr(value, attr_name): value = value._obj while value is not None: attr = getattr(value, "inst_" + attr_name, None) @@ -79,6 +105,7 @@ class BaseRtypingTest(object): return attr raise AttributeError() - def is_of_instance_type(self, val): + @staticmethod + def is_of_instance_type(val): T = lltype.typeOf(val) return isinstance(T, lltype.Ptr) and isinstance(T.TO, lltype.GcStruct) diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py index 0366c88454..eade70e2bd 100755 --- a/rpython/rtyper/tool/rffi_platform.py +++ b/rpython/rtyper/tool/rffi_platform.py @@ -263,10 +263,11 @@ class Struct(CConfigEntry): """An entry in a CConfig class that stands for an externally defined structure. """ - def __init__(self, name, interesting_fields, ifdef=None): + def __init__(self, name, interesting_fields, ifdef=None, adtmeths={}): self.name = name self.interesting_fields = interesting_fields self.ifdef = ifdef + self.adtmeths = adtmeths def prepare_code(self): if self.ifdef is not None: @@ -313,7 +314,9 @@ class Struct(CConfigEntry): offset = info['fldofs ' + fieldname] size = info['fldsize ' + fieldname] sign = info.get('fldunsigned ' + fieldname, False) - if (size, sign) != rffi.size_and_sign(fieldtype): + if is_array_nolength(fieldtype): + pass # ignore size and sign + elif (size, sign) != rffi.size_and_sign(fieldtype): fieldtype = fixup_ctype(fieldtype, fieldname, (size, sign)) layout_addfield(layout, offset, fieldtype, fieldname) @@ -353,7 +356,7 @@ class Struct(CConfigEntry): name = name[7:] else: hints['typedef'] = True - kwds = {'hints': hints} + kwds = {'hints': hints, 'adtmeths': self.adtmeths} return rffi.CStruct(name, *fields, **kwds) class SimpleType(CConfigEntry): @@ -682,8 +685,14 @@ class Field(object): def __repr__(self): return '<field %s: %s>' % (self.name, self.ctype) +def is_array_nolength(TYPE): + return isinstance(TYPE, lltype.Array) and TYPE._hints.get('nolength', False) + def layout_addfield(layout, offset, ctype, prefix): - size = _sizeof(ctype) + if is_array_nolength(ctype): + size = len(layout) - offset # all the rest of the struct + else: + size = _sizeof(ctype) name = prefix i = 0 while name in layout: diff --git a/rpython/rtyper/tool/test/test_rffi_platform.py b/rpython/rtyper/tool/test/test_rffi_platform.py index 3945167464..bf7db60286 100644 --- a/rpython/rtyper/tool/test/test_rffi_platform.py +++ b/rpython/rtyper/tool/test/test_rffi_platform.py @@ -270,6 +270,19 @@ def test_array(): [("d_name", lltype.FixedSizeArray(rffi.CHAR, 1))]) assert dirent.c_d_name.length == 32 +def test_array_varsized_struct(): + dirent = rffi_platform.getstruct("struct dirent", + """ + struct dirent /* for this example only, not the exact dirent */ + { + int d_off; + char d_name[1]; + }; + """, + [("d_name", rffi.CArray(rffi.CHAR))]) + assert rffi.offsetof(dirent, 'c_d_name') == 4 + assert dirent.c_d_name == rffi.CArray(rffi.CHAR) + def test_has_0001(): assert rffi_platform.has("x", "int x = 3;") assert not rffi_platform.has("x", "") diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py index dc13fcdc10..1015118990 100644 --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -842,6 +842,12 @@ class FunctionCodeGenerator(object): def OP_JIT_FFI_SAVE_RESULT(self, op): return '/* JIT_FFI_SAVE_RESULT %s */' % op + def OP_JIT_ENTER_PORTAL_FRAME(self, op): + return '/* JIT_ENTER_PORTAL_FRAME %s */' % op + + def OP_JIT_LEAVE_PORTAL_FRAME(self, op): + return '/* JIT_LEAVE_PORTAL_FRAME %s */' % op + def OP_GET_GROUP_MEMBER(self, op): typename = self.db.gettype(op.result.concretetype) return '%s = (%s)_OP_GET_GROUP_MEMBER(%s, %s);' % ( diff --git a/rpython/translator/c/node.py b/rpython/translator/c/node.py index c8bd2a2f05..dd604b86e3 100644 --- a/rpython/translator/c/node.py +++ b/rpython/translator/c/node.py @@ -546,7 +546,7 @@ class StructNode(ContainerNode): if needs_gcheader(T): gct = self.db.gctransformer if gct is not None: - self.gc_init = gct.gcheader_initdata(self) + self.gc_init = gct.gcheader_initdata(self.obj) db.getcontainernode(self.gc_init) else: self.gc_init = None @@ -677,7 +677,7 @@ class ArrayNode(ContainerNode): if needs_gcheader(T): gct = self.db.gctransformer if gct is not None: - self.gc_init = gct.gcheader_initdata(self) + self.gc_init = gct.gcheader_initdata(self.obj) db.getcontainernode(self.gc_init) else: self.gc_init = None @@ -913,6 +913,7 @@ class ExternalFuncNode(FuncNodeBase): return [] def new_funcnode(db, T, obj, forcename=None): + from rpython.rtyper.rtyper import llinterp_backend if db.sandbox: if (getattr(obj, 'external', None) is not None and not obj._safe_not_sandboxed): @@ -934,6 +935,9 @@ def new_funcnode(db, T, obj, forcename=None): return ExternalFuncNode(db, T, obj, name) elif hasattr(obj._callable, "c_name"): return ExternalFuncNode(db, T, obj, name) # this case should only be used for entrypoints + elif db.translator.rtyper.backend is llinterp_backend: + # on llinterp, anything goes + return ExternalFuncNode(db, T, obj, name) else: raise ValueError("don't know how to generate code for %r" % (obj,)) diff --git a/rpython/translator/sandbox/test/test_sandbox.py b/rpython/translator/sandbox/test/test_sandbox.py index 58e70667d8..ed46da595f 100644 --- a/rpython/translator/sandbox/test/test_sandbox.py +++ b/rpython/translator/sandbox/test/test_sandbox.py @@ -292,6 +292,21 @@ def test_unsafe_mmap(): rescode = pipe.wait() assert rescode == 0 +def test_environ_items(): + def entry_point(argv): + print os.environ.items() + return 0 + + exe = compile(entry_point) + g, f = run_in_subprocess(exe) + expect(f, g, "ll_os.ll_os_envitems", (), []) + expect(f, g, "ll_os.ll_os_write", (1, "[]\n"), 3) + g.close() + tail = f.read() + f.close() + assert tail == "" + + class TestPrintedResults: def run(self, entry_point, args, expected): |