Skip to content

Commit

Permalink
Update to Python3 for CPython code
Browse files Browse the repository at this point in the history
  • Loading branch information
jjwilke authored and jjwilke committed Jun 4, 2020
1 parent fcc3984 commit 679f453
Show file tree
Hide file tree
Showing 7 changed files with 112 additions and 48 deletions.
1 change: 1 addition & 0 deletions Makefile.includes
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir) -I$(top_srcdir)/include -I$(top_srcdir)/sprockit -I$(top_builddir)/sprockit
if INTEGRATED_SST_CORE
AM_CPPFLAGS += $(SST_CPPFLAGS)
AM_CPPFLAGS += $(PY_CPPFLAGS)
endif
AM_CPPFLAGS += $(PTH_CPPFLAGS)
AM_CPPFLAGS += $(UCONTEXT_CPPFLAGS)
39 changes: 36 additions & 3 deletions acinclude/check_python.m4
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,23 @@ AC_ARG_WITH([python-config],
]
)
if test "$have_integrated_core" = "yes"; then
core_pycfg=`$SST/bin/sst-config --PYTHON_CONFIG`
if test -n "$pycfg"; then
#if given python config, make sure consistent with sst-core
if test "$pycfg" != "$core_pycfg"; then
AC_MSG_ERROR([--with-python-config given $pycfg, which is different from sst-core $core_pycfg. Omit --with-python-config or ensure they match.])
fi
fi
pycfg=$core_pycfg
fi
# We now prioritize python3
if test -z "$pycfg"; then
if test -z "$pyprefix"; then
AC_PATH_PROGS([pycfg], ["python-config" "python3-config" "python2-config"], ["notfound"])
AC_PATH_PROGS([pycfg], ["python3-config" "python-config" "python2-config"], ["notfound"])
else
AC_PATH_PROGS([pycfg], ["python-config" "python3-config" "python2-config"], ["notfound"], [$pyprefix/bin])
AC_PATH_PROGS([pycfg], ["python3-config" "python-config" "python2-config"], ["notfound"], [$pyprefix/bin])
fi
fi
Expand All @@ -44,10 +56,31 @@ AC_ARG_WITH([python-config],
AC_MSG_CHECKING([for valid Python])
AC_MSG_RESULT([$pyprefix])
AC_PATH_PROGS([pyexe], ["python" "python3" "python2"], ["notfound"], [$pyprefix/bin])
if [[[ $pycfg == *python3* ]]]; then
py_tryfirst="python3"
py_trysecond="python"
else
py_tryfirst="python"
py_trysecond="python3"
fi
AC_PATH_PROGS([pyexe], ["$py_tryfirst" "$py_trysecond" "python2"], ["notfound"], [$pyprefix/bin])
if test "$pyexe" = "notfound"; then
AC_MSG_ERROR([No python/python2/python3 exe found in $pyprefix/bin])
fi
AC_SUBST([pyexe])
if test "$have_integrated_core" = "yes"; then
PY_INCLUDES="`$pycfg --includes`"
PY_LDFLAGS=`$pyexe $srcdir/bin/config_tools/get_py_ldflags`
PY_CPPFLAGS="$PY_INCLUDES"
SAVE_CPPFLAGS="$CPPFLAGS"
CPPFLAGS="$CPPFLAGS $PY_CPPFLAGS"
AC_CHECK_HEADERS([Python.h], [],
[AC_MSG_ERROR([Could not locate Python installation needed by SST core])])
CPPFLAGS="$SAVE_CPPFLAGS"
fi
AC_SUBST(PY_CPPFLAGS)
AC_SUBST(PY_LDFLAGS)
])
8 changes: 2 additions & 6 deletions acinclude/check_sst_core.m4
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,10 @@ if test "X$have_integrated_core" = "Xyes"; then
SST_INCLUDES="-I$SST/include -I$SST/include/sst -I$SST/include/sst/core"
SST_CPPFLAGS="-DSSTMAC_INTEGRATED_SST_CORE=1 $SST_INCLUDES -D__STDC_FORMAT_MACROS"
SAVE_CPPFLAGS="$CPPFLAGS"
PY_INCLUDES="`$pycfg --includes`"
PY_LDFLAGS=`$pyexe $srcdir/bin/config_tools/get_py_ldflags`
SST_CPPFLAGS="$SST_CPPFLAGS $PY_INCLUDES"
SST_CPPFLAGS="$SST_CPPFLAGS"
CPPFLAGS="$CPPFLAGS $SST_CPPFLAGS"
SST_LDFLAGS="$PY_LDFLAGS"
SST_LDFLAGS=""
# We have to use CXXFLAGS from sst-config script
SAVE_CXXFLAGS="$CXXFLAGS"
Expand All @@ -48,8 +46,6 @@ if test "X$have_integrated_core" = "Xyes"; then
# AC_MSG_ERROR([C Compiler $CC doesn't match SST core compiler $SST_CORE_CC])
#fi
AC_CHECK_HEADERS([Python.h], [],
[AC_MSG_ERROR([Could not locate Python installation needed by SST core])])
AC_CHECK_HEADERS([sst/core/component.h], [],
[AC_MSG_ERROR([Could not locate SST core header files at $SST])])
Expand Down
4 changes: 2 additions & 2 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,12 @@ else
AM_CONDITIONAL([HAVE_UBUNTU], [false])
fi

CHECK_SST_CORE()

CHECK_PYTHON()

CHECK_DLOPEN()

CHECK_SST_CORE()

CHECK_SST_ELEMENTS()

CHECK_CLANG()
Expand Down
8 changes: 7 additions & 1 deletion sstmac/sst_core/integrated_core.cc
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,13 @@ py_extract_params(PyObject* dict, sprockit::SimParameters::ptr params)
PyObject* val = PySequence_GetItem(pair, 1);
PyObject* key_str_obj = PyObject_Str(key);
const char* key_c_str = PyString_AsString(key_str_obj);
if (PyMapping_Check(val)){
bool isMapping = PyMapping_Check(val);
#if PY_MAJOR_VERSION >= 3
bool isString = PyUnicode_Check(val);
#else
bool isString = false; //ignore, not needed
#endif
if (isMapping && !isString){
sprockit::SimParameters::ptr sub_params = params->getOptionalNamespace(key_c_str);
sstmac::py_extract_params(val, sub_params);
} else {
Expand Down
84 changes: 56 additions & 28 deletions sstmac/sst_core/python_topology.cc
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,33 @@ Questions? Contact [email protected]
#include <unordered_map>
#include <algorithm>

#if PY_MAJOR_VERSION >= 3
#ifndef Py_TYPE
#endif
#define PY_OBJ_HEAD PyVarObject_HEAD_INIT(nullptr, 0)
#define ConvertToPythonLong(x) PyLong_FromLong(x)
#define ConvertToCppLong(x) PyLong_AsLong(x)
#define TP_FINALIZE
#define TP_VECTORCALL_OFFSET 0,
#define TP_PRINT
#define TP_COMPARE
#define TP_AS_SYNC nullptr,
#else
#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type)
#define PY_OBJ_HEAD PyVarObject_HEAD_INIT(nullptr, 0)
#define ConvertToPythonLong(x) PyInt_FromLong(x)
#define ConvertToCppLong(x) PyInt_AsLong(x)
#define TP_FINALIZE nullptr,
#define TP_VECTORCALL_OFFSET
#define TP_PRINT nullptr,
#define TP_COMPARE nullptr,
#define TP_AS_SYNC
#endif


using sstmac::SwitchId;
using sstmac::NodeId;

extern "C" {

typedef struct {
Expand Down Expand Up @@ -119,20 +146,17 @@ static PyMethodDef system_methods[] = {


static PyTypeObject SystemType = {
#if PY_MAJOR_VERSION >= 3
PyVarObject_HEAD_INIT(nullptr, 0)
#else
PyObject_HEAD_INIT(nullptr)
0, /* ob_size */
#endif
PY_OBJ_HEAD
"sst.macro.Topology", /* tp_name */
sizeof(SystemPy_t), /* tp_basicsize */
0, /* tp_itemsize */
(destructor)sys_dealloc, /* tp_dealloc */
nullptr, /* tp_print */
TP_VECTORCALL_OFFSET /* Python3 only */
TP_PRINT /* Python2 only */
nullptr, /* tp_getattr */
nullptr, /* tp_setattr */
nullptr, /* tp_compare */
TP_COMPARE /* Python2 only */
TP_AS_SYNC /* Python3 only */
nullptr, /* tp_repr */
nullptr, /* tp_as_number */
nullptr, /* tp_as_sequence */
Expand Down Expand Up @@ -162,6 +186,7 @@ static PyTypeObject SystemType = {
(initproc)sys_init, /* tp_init */
nullptr, /* tp_alloc */
nullptr, /* tp_new */
TP_FINALIZE
};

namespace sstmac {
Expand Down Expand Up @@ -196,11 +221,11 @@ sys_convert_to_list(const std::vector<sstmac::hw::Topology::InjectionPort>& port
for (int i=0; i < ports.size(); ++i){
const sstmac::hw::Topology::InjectionPort& port = ports[i];
PyObject* portTuple = PyTuple_New(3);
PyObject* nodeIdx = PyInt_FromLong(port.nid);
PyObject* nodeIdx = ConvertToPythonLong(port.nid);
PyTuple_SetItem(portTuple, 0, nodeIdx);
PyObject* swPort = PyInt_FromLong(port.switch_port);
PyObject* swPort = ConvertToPythonLong(port.switch_port);
PyTuple_SetItem(portTuple, 1, swPort);
PyObject* epPort = PyInt_FromLong(port.ep_port);
PyObject* epPort = ConvertToPythonLong(port.ep_port);
PyTuple_SetItem(portTuple, 2, epPort);
PyTuple_SetItem(tuple, i, portTuple);
}
Expand All @@ -210,7 +235,8 @@ sys_convert_to_list(const std::vector<sstmac::hw::Topology::InjectionPort>& port
static PyObject*
sys_get_injection_connections(SystemPy_t* self, PyObject* swIdx)
{
int sid = PyInt_AsLong(swIdx);
/** downcast, index guaranteed to fit 32-bit */
SwitchId sid = (SwitchId) ConvertToCppLong(swIdx);
std::vector<sstmac::hw::Topology::InjectionPort> ports;
self->macro_topology->endpointsConnectedToInjectionSwitch(sid, ports);
return sys_convert_to_list(ports);
Expand All @@ -219,20 +245,21 @@ sys_get_injection_connections(SystemPy_t* self, PyObject* swIdx)
static PyObject*
sys_get_switch_connections(SystemPy_t* self, PyObject* idx)
{
int swIdx = PyInt_AsLong(idx);
/** downcast, index guaranteed to fit 32-bit */
SwitchId swIdx = (SwitchId) ConvertToCppLong(idx);
std::vector<sstmac::hw::Topology::Connection> conns;
self->macro_topology->connectedOutports(swIdx, conns);
PyObject* tuple = PyTuple_New(conns.size());
for (int i=0; i < conns.size(); ++i){
sstmac::hw::Topology::Connection& conn = conns[i];
PyObject* connTuple = PyTuple_New(4);
PyObject* srcIdx = PyInt_FromLong(conn.src);
PyObject* srcIdx = ConvertToPythonLong(conn.src);
PyTuple_SetItem(connTuple, 0, srcIdx);
PyObject* dstIdx = PyInt_FromLong(conn.dst);
PyObject* dstIdx = ConvertToPythonLong(conn.dst);
PyTuple_SetItem(connTuple, 1, dstIdx);
PyObject* srcOutport = PyInt_FromLong(conn.src_outport);
PyObject* srcOutport = ConvertToPythonLong(conn.src_outport);
PyTuple_SetItem(connTuple, 2, srcOutport);
PyObject* dstInport = PyInt_FromLong(conn.dst_inport);
PyObject* dstInport = ConvertToPythonLong(conn.dst_inport);
PyTuple_SetItem(connTuple, 3, dstInport);
PyTuple_SetItem(tuple, i, connTuple);
}
Expand All @@ -242,15 +269,17 @@ sys_get_switch_connections(SystemPy_t* self, PyObject* idx)
static PyObject*
sys_nodeToLogpSwitch(SystemPy_t* self, PyObject* idx)
{
int nid = PyInt_AsLong(idx);
int sid = self->macro_topology->nodeToLogpSwitch(nid);
return PyInt_FromLong(sid);
/** downcast, index guaranteed to fit 32-bit */
NodeId nid = (NodeId) ConvertToCppLong(idx);
SwitchId sid = self->macro_topology->nodeToLogpSwitch(nid);
return ConvertToPythonLong(sid);
}

static PyObject*
sys_get_ejection_connections(SystemPy_t* self, PyObject* swIdx)
{
int sid = PyInt_AsLong(swIdx);
/** downcast, index guaranteed to fit 32-bit */
SwitchId sid = (SwitchId) ConvertToCppLong(swIdx);
std::vector<sstmac::hw::Topology::InjectionPort> ports;
self->macro_topology->endpointsConnectedToEjectionSwitch(sid, ports);
return sys_convert_to_list(ports);
Expand All @@ -276,7 +305,10 @@ sys_init(SystemPy_t* self, PyObject* args, PyObject* kwargs)
return -1;
}
}
sstmac::py_extract_params(params_dict, params);

if (params_dict){
sstmac::py_extract_params(params_dict, params);
}

if (PyMapping_Check(kwargs)){
sstmac::py_extract_params(kwargs, params);
Expand All @@ -298,10 +330,6 @@ sys_init(SystemPy_t* self, PyObject* args, PyObject* kwargs)
return 0;
}

#ifndef Py_TYPE
#define Py_TYPE(ob) (((PyObject*)(ob))->ob_type)
#endif

static void
sys_dealloc(SystemPy_t* self)
{
Expand All @@ -311,11 +339,11 @@ sys_dealloc(SystemPy_t* self)
static PyObject*
sys_num_switches(SystemPy_t* self, PyObject* /*null*/)
{
return PyInt_FromLong(self->macro_topology->numSwitches());
return ConvertToPythonLong(self->macro_topology->numSwitches());
}

static PyObject*
sys_num_nodes(SystemPy_t* self, PyObject* /*null*/)
{
return PyInt_FromLong(self->macro_topology->numNodes());
return ConvertToPythonLong(self->macro_topology->numNodes());
}
16 changes: 8 additions & 8 deletions sstmac/sst_core/sstmacro.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
smallLatency = "1ps"

def getParam(params, paramName, paramNS=None):
if not params.has_key(paramName):
if not paramName in params:
import sys
if paramNs:
sys.stderr.write("Missing parameter '%s' in namespace '%s'\n" % (paramName, paramNS))
Expand All @@ -16,7 +16,7 @@ def getParam(params, paramName, paramNS=None):
return params[paramName]

def getParamNamespace(params, ns, parentNs=None):
if not params.has_key(ns):
if not ns in params:
import sys
if parentNs:
sys.stderr.write("Missing parameter namespace '%s' in namespace '%s'\n" % (ns, parentNS))
Expand All @@ -29,7 +29,7 @@ def getNestedParamNamespace(params, *xargs):
nestedNs = ""
nextParams = params
for entry in xargs:
if not nextParams.has_key(entry):
if not entry in nextParams:
sys.stderr.write("Missing parameter namespace %s in params %s\n" % (entry, nestedNs))
raise Exception("failed configuring SST/macro")
nextParams = nextParams[entry]
Expand Down Expand Up @@ -143,7 +143,7 @@ def buildEndpoints(self, epFxn):
self.nodes[i] = epFxn(i)

def latency(self, params):
if params.has_key("latency"):
if "latency" in params:
return params["latency"]
else:
import sys
Expand Down Expand Up @@ -204,7 +204,7 @@ def buildLogPNetwork(self):
import re
nproc = sst.getMPIRankCount() * sst.getThreadCount()
switchParams = self.params["switch"]
if switchParams.has_key("logp"):
if "logp" in switchParams:
switchParams = switchParams["logp"]
lat = switchParams["out_in_latency"]
switches = []
Expand Down Expand Up @@ -281,7 +281,7 @@ def setupDeprecatedParams(params, debugList=[]):

for i in range(10):
ns = "app%d" % i
if params.has_key(ns):
if ns in params:
appParams = params[ns]
nodeParams[ns] = appParams
appName = appParams["name"]
Expand All @@ -305,7 +305,7 @@ def setupDeprecatedParams(params, debugList=[]):
for key in params:
val = params[key]
if isinstance(val, str):
if not nsParams.has_key(key):
if not key in nsParams:
nsParams[key] = val

ic = Interconnect(params)
Expand All @@ -317,7 +317,7 @@ def setupDeprecated():
sst.setProgramOption("timebase", "100as")
params = readCmdLineParams()
debugList = []
if params.has_key("debug"):
if "debug" in params:
debugList = params["debug"].strip().split()
for i in range(len(sys.argv)):
if sys.argv[i] == "-d" or sys.argv[i] == "--debug":
Expand Down

0 comments on commit 679f453

Please sign in to comment.