From 981c442b7e917b7b0417d4231fbad2c6ac03b17c Mon Sep 17 00:00:00 2001 From: george Date: Fri, 26 Dec 2025 03:45:54 +0000 Subject: [PATCH 001/214] Changed exponent from u32 to u64. Exponents over 2^32-1 don't work yet -- debugging is needed. --- src/Args.cpp | 8 ++-- src/Args.h | 6 +-- src/FFTConfig.cpp | 10 ++--- src/FFTConfig.h | 6 +-- src/Gpu.cpp | 93 ++++++++++++++++++++++++----------------------- src/Gpu.h | 18 ++++----- src/PRPState.cpp | 3 +- src/PRPState.h | 10 ++--- src/Primes.cpp | 16 ++++---- src/Primes.h | 12 +++--- src/Proof.cpp | 60 ++++++++++++++++-------------- src/Proof.h | 46 +++++++++++------------ src/Saver.cpp | 45 ++++++++++++----------- src/Saver.h | 14 +++---- src/Task.cpp | 12 +++--- src/Task.h | 2 +- src/TuneEntry.cpp | 13 ++++--- src/Worktodo.cpp | 4 +- src/common.h | 6 +-- src/shared.h | 4 +- src/state.cpp | 4 +- src/state.h | 12 +++--- src/tune.cpp | 56 ++++++++++++++-------------- 23 files changed, 236 insertions(+), 224 deletions(-) diff --git a/src/Args.cpp b/src/Args.cpp index 041202ad..5556e6a5 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -86,7 +86,7 @@ void Args::readConfig(const fs::path& path) { } } -u32 Args::getProofPow(u32 exponent) const { +u32 Args::getProofPow(u64 exponent) const { if (proofPow == -1) { return ProofSet::bestPower(exponent); } assert(proofPow >= 1); return proofPow; @@ -310,9 +310,9 @@ void Args::parse(const string& line) { } else if (key == "-tune") { doTune = true; if (!s.empty()) { tune = s; } - } else if (key == "-ctune") { - doCtune = true; - if (!s.empty()) { ctune.push_back(s); } +// } else if (key == "-ctune") { +// doCtune = true; +// if (!s.empty()) { ctune.push_back(s); } } else if (key == "-ztune") { doZtune = true; } else if (key == "-carryTune") { diff --git a/src/Args.h b/src/Args.h index 795cd99c..d7273afe 100644 --- a/src/Args.h +++ b/src/Args.h @@ -30,7 +30,7 @@ class Args { bool uses(const std::string& key) const { return flags.find(key) != flags.end(); } int value(const std::string& key, int valNotFound = -1) const; void readConfig(const fs::path& path); - u32 getProofPow(u32 exponent) const; + u32 getProofPow(u64 exponent) const; string tailDir() const; bool hasFlag(const string& key) const; @@ -78,8 +78,8 @@ class Args { u32 logStep = 20000; string fftSpec; - u32 prpExp = 0; - u32 llExp = 0; + u64 prpExp = 0; + u64 llExp = 0; size_t maxAlloc = 0; diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index 2308a037..c21607e1 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -182,7 +182,7 @@ if (18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())) > 19.0) return 19.0; return 18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())); } -bool FFTShape::needsLargeCarry(u32 E) const { +bool FFTShape::needsLargeCarry(u64 E) const { return E / double(size()) > carry32BPW(); } @@ -271,12 +271,12 @@ float FFTConfig::maxBpw() const { return (carry == CARRY_32 && (shape.fft_type == FFT64 || shape.fft_type == FFT3231)) ? std::min(shape.carry32BPW(), b) : b; } -FFTConfig FFTConfig::bestFit(const Args& args, u32 E, const string& spec) { +FFTConfig FFTConfig::bestFit(const Args& args, u64 E, const string& spec) { // A FFT-spec was given, simply take the first FFT from the spec that can handle E if (!spec.empty()) { FFTConfig fft{spec}; if (fft.maxExp() * args.fftOverdrive < E) { - log("Warning: %s (max %" PRIu64 ") may be too small for %u\n", fft.spec().c_str(), fft.maxExp(), E); + log("Warning: %s (max %" PRIu64 ") may be too small for %" PRIu64 "\n", fft.spec().c_str(), fft.maxExp(), E); } return fft; } @@ -288,7 +288,7 @@ FFTConfig FFTConfig::bestFit(const Args& args, u32 E, const string& spec) { if (E <= e.fft.maxExp() * args.fftOverdrive) { return e.fft; } } - log("No FFTs found in tune.txt that can handle %u. Consider tuning with -tune\n", E); + log("No FFTs found in tune.txt that can handle %" PRIu64 ". Consider tuning with -tune\n", E); // Take the first FFT that can handle E for (const FFTShape& shape : FFTShape::allShapes()) { @@ -297,7 +297,7 @@ FFTConfig FFTConfig::bestFit(const Args& args, u32 E, const string& spec) { } } - log("No FFT found for %u\n", E); + log("No FFT found for %" PRIu64 "\n", E); throw "No FFT"; } diff --git a/src/FFTConfig.h b/src/FFTConfig.h index c873eb8c..a9ad052c 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -27,7 +27,7 @@ class FFTShape { public: static std::vector allShapes(u32 from=0, u32 to = -1); - static tuple getChainLengths(u32 fftSize, u32 exponent, u32 middle); + static tuple getChainLengths(u32 fftSize, u64 exponent, u32 middle); static vector multiSpec(const string& spec); @@ -51,7 +51,7 @@ class FFTShape { std::string spec() const { return (fft_type ? to_string(fft_type) + ':' : "") + numberK(width) + ':' + numberK(middle) + ':' + numberK(height); } float carry32BPW() const; - bool needsLargeCarry(u32 E) const; + bool needsLargeCarry(u64 E) const; bool isFavoredShape() const; }; @@ -73,7 +73,7 @@ enum CARRY_KIND {CARRY_32=0, CARRY_64=1, CARRY_AUTO=2}; struct FFTConfig { public: - static FFTConfig bestFit(const Args& args, u32 E, const std::string& spec); + static FFTConfig bestFit(const Args& args, u64 E, const std::string& spec); // Which FP and NTT primes are involved in the FFT bool FFT_FP64; diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 44656fa1..433ce0a7 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -43,43 +43,43 @@ namespace { u32 kAt(u32 H, u32 line, u32 col) { return (line + col * H) * 2; } -double weight(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +double weight(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2l((long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -double invWeight(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +double invWeight(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2l(-(long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -double weightM1(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +double weightM1(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2l((long double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; } -double invWeightM1(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +double invWeightM1(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2l(- (long double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; } double boundUnderOne(double x) { return std::min(x, nexttoward(1, 0)); } -float weight32(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +float weight32(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -float invWeight32(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +float invWeight32(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2(-(double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -float weightM132(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +float weightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; } -float invWeightM132(u32 N, u32 E, u32 H, u32 line, u32 col, u32 rep) { +float invWeightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2(- (double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; } float boundUnderOne(float x) { return std::min(x, nexttowardf(1, 0)); } -Weights genWeights(FFTConfig fft, u32 E, u32 W, u32 H, u32 nW, bool AmdGpu) { +Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { u32 N = 2u * W * H; u32 groupWidth = W / nW; @@ -227,7 +227,7 @@ constexpr bool isInList(const string& s, initializer_list list) { return false; } -string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u32 E, bool doLog, +string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u64 E, bool doLog, bool &tail_single_wide, bool &tail_single_kernel, u32 &in_place, u32 &pad_size) { map config; @@ -457,15 +457,15 @@ RoeInfo roeStat(const vector& roe) { class IterationTimer { Timer timer; - u32 kStart; + u64 kStart; public: - explicit IterationTimer(u32 kStart) : kStart(kStart) { } + explicit IterationTimer(u64 kStart) : kStart(kStart) { } - float reset(u32 k) { + float reset(u64 k) { float secs = timer.reset(); - u32 its = max(1u, k - kStart); + u64 its = max(u64(1), k - kStart); kStart = k; return secs / its; } @@ -506,7 +506,7 @@ string toHex(const vector& v) { // -------- -unique_ptr Gpu::make(Queue* q, u32 E, GpuCommon shared, FFTConfig fftConfig, const vector& extraConf, bool logFftSize) { +unique_ptr Gpu::make(Queue* q, u64 E, GpuCommon shared, FFTConfig fftConfig, const vector& extraConf, bool logFftSize) { return make_unique(q, shared, fftConfig, E, extraConf, logFftSize); } @@ -518,7 +518,7 @@ Gpu::~Gpu() { #define ROE_SIZE 100000 #define CARRY_SIZE 100000 -Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& extraConf, bool logFftSize) : +Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : queue(q), background{shared.background}, args{*shared.args}, @@ -649,7 +649,7 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& // Sometimes we do want to run a FFT beyond a reasonable BPW (e.g. during -ztune), and these situations // coincide with logFftSize == false if (fft.maxExp() < E) { - log("Warning: %s (max %" PRIu64 ") may be too small for %u\n", fft.spec().c_str(), fft.maxExp(), E); + log("Warning: %s (max %" PRIu64 ") may be too small for %" PRIu64 "\n", fft.spec().c_str(), fft.maxExp(), E); } } @@ -992,7 +992,7 @@ void Gpu::modMul(Buffer& ioA, Buffer& inB, enum LEAD_TYPE leadInB, b mul(ioA, buf1, buf2, buf3, mul3); }; -void Gpu::writeState(u32 k, const vector& check, u32 blockSize) { +void Gpu::writeState(u64 k, const vector& check, u32 blockSize) { assert(blockSize > 0); writeIn(bufCheck, check); @@ -1022,7 +1022,7 @@ void Gpu::writeState(u32 k, const vector& check, u32 blockSize) { } modMul(bufData, bufAux, true); } - + bool Gpu::doCheck(u32 blockSize) { squareLoop(bufAux, bufCheck, 0, blockSize, true); modMul(bufCheck, bufData); @@ -1278,10 +1278,10 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu } } -u32 Gpu::squareLoop(Buffer& out, Buffer& in, u32 from, u32 to, bool doTailMul3) { +u32 Gpu::squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3) { assert(from < to); enum LEAD_TYPE leadIn = LEAD_NONE; - for (u32 k = from; k < to; ++k) { + for (u64 k = from; k < to; ++k) { enum LEAD_TYPE leadOut = useLongCarry || (k == to - 1) ? LEAD_NONE : LEAD_WIDTH; square(out, (k==from) ? in : out, leadIn, leadOut, doTailMul3 && (k == to - 1)); leadIn = leadOut; @@ -1350,16 +1350,16 @@ string RoeInfo::toString() const { return buf; } -static string makeLogStr(const string& status, u32 k, u64 res, float secsPerIt, u32 nIters) { +static string makeLogStr(const string& status, u64 k, u64 res, float secsPerIt, u64 nIters) { char buf[256]; - snprintf(buf, sizeof(buf), "%2s %9u %016" PRIx64 " %4.0f ETA %s; ", + snprintf(buf, sizeof(buf), "%2s %9" PRIu64 " %016" PRIx64 " %4.0f ETA %s; ", status.c_str(), k, res, /* k / float(nIters) * 100, */ secsPerIt * 1'000'000, getETA(k, nIters, secsPerIt).c_str()); return buf; } -void Gpu::doBigLog(u32 k, u64 res, bool checkOK, float secsPerIt, u32 nIters, u32 nErrors) { +void Gpu::doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors) { auto [roeSq, roeMul] = readROE(); double z = roeSq.z(); zAvg.update(z, roeSq.N); @@ -1480,7 +1480,7 @@ static u32 mod3(const std::vector &words) { return r % 3; } -static void doDiv3(u32 E, Words& words) { +static void doDiv3(u64 E, Words& words) { u32 r = (3 - mod3(words)) % 3; assert(r < 3); int topBits = E % 32; @@ -1497,7 +1497,7 @@ static void doDiv3(u32 E, Words& words) { } } -void Gpu::doDiv9(u32 E, Words& words) { +void Gpu::doDiv9(u64 E, Words& words) { doDiv3(E, words); doDiv3(E, words); } @@ -1532,12 +1532,12 @@ PRPState Gpu::loadPRP(Saver& saver) { u64 res = dataResidue(); if (res == state.res64) { - log("OK %9u on-load: blockSize %d, %016" PRIx64 "\n", state.k, state.blockSize, res); + log("OK %9" PRIu64 " on-load: blockSize %d, %016" PRIx64 "\n", state.k, state.blockSize, res); return state; // return {loaded.k, loaded.blockSize, loaded.nErrors}; } - log("EE %9u on-load: %016" PRIx64 " vs. %016" PRIx64 "\n", state.k, res, state.res64); + log("EE %9" PRIu64 " on-load: %016" PRIx64 " vs. %016" PRIx64 "\n", state.k, res, state.res64); if (!state.k) { break; } // We failed on PRP start } @@ -1545,7 +1545,7 @@ PRPState Gpu::loadPRP(Saver& saver) { throw "Error on load"; } -u32 Gpu::getProofPower(u32 k) { +u32 Gpu::getProofPower(u64 k) { u32 power = ProofSet::effectivePower(E, args.getProofPow(E), k); if (power != args.getProofPow(E)) { @@ -1785,7 +1785,8 @@ PRPResult Gpu::isPrimePRP(const Task& task) { reload: elapsedTimer.reset(); - u32 blockSize{}, k{}; + u32 blockSize{}; + u64 k{}; double elapsedBefore = 0; { @@ -1814,28 +1815,28 @@ PRPResult Gpu::isPrimePRP(const Task& task) { // For M=2^E-1, residue "type-3" == 3^(M+1), and residue "type-1" == type-3 / 9, // See http://www.mersenneforum.org/showpost.php?p=468378&postcount=209 // For both type-1 and type-3 we need to do E squarings (as M+1==2^E). - const u32 kEnd = E; + const u64 kEnd = E; assert(k < kEnd); // We continue beyound kEnd: to the next multiple of blockSize, to do a check there - u32 kEndEnd = roundUp(kEnd, blockSize); + u64 kEndEnd = roundUp(kEnd, blockSize); bool skipNextCheckUpdate = false; - u32 persistK = proofSet.next(k); + u64 persistK = proofSet.next(k); enum LEAD_TYPE leadIn = LEAD_NONE; assert(k % blockSize == 0); assert(checkStep % blockSize == 0); - const u32 startK = k; + const u64 startK = k; IterationTimer iterationTimer{k}; wantROE = 0; // skip the initial iterations while (true) { assert(k < kEndEnd); - + if (!wantROE && k - startK > 30) { wantROE = args.logROE ? ROE_SIZE : 2'000; } if (skipNextCheckUpdate) { @@ -1876,7 +1877,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { res2048.clear(); assert(words.size() >= 64); res2048.insert(res2048.end(), words.begin(), std::next(words.begin(), 64)); - log("%s %8d / %d, %s\n", isPrime ? "PP" : "CC", kEnd, E, hex(finalRes64).c_str()); + log("%s %8" PRIu64 " / %" PRIu64 ", %s\n", isPrime ? "PP" : "CC", kEnd, E, hex(finalRes64).c_str()); } if (!doCheck && !doLog) continue; @@ -1888,7 +1889,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { vector rawCheck = readChecked(bufCheck); if (rawCheck.empty()) { ++nErrors; - log("%9u %016" PRIx64 " read NULL check\n", k, res); + log("%9" PRIu64 " %016" PRIx64 " read NULL check\n", k, res); if (++nSeqErrors > 2) { throw "sequential errors"; } goto reload; } @@ -1899,7 +1900,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { elapsedBefore + elapsedTimer.at()}); }); - log(" %9u %016" PRIx64 " %4.0f\n", k, res, /*k / float(kEndEnd) * 100*,*/ secsPerIt * 1'000'000); + log(" %9" PRIu64 " %016" PRIx64 " %4.0f\n", k, res, /*k / float(kEndEnd) * 100*,*/ secsPerIt * 1'000'000); RoeInfo carryStats = readCarryStats(); if (carryStats.N) { u32 m = ldexp(carryStats.max, 32); @@ -1965,7 +1966,7 @@ LLResult Gpu::isPrimeLL(const Task& task) { reload: elapsedTimer.reset(); - u32 startK = 0; + u64 startK = 0; double elapsedBefore = 0; { LLState state = saver.load(); @@ -1977,13 +1978,13 @@ LLResult Gpu::isPrimeLL(const Task& task) { u64 res = dataResidue(); if (res != expectedRes) { throw "Invalid savefile (res64)"; } assert(res == expectedRes); - log("LL loaded @ %u : %016" PRIx64 "\n", startK, res); + log("LL loaded @ %" PRIu64 " : %016" PRIx64 "\n", startK, res); } IterationTimer iterationTimer{startK}; - u32 k = startK; - u32 kEnd = E - 2; + u64 k = startK; + u64 kEnd = E - 2; enum LEAD_TYPE leadIn = LEAD_NONE; while (true) { @@ -2009,7 +2010,7 @@ LLResult Gpu::isPrimeLL(const Task& task) { if (isAllZero) { if (k < kEnd) { - log("Error: early ZERO @ %u\n", k); + log("Error: early ZERO @ %" PRIu64 "\n", k); if (doStop) { throw "stop requested"; } else { @@ -2025,7 +2026,7 @@ LLResult Gpu::isPrimeLL(const Task& task) { float secsPerIt = iterationTimer.reset(k); queue->setSquareTime((int) (secsPerIt * 1'000'000)); - log("%9u %016" PRIx64 " %4.0f\n", k, res64, secsPerIt * 1'000'000); + log("%9" PRIu64 " %016" PRIx64 " %4.0f\n", k, res64, secsPerIt * 1'000'000); if (k >= kEnd) { return {isAllZero, res64}; } @@ -2039,13 +2040,13 @@ array Gpu::isCERT(const Task& task) { // Get CERT start value char fname[32]; - sprintf(fname, "M%u.cert", E); + sprintf(fname, "M%" PRIu64 ".cert", E); // Autoprimenet.py does not add the cert entry to worktodo.txt until it has successfully downloaded the .cert file. { // Enclosing this code in braces ensures the file will be closed by the File destructor. The later file deletion requires the file be closed in Windows. File fi = File::openReadThrow(fname); - u32 nBytes = (E - 1) / 8 + 1; + u32 nBytes = u32((E - 1) / 8 + 1); Words B = fi.readBytesLE(nBytes); writeIn(bufData, std::move(B)); } diff --git a/src/Gpu.h b/src/Gpu.h index fc5166f3..ad859eac 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -93,7 +93,7 @@ class Gpu { private: std::unique_ptr> saver; - u32 E; + u64 E; u32 N; FFTConfig fft; @@ -250,8 +250,8 @@ class Gpu { void squareCERT(Buffer& io, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut) { square(io, io, leadIn, leadOut, false, false); } void squareLL(Buffer& io, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut) { square(io, io, leadIn, leadOut, false, true); } - u32 squareLoop(Buffer& out, Buffer& in, u32 from, u32 to, bool doTailMul3); - u32 squareLoop(Buffer& io, u32 from, u32 to) { return squareLoop(io, io, from, to, false); } + u32 squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3); + u32 squareLoop(Buffer& io, u64 from, u64 to) { return squareLoop(io, io, from, to, false); } bool isEqual(Buffer& bufCheck, Buffer& bufAux); u64 bufResidue(Buffer& buf); @@ -260,7 +260,7 @@ class Gpu { void exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Buffer& buf2, Buffer& buf3); - void writeState(u32 k, const vector& check, u32 blockSize); + void writeState(u64 k, const vector& check, u32 blockSize); // does either carrryFused() or the expanded version depending on useLongCarry void doCarry(Buffer& out, Buffer& in, Buffer& tmp); @@ -283,13 +283,13 @@ class Gpu { // void measureTransferSpeed(); - static void doDiv9(u32 E, Words& words); + static void doDiv9(u64 E, Words& words); static bool equals9(const Words& words); void selftestTrig(); public: - Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u32 E, const vector& extraConf, bool logFftSize); - static unique_ptr make(Queue* q, u32 E, GpuCommon shared, FFTConfig fft, + Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize); + static unique_ptr make(Queue* q, u64 E, GpuCommon shared, FFTConfig fft, const vector& extraConf = {}, bool logFftSize = true); ~Gpu(); @@ -337,8 +337,8 @@ class Gpu { void clear(bool isPRP); private: - u32 getProofPower(u32 k); - void doBigLog(u32 k, u64 res, bool checkOK, float secsPerIt, u32 nIters, u32 nErrors); + u32 getProofPower(u64 k); + void doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors); }; // Compute the size of an FFT/NTT data buffer depending on the FFT/NTT float/prime. Size is returned in units of sizeof(double). diff --git a/src/PRPState.cpp b/src/PRPState.cpp index bdc2c9c8..02b9ae68 100644 --- a/src/PRPState.cpp +++ b/src/PRPState.cpp @@ -8,7 +8,8 @@ PRPState::PRPState(File&& fi) { string header = fi.readLine(); - u32 fileE, fileK, blockSize, nErrors, crc; + u64 fileE, fileK; + u32 blockSize, nErrors, crc; u64 res64; vector check; u32 b1, nBits, start, nextK; diff --git a/src/PRPState.h b/src/PRPState.h index edf0ccf9..1eab5085 100644 --- a/src/PRPState.h +++ b/src/PRPState.h @@ -10,23 +10,23 @@ class File; class PRPState { // E, k, block-size, res64, nErrors - static constexpr const char *PRP_v10 = "OWL PRP 10 %u %u %u %016" SCNx64 " %u\n"; + static constexpr const char *PRP_v10 = "OWL PRP 10 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u\n"; // Exponent, iteration, block-size, res64, nErrors // B1, nBits, start, nextK, crc - static constexpr const char *PRP_v11 = "OWL PRP 11 %u %u %u %016" SCNx64 " %u %u %u %u %u %u\n"; + static constexpr const char *PRP_v11 = "OWL PRP 11 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u %u %u %u %u %u\n"; // E, k, block-size, res64, nErrors, CRC - static constexpr const char *PRP_v12 = "OWL PRP 12 %u %u %u %016" SCNx64 " %u %u\n"; + static constexpr const char *PRP_v12 = "OWL PRP 12 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u %u\n"; public: - u32 k{}; + u64 k{}; u32 blockSize{}; u64 res64{}; vector check; u32 nErrors{}; - // PRPState(u32 k, u32 blockSize, u64 res64, vector<) + // PRPState(u64 k, u32 blockSize, u64 res64, vector<) PRPState(File&& f); void saveTo(const File& f); }; diff --git a/src/Primes.cpp b/src/Primes.cpp index 865cfaec..6e3ceb36 100644 --- a/src/Primes.cpp +++ b/src/Primes.cpp @@ -15,14 +15,14 @@ Primes::Primes() { } } -bool Primes::isPrimeOdd(u32 n) const { +bool Primes::isPrimeOdd(u64 n) const { assert(n % 2); // must be odd to call here if (n < 3) { return false; } for (u32 k = 0; k < sieve.size(); ++k) { if (sieve[k]) { u32 p = k * 2 + 3; - if (p * p > n) { return true; } + if (u64(p) * u64(p) > n) { return true; } if (n % p == 0) { return false; } } } @@ -30,11 +30,11 @@ bool Primes::isPrimeOdd(u32 n) const { return false; } -bool Primes::isPrime(u32 n) const { +bool Primes::isPrime(u64 n) const { return (n%2 && isPrimeOdd(n)) || (n == 2); } -u32 Primes::prevPrime(u32 n) const { +u64 Primes::prevPrime(u64 n) const { --n; if (n % 2 == 0) { --n; } @@ -43,7 +43,7 @@ u32 Primes::prevPrime(u32 n) const { return 0; } -u32 Primes::nextPrime(u32 n) const { +u64 Primes::nextPrime(u64 n) const { ++n; if (n % 2 == 0) { ++n; } for (; ; n += 2) { if (isPrimeOdd(n)) { return n; }} @@ -51,10 +51,10 @@ u32 Primes::nextPrime(u32 n) const { return 0; } -u32 Primes::nearestPrime(u32 n) const { +u64 Primes::nearestPrime(u64 n) const { if (isPrime(n)) { return n; } - u32 a = prevPrime(n); - u32 b = nextPrime(n); + u64 a = prevPrime(n); + u64 b = nextPrime(n); assert(a < n && n < b); return n-a < b-n ? a : b; } diff --git a/src/Primes.h b/src/Primes.h index b951c02d..7f1f16fb 100644 --- a/src/Primes.h +++ b/src/Primes.h @@ -6,14 +6,14 @@ #include "common.h" class Primes { - std::bitset<50000> sieve; - bool isPrimeOdd(u32 n) const; + std::bitset<50000> sieve; // Allows for testing prims up to 10 billion + bool isPrimeOdd(u64 n) const; public: Primes(); - bool isPrime(u32 n) const; - u32 prevPrime(u32 n) const; - u32 nextPrime(u32 n) const; - u32 nearestPrime(u32 n) const; + bool isPrime(u64 n) const; + u64 prevPrime(u64 n) const; + u64 nextPrime(u64 n) const; + u64 nearestPrime(u64 n) const; }; diff --git a/src/Proof.cpp b/src/Proof.cpp index 31c488f8..bd8d4be6 100644 --- a/src/Proof.cpp +++ b/src/Proof.cpp @@ -19,11 +19,11 @@ namespace proof { -array hashWords(u32 E, const Words& words) { +array hashWords(u64 E, const Words& words) { return std::move(SHA3{}.update(words.data(), (E-1)/8+1)).finish(); } -array hashWords(u32 E, array prefix, const Words& words) { +array hashWords(u64 E, array prefix, const Words& words) { return std::move(SHA3{}.update(prefix).update(words.data(), (E-1)/8+1)).finish(); } @@ -39,7 +39,8 @@ string fileHash(const fs::path& filePath) { ProofInfo getInfo(const fs::path& proofFile) { string hash = proof::fileHash(proofFile); File fi = File::openReadThrow(proofFile); - u32 E = 0, power = 0; + u64 E = 0; + u32 power = 0; char c = 0; if (fi.scanf(Proof::HEADER_v2, &power, &E, &c) != 3 || c != '\n') { log("Proof file '%s' has invalid header\n", proofFile.string().c_str()); @@ -68,7 +69,8 @@ void Proof::save(const fs::path& proofFile) const { Proof Proof::load(const fs::path& path) { File fi = File::openReadThrow(path); - u32 E = 0, power = 0; + u64 E = 0; + u32 power = 0; char c = 0; if (fi.scanf(HEADER_v2, &power, &E, &c) != 3 || c != '\n') { log("Proof file '%s' has invalid header\n", path.string().c_str()); @@ -84,7 +86,7 @@ Proof Proof::load(const fs::path& path) { bool Proof::verify(Gpu *gpu, const vector& hashes) const { // log("B %016" PRIx64 "\n", res64(B)); // for (u32 i = 0; i < middles.size(); ++i) { log("Middle[%u] %016" PRIx64 "\n", i, res64(middles[i])); } - + u32 power = middles.size(); assert(power > 0); @@ -92,10 +94,10 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { Words A{makeWords(E, 3)}; Words B{this->B}; - + auto hash = proof::hashWords(E, B); - u32 span = E; + u64 span = E; for (u32 i = 0; i < power; ++i, span = (span + 1) / 2) { const Words& M = middles[i]; hash = proof::hashWords(E, hash, M); @@ -113,12 +115,12 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { if (gpu->args.verbose) { log("proof [%u] : A %016" PRIx64 ", B %016" PRIx64 ", h %016" PRIx64 "\n", i, res64(A), res64(B), h); } } - log("proof verification: doing %d iterations\n", span); + log("proof verification: doing %" PRIu64 " iterations\n", span); A = gpu->expExp2(A, span); bool ok = (A == B); if (ok) { - log("proof: %u proved %s\n", E, isPrime ? "probable prime" : "composite"); + log("proof: %" PRIu64 " proved %s\n", E, isPrime ? "probable prime" : "composite"); } else { log("proof: invalid (%016" PRIx64 " expected %016" PRIx64 ")\n", res64(A), res64(B)); } @@ -127,9 +129,9 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { // ---- ProofSet ---- -ProofSet::ProofSet(u32 E, u32 power) +ProofSet::ProofSet(u64 E, u32 power) : E{E}, power{power} { - + assert(E & 1); // E is supposed to be prime if (power <= 0 || power > 12) { log("Invalid proof power: %u\n", power); @@ -138,11 +140,13 @@ ProofSet::ProofSet(u32 E, u32 power) fs::create_directories(proofPath(E)); - vector spans; - for (u32 span = (E + 1) / 2; spans.size() < power; span = (span + 1) / 2) { spans.push_back(span); } + vector spans; + for (u64 span = (E + 1) / 2; spans.size() < power; span = (span + 1) / 2) { spans.push_back(span); } points.push_back(0); - for (u32 p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { + u32 p; + u64 span; + for (p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { for (u32 i = 0, end = points.size(); i < end; ++i) { points.push_back(points[i] + span); } @@ -160,15 +164,17 @@ ProofSet::ProofSet(u32 E, u32 power) points.push_back(u32(-1)); // guard element cacheIt = points.begin(); - for ([[maybe_unused]] u32 p : points) { + for ([[maybe_unused]] u64 p : points) { assert(p > E || isInPoints(E, power, p)); } } -bool ProofSet::isInPoints(u32 E, u32 power, u32 k) { +bool ProofSet::isInPoints(u64 E, u32 power, u64 k) { if (k == E) { return true; } // special-case E - u32 start = 0; - for (u32 p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { + u64 start = 0; + u32 p; + u64 span; + for (p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { assert(k >= start); if (k > start + span) { start += span; @@ -179,12 +185,12 @@ bool ProofSet::isInPoints(u32 E, u32 power, u32 k) { return false; } -bool ProofSet::canDo(u32 E, u32 power, u32 currentK) { +bool ProofSet::canDo(u64 E, u32 power, u64 currentK) { assert(power > 0 && power <= 12); return ProofSet{E, power}.isValidTo(currentK); } -u32 ProofSet::bestPower(u32 E) { +u32 ProofSet::bestPower(u64 E) { // Best proof powers assuming no disk space concern. // We increment power by 1 for each fourfold increase of the exponent. // The values below produce power=10 at wavefront, and power=11 at 100Mdigits: @@ -197,7 +203,7 @@ u32 ProofSet::bestPower(u32 E) { return power; } -double ProofSet::diskUsageGB(u32 E, u32 power) { +double ProofSet::diskUsageGB(u64 E, u32 power) { // -3 because convert exponent bits to bytes // -30 because convert bytes to GB // +power because needs 2^power residues for proof generation @@ -205,7 +211,7 @@ double ProofSet::diskUsageGB(u32 E, u32 power) { return power ? ldexp(E, -33 + int(power)) * 1.05 : 0.0; } -u32 ProofSet::effectivePower(u32 E, u32 power, u32 currentK) { +u32 ProofSet::effectivePower(u64 E, u32 power, u64 currentK) { for (u32 p = power; p > 0; --p) { // log("validating proof residues for power %u\n", p); if (canDo(E, p, currentK)) { return p; } @@ -213,11 +219,11 @@ u32 ProofSet::effectivePower(u32 E, u32 power, u32 currentK) { return 0; } -bool ProofSet::fileExists(u32 k) const { +bool ProofSet::fileExists(u64 k) const { return File::size(proofPath(E) / to_string(k)) == i64(E / 32 + 2) * 4; } -bool ProofSet::isValidTo(u32 limitK) const { +bool ProofSet::isValidTo(u64 limitK) const { auto it = upper_bound(points.begin(), points.end(), limitK); if (it == points.begin()) { @@ -238,14 +244,14 @@ bool ProofSet::isValidTo(u32 limitK) const { return true; } -u32 ProofSet::next(u32 k) const { +u64 ProofSet::next(u64 k) const { if (*cacheIt <= k || (cacheIt > points.begin() && *prev(cacheIt) > k)) { cacheIt = upper_bound(points.begin(), points.end(), k); } return *cacheIt; } -void ProofSet::save(u32 E, u32 power, u32 k, const Words& words) { +void ProofSet::save(u64 E, u32 power, u64 k, const Words& words) { assert(k && k <= E); assert(isInPoints(E, power, k)); @@ -253,7 +259,7 @@ void ProofSet::save(u32 E, u32 power, u32 k, const Words& words) { assert(load(E, power, k) == words); } -Words ProofSet::load(u32 E, u32 power, u32 k) { +Words ProofSet::load(u64 E, u32 power, u64 k) { assert(k && k <= E); assert(isInPoints(E, power, k)); return File::openReadThrow(proofPath(E) / to_string(k)).readChecked(E/32 + 1); diff --git a/src/Proof.h b/src/Proof.h index 97e9056d..c83d8ace 100644 --- a/src/Proof.h +++ b/src/Proof.h @@ -4,6 +4,7 @@ #include "File.h" #include "common.h" +#include namespace fs = std::filesystem; @@ -11,15 +12,15 @@ class Gpu; struct ProofInfo { u32 power; - u32 exp; + u64 exp; string md5; }; namespace proof { -array hashWords(u32 E, const Words& words); +array hashWords(u64 E, const Words& words); -array hashWords(u32 E, array prefix, const Words& words); +array hashWords(u64 E, array prefix, const Words& words); string fileHash(const fs::path& filePath); @@ -29,7 +30,7 @@ ProofInfo getInfo(const fs::path& proofFile); class Proof { public: - const u32 E; + const u64 E; const Words B; const vector middles; @@ -40,7 +41,7 @@ class Proof { POWER=8\n NUMBER=M216091\n */ - static const constexpr char* HEADER_v2 = "PRP PROOF\nVERSION=2\nHASHSIZE=64\nPOWER=%u\nNUMBER=M%u%c"; + static const constexpr char* HEADER_v2 = "PRP PROOF\nVERSION=2\nHASHSIZE=64\nPOWER=%u\nNUMBER=M%" PRIu64 "%c"; static Proof load(const fs::path& path); @@ -53,38 +54,37 @@ class Proof { class ProofSet { public: - const u32 E; + const u64 E; const u32 power; private: - vector points; + vector points; - bool isValidTo(u32 limitK) const; + bool isValidTo(u64 limitK) const; - static bool canDo(u32 E, u32 power, u32 currentK); + static bool canDo(u64 E, u32 power, u64 currentK); mutable decltype(points)::const_iterator cacheIt{}; - bool fileExists(u32 k) const; + bool fileExists(u64 k) const; - static fs::path proofPath(u32 E) { return fs::path(to_string(E)) / "proof"; } + static fs::path proofPath(u64 E) { return fs::path(to_string(E)) / "proof"; } public: - static u32 bestPower(u32 E); - static u32 effectivePower(u32 E, u32 power, u32 currentK); - static double diskUsageGB(u32 E, u32 power); - static bool isInPoints(u32 E, u32 power, u32 k); + static u32 bestPower(u64 E); + static u32 effectivePower(u64 E, u32 power, u64 currentK); + static double diskUsageGB(u64 E, u32 power); + static bool isInPoints(u64 E, u32 power, u64 k); - ProofSet(u32 E, u32 power); - - u32 next(u32 k) const; + ProofSet(u64 E, u32 power); - static void save(u32 E, u32 power, u32 k, const Words& words); - static Words load(u32 E, u32 power, u32 k); - - void save(u32 k, const Words& words) const { return save(E, power, k, words); } - Words load(u32 k) const { return load(E, power, k); } + u64 next(u64 k) const; + static void save(u64 E, u32 power, u64 k, const Words& words); + static Words load(u64 E, u32 power, u64 k); + + void save(u64 k, const Words& words) const { return save(E, power, k, words); } + Words load(u64 k) const { return load(E, power, k); } std::pair> computeProof(Gpu *gpu) const; }; diff --git a/src/Saver.cpp b/src/Saver.cpp index 118d1eae..68caba41 100644 --- a/src/Saver.cpp +++ b/src/Saver.cpp @@ -15,19 +15,19 @@ namespace { // E, k, block-size, res64, nErrors, CRC -static constexpr const char *PRP_v12 = "OWL PRP 12 %u %u %u %016" SCNx64 " %u %u\n"; +static constexpr const char *PRP_v12 = "OWL PRP 12 %" PRIu64 " %" PRIu64 " %u %016" SCNx64 " %u %u\n"; // Anticipated next version of the header. // Has general number form N=k*b^E+c, and labels for values. -static constexpr const char *PRP_v13 = "OWL PRP 13 N=1*2^%u-1 k=%u block=%u res64=%016" SCNx64 " err=%u time=%lf\n"; +static constexpr const char *PRP_v13 = "OWL PRP 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " block=%u res64=%016" SCNx64 " err=%u time=%lf\n"; // static constexpr const char *PRP_v13_PRI = "OWL PRP 13 N=1*2^%u-1 k=%u block=%u res64=%016" PRIx64 " err=%u time=%.0lf\n"; // E, k, CRC -static constexpr const char *LL_v1 = "OWL LL 1 E=%u k=%u CRC=%u\n"; +static constexpr const char *LL_v1 = "OWL LL 1 E=%" PRIu64 " k=%" PRIu64 " CRC=%u\n"; // Anticipated next version. // Push version number to sync it with PRP. -static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%u-1 k=%u time=%lf\n"; +static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " time=%lf\n"; struct BadHeaderError { string name; }; @@ -35,8 +35,8 @@ bool startsWith(const string& s, const string& prefix) { return s.rfind(prefix, 0) == 0; } -vector savefiles(fs::path dir, const string& prefix, const string& kind) { - vector v; +vector savefiles(fs::path dir, const string& prefix, const string& kind) { + vector v; for (const auto& entry: fs::directory_iterator(dir)) { if (entry.is_regular_file()) { string filename = entry.path().filename().string(); @@ -45,7 +45,7 @@ vector savefiles(fs::path dir, const string& prefix, const string& kind) { assert(dot > prefix.size()); string id = filename.substr(prefix.size(), dot - prefix.size()); if (id == "unverified") { continue; } - u32 k = 0; + u64 k = 0; const char* first = id.data(); const char* end = first + id.size(); auto res = from_chars(first, end, k); @@ -61,13 +61,13 @@ vector savefiles(fs::path dir, const string& prefix, const string& kind) { return v; } -string str9(u32 k) { +string str9(u64 k) { char buf[32]; - snprintf(buf, sizeof(buf), "%09u", k); + snprintf(buf, sizeof(buf), "%09" PRIu64, k); return buf; } -fs::path pathFor(fs::path base, const string& prefix, const string& kind, u32 k) { +fs::path pathFor(fs::path base, const string& prefix, const string& kind, u64 k) { return base / (prefix + str9(k) + '.' + kind); } @@ -79,16 +79,17 @@ fs::path pathUnverified(fs::path base, const string& prefix) { // . // e.g.: 125784077-010000000.prp fs::path findLast(fs::path dir, const string& prefix, const string& kind) { - vector v = savefiles(dir, prefix, kind); + vector v = savefiles(dir, prefix, kind); if (v.empty()) { return {}; } - u32 lastK = v.back(); + u64 lastK = v.back(); fs::path path = pathFor(dir, prefix, kind, lastK); assert(is_regular_file(path)); return path; } PRPState readState(const PRPState& dummy, File fi) { - u32 exponent{}, k{}, blockSize{}, nErrors{}; + u64 exponent{}, k{}; + u32 blockSize{}, nErrors{}; u64 res64{}; double elapsed{}; @@ -108,7 +109,7 @@ PRPState readState(const PRPState& dummy, File fi) { } LLState readState(const LLState& dummy, File fi) { - u32 exponent{}, k{}; + u64 exponent{}, k{}; double elapsed{}; string header = fi.readLine(); @@ -142,7 +143,7 @@ void writeState(const File& fo, const LLState& state) { fo.writeChecked(state.data); } -double roundNumberScore(u32 x) { +double roundNumberScore(u64 x) { if (x == 0) { return 1; } double score = 0; @@ -169,7 +170,7 @@ template<> LLState Saver::initState() { // ---- Saver ---- template -Saver::Saver(u32 exponent, u32 blockSize, u32 nSavefiles) : +Saver::Saver(u64 exponent, u32 blockSize, u32 nSavefiles) : exponent{exponent}, blockSize{blockSize}, prefix{to_string(exponent) + '-'}, @@ -188,7 +189,7 @@ template Saver::~Saver() = default; template -void Saver::clear(u32 exponent) { +void Saver::clear(u64 exponent) { error_code dummy; fs::path base = std::is_same_v ? fs::current_path() / to_string(exponent) @@ -244,16 +245,16 @@ State Saver::load() { template void Saver::trimFiles() { - vector v = savefiles(base, prefix, State::KIND); + vector v = savefiles(base, prefix, State::KIND); assert(nSavefiles > 0); while (v.size() > nSavefiles) { int bestIdx = -1; double bestSpan = 1e20; - u32 prevK = 0; + u64 prevK = 0; for (u32 i = 0; i < v.size() - 1; ++i) { - u32 k = v[i]; + u64 k = v[i]; double niceBias = std::min(1.0, roundNumberScore(k) - 4); double span = (v[i + 1] - prevK) * niceBias; prevK = k; @@ -263,8 +264,8 @@ void Saver::trimFiles() { } } assert(bestIdx >= 0); - u32 k = v[bestIdx]; - // log("Deleting savefile %u\n", k); + u64 k = v[bestIdx]; + // log("Deleting savefile %" PRIu64 "\n", k); fs::path path = pathFor(base, prefix, State::KIND, k); fs::remove(path); v.erase(v.begin() + bestIdx); diff --git a/src/Saver.h b/src/Saver.h index 3bf8e6e7..5a3a7ba6 100644 --- a/src/Saver.h +++ b/src/Saver.h @@ -13,8 +13,8 @@ class SaveMan; struct PRPState { static const constexpr char* KIND = "prp"; - u32 exponent; - u32 k; + u64 exponent; + u64 k; u32 blockSize; u64 res64; vector check; @@ -25,15 +25,15 @@ struct PRPState { struct LLState { static const constexpr char* KIND = "ll"; - u32 exponent; - u32 k; + u64 exponent; + u64 k; vector data; double elapsed{}; }; template class Saver { - u32 exponent; + u64 exponent; u32 blockSize; fs::path base; string prefix; @@ -45,7 +45,7 @@ class Saver { fs::path mostRecentSavefile(); public: - Saver(u32 exponent, u32 blockSize, u32 nSavefiles); + Saver(u64 exponent, u32 blockSize, u32 nSavefiles); ~Saver(); State load(); @@ -53,7 +53,7 @@ class Saver { void dropMostRecent(); - static void clear(u32 exponent); + static void clear(u64 exponent); // For PRP, we can save a verified save (see save() above) or an unverified save. void saveUnverified(const PRPState& s) const; diff --git a/src/Task.cpp b/src/Task.cpp index d72f4db2..9a49654f 100644 --- a/src/Task.cpp +++ b/src/Task.cpp @@ -103,7 +103,9 @@ string json(const vector& v) { } string json(const string& s) { return '"' + s + '"'; } +string json(int x) { return to_string(x); } string json(u32 x) { return to_string(x); } +string json(u64 x) { return to_string(x); } template string json(const string& key, const T& value) { return json(key) + ':' + json(value); } @@ -112,7 +114,7 @@ string maybe(const string& key, const string& value) { return value.empty() ? "" template void operator+=(vector& a, const vector& b) { a.insert(a.end(), b.begin(), b.end()); } -vector commonFields(u32 E, const char *worktype, const string &status) { +vector commonFields(u64 E, const char *worktype, const string &status) { return { json("status", status), json("exponent", E), @@ -140,7 +142,7 @@ vector tailFields(const std::string &AID, const Args &args) { }; } -void writeResult(u32 instance, u32 E, const char *workType, const string &status, const std::string &AID, const Args &args, +void writeResult(u32 instance, u64 E, const char *workType, const string &status, const std::string &AID, const Args &args, const vector& extras) { fs::path resultsFile = "results-" + to_string(instance) + ".txt"; vector fields = commonFields(E, workType, status); @@ -220,8 +222,8 @@ void Task::execute(GpuCommon shared, Queue *q, u32 instance) { { Primes primes; if (!primes.isPrime(exponent)) { - u32 new_exponent = primes.prevPrime(exponent); - log("Warning: Exponent %u is not prime. Using exponent %u instead.\n", exponent, new_exponent); + u64 new_exponent = primes.prevPrime(exponent); + log("Warning: Exponent %" PRIu64 " is not prime. Using exponent %" PRIu64 " instead.\n", exponent, new_exponent); exponent = new_exponent; } } @@ -253,7 +255,7 @@ void Task::execute(GpuCommon shared, Queue *q, u32 instance) { Worktodo::deleteTask(*this, instance); if (isPrime) { - log("%u is PRIME!\n", exponent); + log("%" PRIu64 " is PRIME!\n", exponent); } else if (shared.args->clean) { gpu->clear(kind == PRP); } diff --git a/src/Task.h b/src/Task.h index 95f08024..4c130446 100644 --- a/src/Task.h +++ b/src/Task.h @@ -20,7 +20,7 @@ class Task { enum Kind {PRP, VERIFY, LL, CERT}; Kind kind; - u32 exponent; + u64 exponent; string AID; // Assignment ID string line; // the verbatim worktodo line, used in deleteTask(). u32 squarings; // For CERTs diff --git a/src/TuneEntry.cpp b/src/TuneEntry.cpp index c3288d24..68b6915d 100644 --- a/src/TuneEntry.cpp +++ b/src/TuneEntry.cpp @@ -3,10 +3,11 @@ #include "CycleFile.h" #include +#include // Returns whether *results* was updated. bool TuneEntry::update(vector& results) const { - u32 maxExp = fft.maxExp(); + u64 maxExp = fft.maxExp(); [[maybe_unused]] bool didErase = false; int i{}; @@ -28,7 +29,7 @@ bool TuneEntry::update(vector& results) const { // Returns whether entry *e* represents an improvement over *results* (i.e. would update the results). bool TuneEntry::willUpdate(const vector& results) const { - u32 maxExp = fft.maxExp(); + u64 maxExp = fft.maxExp(); for (const auto& r : results) { if (r.cost > cost) { break; @@ -51,7 +52,7 @@ vector TuneEntry::readTuneFile(const Args& args) { File fi = File::openRead(tuneFile); if (!fi) { return {}; } - [[maybe_unused]] u32 prevMaxExp{}; + [[maybe_unused]] u64 prevMaxExp{}; [[maybe_unused]] double prevCost{}; for (const string& line : fi) { @@ -71,14 +72,14 @@ vector TuneEntry::readTuneFile(const Args& args) { } void TuneEntry::writeTuneFile(const vector& results) { - [[maybe_unused]] u32 prevMaxExp{}; + [[maybe_unused]] u64 prevMaxExp{}; [[maybe_unused]] double prevCost{}; CycleFile tune{"tune.txt"}; for (const TuneEntry& r : results) { - u32 maxExp = r.fft.maxExp(); + u64 maxExp = r.fft.maxExp(); assert(r.cost >= prevCost && maxExp > prevMaxExp); prevCost = r.cost; prevMaxExp = maxExp; - tune->printf("%6.1f %14s # %u\n", r.cost, r.fft.spec().c_str(), maxExp); + tune->printf("%6.1f %14s # %" PRIu64 "\n", r.cost, r.fft.spec().c_str(), maxExp); } } diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 0a981a39..80afdc51 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -167,11 +167,11 @@ optional getWork(Args& args, i32 instance) { std::optional Worktodo::getTask(Args &args, i32 instance) { if (instance == 0) { if (args.prpExp) { - u32 exp = args.prpExp; + u64 exp = args.prpExp; args.prpExp = 0; return Task{Task::PRP, exp}; } else if (args.llExp) { - u32 exp = args.llExp; + u64 exp = args.llExp; args.llExp = 0; return Task{Task::LL, exp}; } else if (!args.verifyPath.empty()) { diff --git a/src/common.h b/src/common.h index 516b099d..530795bb 100644 --- a/src/common.h +++ b/src/common.h @@ -46,15 +46,15 @@ using Words = vector; inline u64 res64(const Words& words) { return words.empty() ? 0 : ((u64(words[1]) << 32) | words[0]); } -inline u32 nWords(u32 E) { return (E - 1) / 32 + 1; } +inline u32 nWords(u64 E) { return u32((E - 1) / 32 + 1); } -inline Words makeWords(u32 E, u32 value) { +inline Words makeWords(u64 E, u32 value) { Words ret(nWords(E)); ret[0] = value; return ret; } -inline u32 roundUp(u32 x, u32 multiple) { return ((x - 1) / multiple + 1) * multiple; } +inline u64 roundUp(u64 x, u32 multiple) { return ((x - 1) / multiple + 1) * multiple; } u32 crc32(const void* data, size_t size); diff --git a/src/shared.h b/src/shared.h index c2d90dbc..6405d9c1 100644 --- a/src/shared.h +++ b/src/shared.h @@ -1,4 +1,4 @@ // included from both C++ and OpenCL. -u32 bitposToWord(u32 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } -u32 wordToBitpos(u32 E, u32 N, u32 word) { return (word * ((u64) E) + (N - 1)) / N; } +u32 bitposToWord(u64 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } +u32 wordToBitpos(u64 E, u32 N, u32 word) { return (word * ((u64) E) + (N - 1)) / N; } diff --git a/src/state.cpp b/src/state.cpp index 4ac159d8..77ebff8f 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -10,7 +10,7 @@ static i64 lowBits(i64 u, int bits) { return (u << (64 - bits)) >> (64 - bits); } -std::vector compactBits(const vector &dataVect, u32 E) { +std::vector compactBits(const vector &dataVect, u64 E) { if (dataVect.empty()) { return {}; } // Indicating all zero u32 N = dataVect.size(); @@ -87,7 +87,7 @@ struct BitBucket { } }; -vector expandBits(const vector &compactBits, u32 N, u32 E) { +vector expandBits(const vector &compactBits, u32 N, u64 E) { assert(E % 32 != 0); std::vector out(N); diff --git a/src/state.h b/src/state.h index 9b37c0fd..02fab3fb 100644 --- a/src/state.h +++ b/src/state.h @@ -8,10 +8,10 @@ #include #include -vector compactBits(const vector &dataVect, u32 E); -vector expandBits(const vector &compactBits, u32 N, u32 E); +vector compactBits(const vector &dataVect, u64 E); +vector expandBits(const vector &compactBits, u32 N, u64 E); -constexpr u32 step(u32 N, u32 E) { return N - (E % N); } -constexpr u32 extra(u32 N, u32 E, u32 k) { return u64(step(N, E)) * k % N; } -constexpr bool isBigWord(u32 N, u32 E, u32 k) { return extra(N, E, k) + step(N, E) < N; } -constexpr u32 bitlen(u32 N, u32 E, u32 k) { return E / N + isBigWord(N, E, k); } +constexpr u32 step(u32 N, u64 E) { return N - (E % N); } +constexpr u32 extra(u32 N, u64 E, u32 k) { return u64(step(N, E)) * k % N; } +constexpr bool isBigWord(u32 N, u64 E, u32 k) { return extra(N, E, k) + step(N, E) < N; } +constexpr u32 bitlen(u32 N, u64 E, u32 k) { return E / N + isBigWord(N, E, k); } diff --git a/src/tune.cpp b/src/tune.cpp index 3b773437..a3a12321 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -173,7 +173,7 @@ printf ("Reguess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bp } float Tune::zForBpw(float bpw, FFTConfig fft, u32 count) { - u32 exponent = (count == 1) ? primes.prevPrime(fft.size() * bpw) : primes.nextPrime(fft.size() * bpw); + u64 exponent = (count == 1) ? primes.prevPrime(fft.size() * bpw) : primes.nextPrime(fft.size() * bpw); float total_z = 0.0; for (u32 i = 0; i < count; i++, exponent = primes.nextPrime (exponent + 1)) { auto [ok, res, roeSq, roeMul] = Gpu::make(q, exponent, shared, fft, {}, false)->measureROE(true); @@ -249,7 +249,7 @@ void Tune::carryTune() { double m = 0; const float mid = fft.shape.carry32BPW(); for (float bpw : {mid - 0.05, mid + 0.05}) { - u32 exponent = primes.nearestPrime(fft.size() * bpw); + u64 exponent = primes.nearestPrime(fft.size() * bpw); auto [ok, carry] = Gpu::make(q, exponent, shared, fft, {}, false)->measureCarry(); m = carry.max; if (!ok) { log("Error %s at %f\n", fft.spec().c_str(), bpw); } @@ -257,7 +257,7 @@ void Tune::carryTune() { } float avg = (zv[0] + zv[1]) / 2; - u32 exponent = fft.shape.carry32BPW() * fft.size(); + u64 exponent = fft.shape.carry32BPW() * fft.size(); double pErr100 = -expm1(-exp(-avg) * exponent * 100); log("%14s %.3f : %.3f (%.3f %.3f) %f %.0f%%\n", fft.spec().c_str(), mid, avg, zv[0], zv[1], m, pErr100 * 100); fo.printf("%f %f\n", log2(fft.size()), avg); @@ -292,8 +292,8 @@ void Tune::ctune() { for (FFTShape shape : shapes) { FFTConfig fft{shape, 101, CARRY_32}; - u32 exponent = primes.prevPrime(fft.maxExp()); - // log("tuning %10s with exponent %u\n", fft.shape.spec().c_str(), exponent); + u64 exponent = primes.prevPrime(fft.maxExp()); + // log("tuning %10s with exponent %" PRIu64 "\n", fft.shape.spec().c_str(), exponent); vector bestPos(configsVect.size()); Entry best{{1, 1, 1}, {}, 1e9}; @@ -448,7 +448,7 @@ void Tune::tune() { // Find best IN_WG,IN_SIZEX,OUT_WG,OUT_SIZEX settings if (1/*option to time IN/OUT settings*/) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_in_wg = 0; u32 best_in_sizex = 0; u32 current_in_wg = args->value("IN_WG", 128); @@ -497,7 +497,7 @@ void Tune::tune() { // Find best PAD setting. Default is 256 bytes for AMD, 0 for all others. if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_pad = 0; u32 current_pad = args->value("PAD", AMDGPU ? 256 : 0); double best_cost = -1.0; @@ -517,7 +517,7 @@ void Tune::tune() { // Find best MIDDLE_IN_LDS_TRANSPOSE setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_in_lds_transpose = 0; u32 current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); double best_cost = -1.0; @@ -537,7 +537,7 @@ void Tune::tune() { // Find best MIDDLE_OUT_LDS_TRANSPOSE setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_out_lds_transpose = 0; u32 current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); double best_cost = -1.0; @@ -557,7 +557,7 @@ void Tune::tune() { // Find best INPLACE setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_inplace = 0; double best_cost = -1.0; double current_cost = -1.0; @@ -576,7 +576,7 @@ void Tune::tune() { // Find best NONTEMPORAL setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_nontemporal = 0; u32 current_nontemporal = args->value("NONTEMPORAL", 0); double best_cost = -1.0; @@ -596,7 +596,7 @@ void Tune::tune() { // Find best FAST_BARRIER setting if (AMDGPU) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_fast_barrier = 0; u32 current_fast_barrier = args->value("FAST_BARRIER", 0); double best_cost = -1.0; @@ -616,7 +616,7 @@ void Tune::tune() { // Find best TAIL_KERNELS setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_kernels = 0; u32 current_tail_kernels = args->value("TAIL_KERNELS", 2); double best_cost = -1.0; @@ -639,7 +639,7 @@ void Tune::tune() { // Find best TAIL_TRIGS setting if (time_FFTs) { FFTConfig fft{defaultFFTShape, 101, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; u32 current_tail_trigs = args->value("TAIL_TRIGS", 2); double best_cost = -1.0; @@ -660,7 +660,7 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; u32 current_tail_trigs = args->value("TAIL_TRIGS31", 0); double best_cost = -1.0; @@ -681,7 +681,7 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw u32 best_tail_trigs = 0; u32 current_tail_trigs = args->value("TAIL_TRIGS32", 2); double best_cost = -1.0; @@ -702,7 +702,7 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF61) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; u32 current_tail_trigs = args->value("TAIL_TRIGS61", 0); double best_cost = -1.0; @@ -722,7 +722,7 @@ void Tune::tune() { // Find best TABMUL_CHAIN setting if (time_FFTs) { FFTConfig fft{defaultFFTShape, 101, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; u32 current_tabmul_chain = args->value("TABMUL_CHAIN", 0); double best_cost = -1.0; @@ -743,7 +743,7 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; u32 current_tabmul_chain = args->value("TABMUL_CHAIN31", 0); double best_cost = -1.0; @@ -764,7 +764,7 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw u32 best_tabmul_chain = 0; u32 current_tabmul_chain = args->value("TABMUL_CHAIN32", 0); double best_cost = -1.0; @@ -785,7 +785,7 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF61) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; u32 current_tabmul_chain = args->value("TABMUL_CHAIN61", 0); double best_cost = -1.0; @@ -806,7 +806,7 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_modm31 = 0; u32 current_modm31 = args->value("MODM31", 0); double best_cost = -1.0; @@ -826,7 +826,7 @@ void Tune::tune() { // Find best UNROLL_W setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_unroll_w = 0; u32 current_unroll_w = args->value("UNROLL_W", AMDGPU ? 0 : 1); double best_cost = -1.0; @@ -846,7 +846,7 @@ void Tune::tune() { // Find best UNROLL_H setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_unroll_h = 0; u32 current_unroll_h = args->value("UNROLL_H", AMDGPU && defaultShape->height >= 1024 ? 0 : 1); double best_cost = -1.0; @@ -866,7 +866,7 @@ void Tune::tune() { // Find best ZEROHACK_W setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_zerohack_w = 0; u32 current_zerohack_w = args->value("ZEROHACK_W", 1); double best_cost = -1.0; @@ -886,7 +886,7 @@ void Tune::tune() { // Find best ZEROHACK_H setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_zerohack_h = 0; u32 current_zerohack_h = args->value("ZEROHACK_H", 1); double best_cost = -1.0; @@ -906,7 +906,7 @@ void Tune::tune() { // Find best BIGLIT setting if (time_FFTs) { FFTConfig fft{*defaultShape, 101, CARRY_AUTO}; - u32 exponent = primes.prevPrime(fft.maxExp()); + u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_biglit = 0; u32 current_biglit = args->value("BIGLIT", 1); double best_cost = -1.0; @@ -987,7 +987,7 @@ skip_1K_256 = 0; if ((shape.fft_type == FFT3261 || shape.fft_type == FFT323161 || shape.fft_type == FFT3231 || shape.fft_type == FFT32) && !time_FP32) continue; // Time an exponent that's good for all variants and carry-config. - u32 exponent = primes.prevPrime(FFTConfig{shape, shape.width <= 1024 ? 0u : 100u, CARRY_32}.maxExp()); + u64 exponent = primes.prevPrime(FFTConfig{shape, shape.width <= 1024 ? 0u : 100u, CARRY_32}.maxExp()); u32 adjusted_quick = (exponent < 50000000) ? quick - 1 : (exponent < 170000000) ? quick : (exponent < 350000000) ? quick + 1 : quick + 2; if (adjusted_quick < 1) adjusted_quick = 1; if (adjusted_quick > 10) adjusted_quick = 10; From 205754df4610dfccef54f314dee11fe1fde413e5 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 26 Dec 2025 03:58:25 +0000 Subject: [PATCH 002/214] Merged in previous inplace changes to tune.cpp --- src/tune.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/tune.cpp b/src/tune.cpp index a3a12321..4d613d0f 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -348,6 +348,7 @@ void Tune::tune() { bool time_FFTs = 0; bool time_NTTs = 0; bool time_FP32 = 1; + bool time_inplace_only = 0; int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) u64 min_exponent = 75000000; u64 max_exponent = 350000000; @@ -360,6 +361,7 @@ void Tune::tune() { if (s == "fp64") time_FFTs = 1; if (s == "ntt") time_NTTs = 1; if (s == "nofp32") time_FP32 = 0; + if (s == "inplace") time_inplace_only = 1; auto keyVal = split(s, '='); if (keyVal.size() == 2) { if (keyVal.front() == "quick") quick = stod(keyVal.back()); @@ -446,7 +448,7 @@ void Tune::tune() { args->flags["INPLACE"] = to_string(0); // Find best IN_WG,IN_SIZEX,OUT_WG,OUT_SIZEX settings - if (1/*option to time IN/OUT settings*/) { + if (!time_inplace_only) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_in_wg = 0; @@ -495,7 +497,7 @@ void Tune::tune() { } // Find best PAD setting. Default is 256 bytes for AMD, 0 for all others. - if (1) { + if (!time_inplace_only) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_pad = 0; @@ -515,7 +517,7 @@ void Tune::tune() { } // Find best MIDDLE_IN_LDS_TRANSPOSE setting - if (1) { + if (!time_inplace_only) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_in_lds_transpose = 0; @@ -535,7 +537,7 @@ void Tune::tune() { } // Find best MIDDLE_OUT_LDS_TRANSPOSE setting - if (1) { + if (!time_inplace_only) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_out_lds_transpose = 0; @@ -554,8 +556,13 @@ void Tune::tune() { args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(best_middle_out_lds_transpose); } + // If only timing INPLACE=1 options, then set INPLACE + if (time_inplace_only) { + args->flags["INPLACE"] = to_string(1); + newConfigKeyVals.push_back({"INPLACE", 1}); + } // Find best INPLACE setting - if (1) { + else { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_inplace = 0; @@ -905,7 +912,7 @@ void Tune::tune() { // Find best BIGLIT setting if (time_FFTs) { - FFTConfig fft{*defaultShape, 101, CARRY_AUTO}; + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_biglit = 0; u32 current_biglit = args->value("BIGLIT", 1); From a3719b01449edf18a961cc5b44acbced44deca15 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 26 Dec 2025 04:10:47 +0000 Subject: [PATCH 003/214] Fixed carry propagtion bug when BPW was very low (lower than one would see in normal usage). The optimization that generated 32-bit FFT data words requiring long carries was modified. --- src/cl/carryutil.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 6cdff372..4e04812b 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -681,7 +681,7 @@ Word OVERLOAD carryStepSignedSloppy(i96 x, i64 *outCarry, bool isBigWord) { // i64 xhi = i96_hi64(x) + xmid_topbit; // *outCarry = xhi >> (nBits - 32); // return as_long((int2)(i96_lo32(x), whi)); -#elif EXP / NWORDS == 31 || SLOPPY_MAXBPW >= 3200 // nBits = 31 or 32, bigwordBits = 32 (or allowed to create 32-bit word for better performance) +#elif EXP / NWORDS == 31 || (SLOPPY_MAXBPW >= 3200 && EXP / NWORDS >= 22) // nBits = 31 or 32, bigwordBits = 32 (or allowed to create 32-bit word for better performance) i32 w = i96_lo32(x); // lowBits(x, bigwordBits = 32); *outCarry = (i96_hi64(x) + (w < 0)) << (32 - nBits); return w; From eceaad6173232be7af9980f8cd961ad9792ea9ac Mon Sep 17 00:00:00 2001 From: george Date: Tue, 30 Dec 2025 01:34:09 +0000 Subject: [PATCH 004/214] Output num squarings a cert will perform as well as an ETA. --- src/Gpu.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 433ce0a7..e9d20522 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -2086,7 +2086,7 @@ array Gpu::isCERT(const Task& task) { float secsPerIt = iterationTimer.reset(k); queue->setSquareTime((int) (secsPerIt * 1'000'000)); - log("%9u %016" PRIx64 " %4.0f\n", k, res64, secsPerIt * 1'000'000); + log("%7u / %7u %016" PRIx64 " %4.0f ETA %s\n", k, kEnd, res64, secsPerIt * 1'000'000, getETA(k, kEnd, secsPerIt).c_str()); if (k >= kEnd) { fs::remove (fname); From 5cfbe0b3d0adb42ede81fdea15a3d17a3c59fceb Mon Sep 17 00:00:00 2001 From: george Date: Tue, 30 Dec 2025 01:43:17 +0000 Subject: [PATCH 005/214] Output error message when worktodo-N.txt is empty. Helpful for novice users. --- src/Worktodo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 80afdc51..1628a391 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -114,7 +114,7 @@ optional getWork(Args& args, i32 instance) { // Try to get a task from the local worktodo- file. if (optional task = bestTask(localWork, args.smallest)) { return task; } - if (args.masterDir.empty()) { return {}; } + if (args.masterDir.empty()) { log("No work to do found. Add work to %s.\n", localWork.c_str()); return {}; } fs::path worktodo = args.masterDir / "worktodo.txt"; From 3a7a76ddb33a3c69999065b2274fe2d0e4e7788b Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Sun, 14 Dec 2025 03:29:45 -0800 Subject: [PATCH 006/214] Fixed GitHub Actions CI and replaced macOS 13. --- .github/workflows/ci.yml | 47 +++++++++++++++++++++++++--------------- Makefile | 8 +++---- genbundle.sh | 17 ++++++++------- src/Args.cpp | 2 +- src/common.h | 2 +- src/tune.cpp | 2 +- 6 files changed, 45 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3947c982..6d43b725 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,13 +13,16 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-22.04-arm, ubuntu-24.04-arm] cxx: [g++, clang++] + exclude: + - os: ubuntu-22.04-arm + cxx: clang++ fail-fast: false env: CXX: ${{ matrix.cxx }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install run: | sudo apt-get update -y @@ -27,14 +30,14 @@ jobs: $CXX --version - name: Script run: | - make prpll -O -j "$(nproc)" + make -O -j "$(nproc)" cd build-release rm -f -- *.o ./prpll -h - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: - name: ${{ matrix.os }}_${{ matrix.cxx }}_prpll + name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_prpll path: ${{ github.workspace }} - name: Cppcheck run: cppcheck --enable=all --force . @@ -49,15 +52,17 @@ jobs: Windows: name: Windows - runs-on: windows-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [windows-latest] # windows-11-arm cxx: [g++, clang++] fail-fast: false env: CXX: ${{ matrix.cxx }} + PACKAGE_PREFIX: mingw-w64-${{ endsWith(matrix.os, '-arm') && 'clang-aarch64' || 'x86_64' }}- steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Before Install run: | echo "C:\msys64\mingw64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append @@ -66,7 +71,7 @@ jobs: echo "LIBPATH=-LC:\msys64\mingw64\lib" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Install run: | - pacman -S --noconfirm mingw-w64-x86_64-gmp mingw-w64-x86_64-opencl-icd + pacman -S --noconfirm "${env:PACKAGE_PREFIX}opencl-icd" & $env:CXX --version - name: Install Clang if: ${{ matrix.cxx == 'clang++' }} @@ -74,34 +79,40 @@ jobs: pacman -S --noconfirm mingw-w64-x86_64-clang & $env:CXX --version - name: Script - run: | # Cannot use `make exe`, as the OpenCL ICD Loader does not support static linking - make prpll -O -j $env:NUMBER_OF_PROCESSORS + run: | + make -O -j $env:NUMBER_OF_PROCESSORS cd build-release rm *.o .\prpll.exe -h - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: - name: win_${{ matrix.cxx }}_prpll + name: win_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_prpll path: ${{ github.workspace }} macOS: name: macOS - runs-on: macos-13 + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-15-intel, macos-latest] + fail-fast: false + env: + CXX: g++-15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Install run: | - brew install gcc@14 + $CXX --version - name: Script run: | - make prpll -j "$(sysctl -n hw.ncpu)" + make -j "$(sysctl -n hw.ncpu)" cd build-release rm -f -- *.o ./prpll -h - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: - name: macos_prpll + name: macos_${{ endsWith(matrix.os, '-intel') && 'x86' || 'arm' }}_prpll path: ${{ github.workspace }} diff --git a/Makefile b/Makefile index 26bffeda..b1c021e0 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,9 @@ HOST_OS = $(shell uname -s) ifeq ($(HOST_OS), Darwin) # Real GCC (not clang), needed for 128-bit floats and std::filesystem::path -CXX = g++-14 +CXX ?= g++-15 else -CXX = g++ +CXX ?= g++ endif ifneq ($(findstring MINGW, $(HOST_OS)), MINGW) @@ -45,7 +45,7 @@ else BIN=build-release -CXXFLAGS = -O2 -DNDEBUG $(COMMON_FLAGS) +CXXFLAGS = -O3 -DNDEBUG $(COMMON_FLAGS) STRIP=-s endif @@ -90,7 +90,7 @@ $(BIN)/%.o : src/%.cpp $(DEPDIR)/%.d # src/bundle.cpp is just a wrapping of the OpenCL sources (*.cl) as a C string. src/bundle.cpp: genbundle.sh src/cl/*.cl - ./genbundle.sh $^ > src/bundle.cpp + bash genbundle.sh $^ > src/bundle.cpp $(DEPDIR)/%.d: ; .PRECIOUS: $(DEPDIR)/%.d diff --git a/genbundle.sh b/genbundle.sh index ec042bb2..9d176062 100755 --- a/genbundle.sh +++ b/genbundle.sh @@ -1,3 +1,4 @@ +#!/bin/bash cat < CL_FILE_NAMES\{${names}\}\; +echo "static const std::vector CL_FILE_NAMES{${names}};" cat <& getClFileNames() { return CL_FILE_NAMES; } diff --git a/src/Args.cpp b/src/Args.cpp index 041202ad..b18a95ba 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -118,7 +118,7 @@ and should be able to run. PRPLL keeps the active tasks in per-worker files worktodo-0.txt, worktodo-1.txt etc in the local directory. These per-worker files are supplied from the global worktodo.txt file if -pool is used. In turn the global worktodo.txt can be supplied through the primenet.py script, -either the one located at gpuowl/tools/primenet.py or https://download.mersenne.ca/primenet.py +either the one located at gpuowl/tools/primenet.py or https://download.mersenne.ca/AutoPrimeNet It is also possible to manually add exponents by adding lines of the form "PRP=118063003" to worktodo-.txt diff --git a/src/common.h b/src/common.h index 516b099d..6303e539 100644 --- a/src/common.h +++ b/src/common.h @@ -13,7 +13,7 @@ using i64 = int64_t; using u64 = uint64_t; using i128 = __int128; using u128 = unsigned __int128; -using f128 = __float128; +// using f128 = __float128; static_assert(sizeof(u8) == 1, "size u8"); static_assert(sizeof(u32) == 4, "size u32"); diff --git a/src/tune.cpp b/src/tune.cpp index 3b773437..b9cf1f71 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -947,7 +947,7 @@ void Tune::tune() { config.write("\n -log 1000000\n"); } if (args->workers < 2) { - config.write("\n# Running two workers sometimes gives better throughput. Autoprimenet will need to create up a second worktodo file."); + config.write("\n# Running two workers sometimes gives better throughput. AutoPrimeNet will need to create up a second worktodo file (use --num-workers 2)."); config.write("\n# -workers 2\n"); config.write("\n# Changing TAIL_KERNELS to 3 when running two workers may be better."); config.write("\n# -use TAIL_KERNELS=3\n"); From 89be60a188253b74593e4f36cb14b189d08a7c3c Mon Sep 17 00:00:00 2001 From: george Date: Wed, 4 Mar 2026 19:09:21 +0000 Subject: [PATCH 007/214] Changed type 3 4M FFT max bpw. LL test of 100028317 failed under old limit. --- src/fftbpw.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fftbpw.h b/src/fftbpw.h index c62db530..f3af7da7 100644 --- a/src/fftbpw.h +++ b/src/fftbpw.h @@ -138,9 +138,9 @@ {"3:256:16:256", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, { "3:512:8:256", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, { "3:512:4:512", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, -{ "3:1K:8:256", {23.94, 23.94, 23.94, 23.94, 23.94, 23.94}}, -{"3:512:16:256", {23.94, 23.94, 23.94, 23.94, 23.94, 23.94}}, -{ "3:512:8:512", {23.94, 23.94, 23.94, 23.94, 23.94, 23.94}}, +{ "3:1K:8:256", {23.84, 23.84, 23.84, 23.84, 23.84, 23.84}}, // LL of 100028317 failed (ROEmax=0.294, ROEavg=0.247). Lowering bpw from 23.94 to 23.84. +{"3:512:16:256", {23.84, 23.84, 23.84, 23.84, 23.84, 23.84}}, +{ "3:512:8:512", {23.84, 23.84, 23.84, 23.84, 23.84, 23.84}}, { "3:1K:16:256", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, { "3:1K:8:512", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, {"3:512:16:512", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, From 6b39a5f660ba21f103a0dc3f4bb36867f6301af8 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 11 Mar 2026 22:42:11 +0000 Subject: [PATCH 008/214] CarryFused can now process multiple lines at the same time -- 1% performance increase on TitanV. Standardized LDS memory layout and bar() strategy. Made a cleaner, common shufl routine to handle multiple lines using new constants SHUFL_BYTES_W and SHUFL_BYTES_H. Reverse line routines overhauled to use LDS memory layout and bar() strategy. Added L2STORE and LULOAD routines for nVidia. Need to study which GPUs might benefit. Deprecated BIGLIT=0. --- src/Gpu.cpp | 52 +- src/Gpu.h | 2 +- src/cl/base.cl | 63 +- src/cl/carryfused.cl | 1349 +++++++++++++++++++++--------------------- src/cl/fftbase.cl | 412 +++++-------- src/cl/fftheight.cl | 214 ++----- src/cl/ffthin.cl | 25 +- src/cl/fftp.cl | 90 +-- src/cl/fftw.cl | 28 +- src/cl/fftwidth.cl | 171 +++--- src/cl/math.cl | 6 +- src/cl/middle.cl | 62 +- src/cl/tailmul.cl | 104 ++-- src/cl/tailsquare.cl | 222 +++---- src/cl/tailutil.cl | 639 ++++++++------------ src/tune.cpp | 4 +- 16 files changed, 1513 insertions(+), 1930 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index e9d20522..84cc4434 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -85,7 +85,6 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { vector weightsConstIF; vector weightsIF; - vector bits; if (fft.FFT_FP64) { // Inverse + Forward @@ -141,24 +140,7 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { memcpy((double *) weightsIF.data(), weightsIF32.data(), weightsIF32.size() * sizeof(float)); } - if (fft.FFT_FP64 || fft.FFT_FP32) { - for (u32 line = 0; line < H; ++line) { - for (u32 thread = 0; thread < groupWidth; ) { - std::bitset<32> b; - for (u32 bitoffset = 0; bitoffset < 32; bitoffset += nW*2, ++thread) { - for (u32 block = 0; block < nW; ++block) { - for (u32 rep = 0; rep < 2; ++rep) { - if (isBigWord(N, E, kAt(H, line, block * groupWidth + thread) + rep)) { b.set(bitoffset + block * 2 + rep); } - } - } - } - bits.push_back(b.to_ulong()); - } - } - assert(bits.size() == N / 32); - } - - return Weights{weightsConstIF, weightsIF, bits}; + return Weights{weightsConstIF, weightsIF}; } string toLiteral(i32 value) { return to_string(value); } @@ -228,7 +210,7 @@ constexpr bool isInList(const string& s, initializer_list list) { } string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u64 E, bool doLog, - bool &tail_single_wide, bool &tail_single_kernel, u32 &in_place, u32 &pad_size) { + bool &tail_single_wide, bool &tail_single_kernel, u32 &in_place, u32 &pad_size, u32 &wmul) { map config; // Highest priority is the requested "extra" conf @@ -246,6 +228,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Default value for -use options that must also be parsed in C++ code tail_single_wide = 0, tail_single_kernel = 1; // Default tailSquare is double-wide in one kernel in_place = 0; // Default is not in-place + wmul = 1; // Default is carryFused processes one workgroup at a time pad_size = isAmdGpu(id) ? 256 : 0; // Default is 256 bytes for AMD, 0 for others // Validate -use options @@ -264,7 +247,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "NO_ASM", "DEBUG", "CARRY64", - "BIGLIT", + "BIGLIT", // Deprecated "NONTEMPORAL", "INPLACE", "PAD", @@ -279,7 +262,8 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "TABMUL_CHAIN31", "TABMUL_CHAIN32", "TABMUL_CHAIN61", - "MODM31" + "MODM31", + "WMUL" }); if (!isValid) { log("Warning: unrecognized -use key '%s'\n", k.c_str()); @@ -293,6 +277,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< if (atoi(v.c_str()) == 3) tail_single_wide = 0, tail_single_kernel = 0; } if (k == "INPLACE") in_place = atoi(v.c_str()); + if (k == "WMUL") wmul = atoi(v.c_str()); if (k == "PAD") pad_size = atoi(v.c_str()); } @@ -532,7 +517,7 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& nW(fft.shape.nW()), nH(fft.shape.nH()), useLongCarry{args.carry == Args::CARRY_LONG}, - compiler{args, queue->context, clDefines(args, queue->context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size)}, + compiler{args, queue->context, clDefines(args, queue->context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size, wmul)}, #define K(name, ...) name(#name, &compiler, profile.make(#name), queue, __VA_ARGS__) @@ -581,11 +566,11 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1"), K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1 -DROE=1"), K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, "-DLL=1"), - K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW), - K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DROE=1"), - K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DMUL3=1"), - K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DMUL3=1 -DROE=1"), - K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + 1) / nW, "-DLL=1"), + K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW), + K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DROE=1"), + K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DMUL3=1"), + K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DMUL3=1 -DROE=1"), + K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DLL=1"), K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN), @@ -615,7 +600,6 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& weights{genWeights(fft, E, WIDTH, BIG_H, nW, isAmdGpu(q->context->deviceId()))}, bufConstWeights{q->context, std::move(weights.weightsConstIF)}, bufWeights{q->context, std::move(weights.weightsIF)}, - bufBits{q->context, std::move(weights.bitsCF)}, #define BUF(name, ...) name{profile.make(#name), queue, __VA_ARGS__} @@ -695,16 +679,16 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& kfftWGF61.setFixedArgs(2, bufTrigW); } - if (fft.FFT_FP64 || fft.FFT_FP32) { // The FP versions take bufWeight arguments (and bufBits which may be deleted) + if (fft.FFT_FP64 || fft.FFT_FP32) { // The FP versions take bufWeight arguments kfftP.setFixedArgs(2, bufTrigW, bufWeights); for (Kernel* k : {&kCarryA, &kCarryAROE, &kCarryM, &kCarryMROE, &kCarryLL}) { k->setFixedArgs(3, bufCarry, bufWeights); } for (Kernel* k : {&kCarryA, &kCarryM, &kCarryLL}) { k->setFixedArgs(5, bufStatsCarry); } for (Kernel* k : {&kCarryAROE, &kCarryMROE}) { k->setFixedArgs(5, bufROE); } for (Kernel* k : {&kCarryFused, &kCarryFusedROE, &kCarryFusedMul, &kCarryFusedMulROE, &kCarryFusedLL}) { - k->setFixedArgs(3, bufCarry, bufReady, bufTrigW, bufBits, bufConstWeights, bufWeights); + k->setFixedArgs(3, bufCarry, bufReady, bufTrigW, bufConstWeights, bufWeights); } - for (Kernel* k : {&kCarryFusedROE, &kCarryFusedMulROE}) { k->setFixedArgs(9, bufROE); } - for (Kernel* k : {&kCarryFused, &kCarryFusedMul, &kCarryFusedLL}) { k->setFixedArgs(9, bufStatsCarry); } + for (Kernel* k : {&kCarryFusedROE, &kCarryFusedMulROE}) { k->setFixedArgs(8, bufROE); } + for (Kernel* k : {&kCarryFused, &kCarryFusedMul, &kCarryFusedLL}) { k->setFixedArgs(8, bufStatsCarry); } } else { kfftP.setFixedArgs(2, bufTrigW); for (Kernel* k : {&kCarryA, &kCarryAROE, &kCarryM, &kCarryMROE, &kCarryLL}) { k->setFixedArgs(3, bufCarry); } @@ -2042,7 +2026,7 @@ array Gpu::isCERT(const Task& task) { char fname[32]; sprintf(fname, "M%" PRIu64 ".cert", E); -// Autoprimenet.py does not add the cert entry to worktodo.txt until it has successfully downloaded the .cert file. +// AutoPrimenet.py does not add the cert entry to worktodo.txt until it has successfully downloaded the .cert file. { // Enclosing this code in braces ensures the file will be closed by the File destructor. The later file deletion requires the file be closed in Windows. File fi = File::openReadThrow(fname); diff --git a/src/Gpu.h b/src/Gpu.h index ad859eac..b184c4d6 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -172,6 +172,7 @@ class Gpu { bool tail_single_wide; // TailSquare processes one line at a time bool tail_single_kernel; // TailSquare does not use a separate kernel for line zero u32 in_place; // Should GPU perform transform in-place. 1 = nVidia friendly memory layout, 2 = AMD friendly. + u32 wmul; // Number of workgroups carryFused kernel should process ("width multiplier"). u32 pad_size; // Pad size in bytes as specified on the command line or config.txt. Maximum value is 512. // Twiddles: trigonometry constant buffers, used in FFTs. @@ -185,7 +186,6 @@ class Gpu { Weights weights; Buffer bufConstWeights; Buffer bufWeights; - Buffer bufBits; // bigWord bits aligned for CarryFused/fftP // "integer word" buffers. These are "small buffers": N x int. Buffer bufData; // Main int buffer with the words. diff --git a/src/cl/base.cl b/src/cl/base.cl index df1ef02b..f252cfe1 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -58,6 +58,13 @@ G_H "group height" == SMALL_HEIGHT / NH //__builtin_assume(condition) #endif // DEBUG +#ifndef AMDGPU +#define AMDGPU 0 +#endif +#ifndef NVIDIAGPU +#define NVIDIAGPU 0 +#endif + #if NO_ASM #define HAS_ASM 0 #define HAS_PTX 0 @@ -128,8 +135,14 @@ G_H "group height" == SMALL_HEIGHT / NH #endif #endif -#if !defined(BIGLIT) -#define BIGLIT 1 +// Shufl width in bytes (can be 4, 8, or 16). See fftbase.cl. Allow different shufl widths for fft_width and fft_height. +// Default is 8 bytes (one double). Historically best for Radeon VII and TitanV. This setting will affect how much LDS +// memory is needed which in turn may affect occupancy and thus performance. +#if !defined(SHUFL_BYTES_W) +#define SHUFL_BYTES_W 8 +#endif +#if !defined(SHUFL_BYTES_H) +#define SHUFL_BYTES_H 8 #endif #if !defined(TABMUL_CHAIN) @@ -259,6 +272,42 @@ ulong2 OVERLOAD U2(ulong a, ulong b) { return (ulong2) (a, b); } #define NTSTORE(mem,val) (mem) = val #endif +// Routines for storing to L2 cache bypassing L1 cache. +void OVERLOAD L2STORE(i64 *mem, i64 val) { +#if ENABLE_L2STORE && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher + __asm("st.global.cg.b64 [%0], %1;" : : "l"(mem), "l"(val)); +#else + *mem = val; +#endif +} +void OVERLOAD L2STORE(i32 *mem, i32 val) { +#if ENABLE_L2STORE && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher + __asm("st.global.cg.b32 [%0], %1;" : : "l"(mem), "r"(val)); +#else + *mem = val; +#endif +} + +// Routines for loading a value and marking it for "last use". +i64 OVERLOAD LULOAD(i64 *mem) { +#if ENABLE_LULOAD && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher + i64 retval; + __asm("ld.global.lu.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +#else + return *mem; +#endif +} +i32 OVERLOAD LULOAD(i32 *mem) { +#if ENABLE_LULOAD && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher + i32 retval; + __asm("ld.global.lu.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +#else + return *mem; +#endif +} + // Prefetch macros. Unused at present, I tried using them in fftMiddleInGF61 on a 5080 with no benefit. void PREFETCHL1(const __global void *addr) { #if HAS_PTX >= 200 // Prefetch instruction requires sm_20 support or higher @@ -371,7 +420,15 @@ void OVERLOAD bar(void) { #endif } -void OVERLOAD bar(u32 WG) { if (WG > WAVEFRONT) { bar(); } } +void OVERLOAD bar(const u32 WG) { + if (WG > WAVEFRONT) { +#if ENABLE_BARSYNC && HAS_PTX >= 200 // bar.sync with thread count requires sm_20 support or higher. Slower on TitanV, need to try on later nVidia GPUs. + __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); +#else + bar(); +#endif + } +} // A half-barrier is only needed when half-a-workgroup needs a barrier. // This is used e.g. by the double-wide tailSquare, where LDS is split between the halves. diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 05e4ca4c..ba48c8bb 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -16,18 +16,101 @@ void spin() { #endif } +// Increasing WMUL to 2 will reduce carryShuttle activity. This led to a 1% speedup on Titan V. Testing on other GPUs is needed. +#ifndef WMUL +#define WMUL 1 +#endif + +#if AMDGPU +#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions +//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 +#else +#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better +#endif + +// The last WMUL workgroup's carries have been written to global memory. Now we shuffle WMUL-1 workgroups carries up using local memory. +void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) { + // If WMUL is one, there is no shuffling of carries + if (WMUL == 1) return; + + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes used by shufl for each WMUL workgroup + const u32 lds_i64s = lds_bytes / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL workgroup + local i64 *lds = (local i64 *) lds2; + + // Handle nasty case where we are writing 8-byte quantities but SHUFL_BYTES_W is only 4 bytes + if (SHUFL_BYTES_W == 4) { + if (WMUL == 2) { + // Full barrier needed as we are using the entire LDS buffer. + bar(); + // Write the carries. This will use the entire LDS buffer. + if (me < G_W) for (i32 i = 0; i < NW; ++i) lds[i * G_W + lowMe] = carry[i]; + // Read carries from previous WMUL workgroup + bar(); + if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W + lowMe]; + // Full barrier needed as one workgroup just read data from two workgroups LDS buffer. Not compatible with shufl(). + bar(); + } + + // The really nasty case where all the carries will not fit in LDS memory + else { + lds += (me / G_W) * lds_i64s + lowMe; // This WMUL workgroup's LDS area + // Write half the carries to next WMUL's workgroup LDS area + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW/2; ++i) lds[lds_i64s + i * G_W] = carry[i]; + // Read carries from our WMUL workgroup LDS area + bar(); + if (me >= G_W) for (i32 i = 0; i < NW/2; ++i) carry[i] = lds[i * G_W]; + // Write the other half of the carries + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW/2; ++i) lds[lds_i64s + i * G_W] = carry[i + NW/2]; + // Read carries from our WMUL workgroup LDS area. Compatible with shufl, no trailing bar() needed. + bar(); + if (me >= G_W) for (i32 i = 0; i < NW/2; ++i) carry[i + NW/2] = lds[i * G_W]; + } + } + + // Easy case. Write carries to local memory (except last WMUL workgroup which was written to global memory). + else { + lds += (me / G_W) * lds_i64s + lowMe; // This WMUL workgroup's LDS area + // Full barrier needed as we are moving data to next WMUL workgroup's LDS area + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW; ++i) lds[lds_i64s + i * G_W] = carry[i]; + // Full barrier needed as we just moved data from one WMUL workgroup LDS area to the another WMUL workgroup's LDS area + bar(); + // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl and no trailing bar() is required. + if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; + } +} + +// The last WMUL workgroup's carries have been written to global memory. Now we shuffle WMUL-1 workgroup carries up using local memory. +void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) { + // If WMUL is one, there is no shuffling of carries + if (WMUL == 1) return; + + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes used by shufl for each WMUL workgroup + const u32 lds_i32s = lds_bytes / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL workgroup + local i32 *lds = (local i32 *) lds2; + lds += (me / G_W) * lds_i32s + lowMe; // This WMUL workgroup's LDS area + + // Write carries to local memory (except last WMUL workgroup which was written to global memory) + // Full barrier needed as we are moving data to next WMUL workgroup's LDS area + bar(); + if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW; ++i) lds[lds_i32s + i * G_W] = carry[i]; + // Full barrier needed as we just moved data from one WMUL workgroup LDS area to the another WMUL workgroup's LDS area + bar(); + // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl and no trailing bar() is required. + if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; +} + + #if FFT_TYPE == FFT64 // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local T2 lds[WIDTH / 4]; -#else - local T2 lds[WIDTH / 2]; -#endif +KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local T2 lds[WMUL * lds_bytes / sizeof(T2)]; T2 u[NW]; @@ -35,36 +118,31 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); - -// Split 32 bits into NW groups of 2 bits. See later for different way to do this. -#if !BIGLIT -#define GPW (16 / NW) - u32 b = NTLOAD(bits[(G_W * line + me) / GPW]) >> (me % GPW * (2 * NW)); -#undef GPW -#endif + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; #if AMDGPU - T2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + T2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); #else - T2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + T2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights #endif #if MUL3 @@ -75,24 +153,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( CFcarry carry[NW+1]; #endif -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - // On Titan V it is faster to derive the big vs. little flags from the fractional number of bits in each FFT word rather than read the flags from memory. - // On Radeon VII this code is about the same speed. Not sure which is better on other GPUs. -#if BIGLIT // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; u32 frac_bits = word_index * FRAC_BPW_HI + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); -#endif // Apply the inverse weights and carry propagate pairs to generate the output carries @@ -103,13 +170,8 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( T invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); // Generate big-word/little-word flags -#if BIGLIT bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; bool biglit1 = frac_bits + i * frac_bits_bigstep >= -FRAC_BPW_HI; // Same as frac_bits + i * frac_bits_bigstep + FRAC_BPW_HI <= FRAC_BPW_HI; -#else - bool biglit0 = test(b, 2 * i); - bool biglit1 = test(b, 2 * i + 1); -#endif // Apply the inverse weights, optionally compute roundoff error, and convert to integer. Also apply MUL3 here. // Then propagate carries through two words (the first carry does not have to be accurately calculated because it will @@ -126,28 +188,28 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -163,68 +225,66 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u[i] = U2(weight1, weight2); } + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } // Apply each 32 or 64 bit carry to the 2 words for (i32 i = 0; i < NW; ++i) { -#if BIGLIT bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; -#else - bool biglit0 = test(b, 2 * i); -#endif wu[i] = carryFinal(wu[i], carry[i], biglit0); u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); } - bar(); + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); -// fft_WIDTH(lds, u, smallTrig); - new_fft_WIDTH2(lds, u, smallTrig); - - writeCarryFusedLine(u, out, line); + writeCarryFusedLine(u, out, line, lowMe); } @@ -236,14 +296,10 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local F2 lds[WIDTH / 4]; -#else - local F2 lds[WIDTH / 2]; -#endif +KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local F2 lds[WMUL * lds_bytes / sizeof(F2)]; F2 u[NW]; @@ -251,46 +307,41 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; #if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights #endif P(CFcarry) carryShuttlePtr = (P(CFcarry)) carryShuttle; CFcarry carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; u32 frac_bits = word_index * FRAC_BPW_HI + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); @@ -321,28 +372,28 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -358,48 +409,53 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( u[i] = U2(weight1, weight2); } + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -410,12 +466,9 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); } - bar(); - -// fft_WIDTH(lds, u, smallTrig); - new_fft_WIDTH2(lds, u, smallTrig); + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); - writeCarryFusedLine(u, out, line); + writeCarryFusedLine(u, out, line, lowMe); } @@ -427,13 +480,9 @@ KERNEL(G_W) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF31 lds[WIDTH / 4]; -#else - local GF31 lds[WIDTH / 2]; -#endif +KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local GF31 lds[WMUL * lds_bytes / sizeof(GF31)]; GF31 u[NW]; @@ -441,40 +490,35 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; P(CFcarry) carryShuttlePtr = (P(CFcarry)) carryShuttle; CFcarry carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - u32 roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Weights can be applied with shifts because 2 is the 60th root GF31. @@ -537,28 +581,28 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -566,48 +610,52 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle __asm("s_setprio 0"); #endif + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -627,11 +675,9 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle if (weight_shift > 31) weight_shift -= 31; } - bar(); - - new_fft_WIDTH2(lds, u, smallTrig); + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); - writeCarryFusedLine(u, out, line); + writeCarryFusedLine(u, out, line, lowMe); } @@ -643,13 +689,9 @@ KERNEL(G_W) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF61 lds[WIDTH / 4]; -#else - local GF61 lds[WIDTH / 2]; -#endif +KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local GF61 lds[WMUL * lds_bytes / sizeof(GF61)]; GF61 u[NW]; @@ -657,23 +699,25 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif #if HAS_ASM __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; @@ -685,17 +729,10 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle CFcarry carry[NW+1]; #endif -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - u32 roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Weights can be applied with shifts because 2 is the 60th root GF61. @@ -758,28 +795,28 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -787,48 +824,53 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle __asm("s_setprio 0"); #endif + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -848,11 +890,9 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle if (weight_shift > 61) weight_shift -= 61; } - bar(); - - new_fft_WIDTH2(lds, u, smallTrig); + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); - writeCarryFusedLine(u, out, line); + writeCarryFusedLine(u, out, line, lowMe); } @@ -864,10 +904,10 @@ KERNEL(G_W) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - - local T2 lds[WIDTH / 2]; +KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local T2 lds[WMUL * lds_bytes / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; T2 u[NW]; @@ -877,7 +917,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -887,43 +933,30 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line); - readCarryFusedLine(in31, u31, line); + readCarryFusedLine(in, u, line, lowMe); + readCarryFusedLine(in31, u31, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack); - bar(); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); -#else - new_fft_WIDTH1(lds, u, smallTrig); - bar(); - new_fft_WIDTH1(lds31, u31, smallTrig31); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; #if AMDGPU - T2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + T2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); #else - T2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + T2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights #endif + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -987,28 +1020,28 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1024,48 +1057,53 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u[i] = U2(weight1, weight2); } + // Shuffle carries up + shufl_carries_up(lds, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -1087,15 +1125,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( if (weight_shift > 31) weight_shift -= 31; } - bar(); - - new_fft_WIDTH2(lds, u, smallTrig); - writeCarryFusedLine(u, out, line); + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u, out, line, lowMe); - bar(); - - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); + new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1107,10 +1141,10 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - - local F2 ldsF2[WIDTH / 2]; +KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local F2 ldsF2[WMUL * lds_bytes / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; F2 uF2[NW]; @@ -1120,7 +1154,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1133,43 +1173,30 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line); - readCarryFusedLine(in31, u31, line); + readCarryFusedLine(inF2, uF2, line, lowMe); + readCarryFusedLine(in31, u31, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack); - bar(); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); -#else - new_fft_WIDTH1(ldsF2, uF2, smallTrigF2); - bar(); - new_fft_WIDTH1(lds31, u31, smallTrig31); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; #if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights #endif + P(i32) carryShuttlePtr = (P(i32)) carryShuttle; i32 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -1233,28 +1260,28 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1270,48 +1297,53 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( uF2[i] = U2(weight1, weight2); } + // Shuffle carries up + shufl_carries_up(ldsF2, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -1333,15 +1365,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( if (weight_shift > 31) weight_shift -= 31; } - bar(); - - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, line); + new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(uF2, outF2, line, lowMe); - bar(); - - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); + new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1353,10 +1381,10 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - - local GF61 lds61[WIDTH / 2]; +KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; F2 uF2[NW]; @@ -1366,7 +1394,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1379,43 +1413,30 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line); - readCarryFusedLine(in61, u61, line); + readCarryFusedLine(inF2, uF2, line, lowMe); + readCarryFusedLine(in61, u61, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack); - bar(); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack); -#else - new_fft_WIDTH1(ldsF2, uF2, smallTrigF2); - bar(); - new_fft_WIDTH1(lds61, u61, smallTrig61); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; #if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights #endif + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 61. @@ -1479,28 +1500,28 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1516,48 +1537,53 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( uF2[i] = U2(weight1, weight2); } + // Shuffle carries up + shufl_carries_up(lds61, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -1579,15 +1605,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( if (weight_shift > 61) weight_shift -= 61; } - bar(); + new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(uF2, outF2, line, lowMe); - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, line); - - bar(); - - new_fft_WIDTH2(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, line); + new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1599,13 +1621,9 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF61 lds61[WIDTH / 4]; -#else - local GF61 lds61[WIDTH / 2]; -#endif +KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; GF31 u31[NW]; @@ -1615,7 +1633,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -1628,38 +1652,25 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(in31, u31, line); - readCarryFusedLine(in61, u61, line); + readCarryFusedLine(in31, u31, line, lowMe); + readCarryFusedLine(in61, u61, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); - bar(); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack); -#else - new_fft_WIDTH1(lds31, u31, smallTrig31); - bar(); - new_fft_WIDTH1(lds61, u61, smallTrig61); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - u32 roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -1738,28 +1749,28 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -1767,48 +1778,53 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 0"); #endif + // Shuffle carries up + shufl_carries_up(lds61, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -1836,15 +1852,11 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - bar(); - - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); - - bar(); + new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); - new_fft_WIDTH2(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, line); + new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1856,14 +1868,10 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, - CP(u32) bits, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - -#if 0 // fft_WIDTH uses shufl_int instead of shufl - local GF61 lds61[WIDTH / 4]; -#else - local GF61 lds61[WIDTH / 2]; -#endif +KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, + ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { + const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup + local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; @@ -1875,7 +1883,13 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( u32 me = get_local_id(0); u32 H = BIG_HEIGHT; +#if WMUL == 1 + u32 lowMe = me; u32 line = gr % H; +#else + u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. + u32 line = (gr * WMUL + me / G_W) % H; +#endif CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1891,48 +1905,32 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line); - readCarryFusedLine(in31, u31, line); - readCarryFusedLine(in61, u61, line); + readCarryFusedLine(inF2, uF2, line, lowMe); + readCarryFusedLine(in31, u31, line, lowMe); + readCarryFusedLine(in61, u61, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. -#if ZEROHACK_W - u32 zerohack = get_group_id(0) / 131072; - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack); - bar(); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack); - bar(); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack); -#else - new_fft_WIDTH1(ldsF2, uF2, smallTrigF2); - bar(); - new_fft_WIDTH1(lds31, u31, smallTrig31); - bar(); - new_fft_WIDTH1(lds61, u61, smallTrig61); -#endif + u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; #if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); + F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[me], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights #endif + P(i64) carryShuttlePtr = (P(i64)) carryShuttle; i64 carry[NW+1]; -#if AMDGPU -#define CarryShuttleAccess(me,i) ((me) * NW + (i)) // Generates denser global_load_dwordx4 instructions -//#define CarryShuttleAccess(me,i) ((me) * 4 + (i)%4 + (i)/4 * 4*G_W) // Also generates global_load_dwordx4 instructions and unit stride when NW=8 -#else -#define CarryShuttleAccess(me,i) ((me) + (i) * G_W) // nVidia likes this unit stride better -#endif - float roundMax = 0; float carryMax = 0; - u32 word_index = (me * H + line) * 2; + u32 word_index = (lowMe * H + line) * 2; // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. @@ -2013,28 +2011,28 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( updateStats(bufROE, posROE, carryMax); #endif - // Write out our carries. Only groups 0 to H-1 need to write carries out. - // Group H is a duplicate of group 0 (producing the same results) so we don't care about group H writing out, + // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. + // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. - if (gr < H) { for (i32 i = 0; i < NW; ++i) { carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(me, i)] = carry[i]; } } + if (gr < H / WMUL && me >= (WMUL-1) * G_W) { + for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next line that its carries are ready - if (gr < H) { + // Tell next group that its carries are ready #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(); - if (me == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + bar(G_W); + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (me % WAVEFRONT == 0) { - u32 pos = gr * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (lowMe % WAVEFRONT == 0) { + u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } #endif } - // Line zero will be redone when gr == H + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } // Do some work while our carries may not be ready @@ -2050,48 +2048,53 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( uF2[i] = U2(weight1, weight2); } + // Shuffle carries up + shufl_carries_up(lds61, carry, me, lowMe); + // Wait until our carries are ready + if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); - read_mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me == 0) ready[gr - 1] = 0; + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me == 0) ready[gr - 1] = 0; #else - u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; - if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); - } - mem_fence(CLK_GLOBAL_MEM_FENCE); - // Clear carry ready flag for next iteration - if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; + if (me % WAVEFRONT == 0) { + do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + } + mem_fence(CLK_GLOBAL_MEM_FENCE); + // Clear carry ready flag for next iteration + if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; #endif #if HAS_ASM - __asm("s_setprio 1"); + __asm("s_setprio 1"); #endif - // Read from the carryShuttle carries produced by the previous WIDTH row. Rotate carries from the last WIDTH row. - // The new carry layout lets the compiler generate global_load_dwordx4 instructions. - if (gr < H) { - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]; - } - } else { + // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. + // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. + if (gr < H / WMUL) { + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + } + } else { #if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); + // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. + bar(); #endif - for (i32 i = 0; i < NW; ++i) { - carry[i] = carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]; - } + for (i32 i = 0; i < NW; ++i) { + carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + } - if (me == 0) { - carry[NW] = carry[NW-1]; - for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } - carry[0] = carry[NW]; + if (me == 0) { + carry[NW] = carry[NW-1]; + for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } + carry[0] = carry[NW]; + } } } @@ -2120,20 +2123,14 @@ KERNEL(G_W) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P( m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - bar(); - - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, line); + new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(uF2, outF2, line, lowMe); - bar(); - - new_fft_WIDTH2(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, line); - - bar(); + new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u31, out31, line, lowMe); - new_fft_WIDTH2(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, line); + new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); + writeCarryFusedLine(u61, out61, line, lowMe); } diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index c955e803..9d2fde10 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -5,6 +5,119 @@ #include "trig.cl" // #include "math.cl" + +#if FFT_FP64 | NTT_GF61 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values. Each WG uses WG * sb bytes of LDS memory. +// Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- +// even when operating on differernt sized data elements as can happen in an M31+M61 NTT. +// WG = workgroup size of a single fft_WIDTH or fft_HEIGHT +// n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT +// sb = The number of bytes to write to LDS memory at a time. SHUFL_BYTES_W or SHUFL_BYTES_H +// numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously +// lowMe = me % WG +// NOTE: shufl routines perform a bar(WG) at the start but not at the end. After calling shufl, a bar(WG) is required +// before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and +// only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively +// minor optimization would be to spedial case the first shufl call. +void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + if (sb == 16) { + local T2* lds = ((local T2*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(T2); + + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + lowMe]; } + } + + else if (sb == 8) { + // Accessing lds memory as doubles is faster than T2 accesses on Radeon VII (halving LDS memory requirements) + local T* lds = ((local T*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(T); + + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + lowMe]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + lowMe]; } + } + + else if (sb == 4) { + // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. + // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly + // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. + local int* lds = (local int*) lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(int); + + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).z; } + bar(WG); + for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).w; } + bar(WG); + for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + } +} + +#endif + + +#if FFT_FP32 | NTT_GF31 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats. +void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + //GW - would a 16 byte implementation be useful? + + if (sb >= 8) { + local F2* lds = ((local F2*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(F2); + + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + lowMe]; } + } + + else if (sb == 4) { + // Accessing lds memory as ints might be faster than F2 accesses (halving LDS memory requirements) + local F* lds = ((local F*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(F); + + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + lowMe]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + lowMe]; } + } +} + +#endif + + #if FFT_FP64 void OVERLOAD chainMul4(T2 *u, T2 w) { @@ -45,7 +158,7 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { u[3] = cmulFancy(u[3], w3); w3.x += 1; - T2 base = cmulFancy (w3, w); + T2 base = cmulFancy(w3, w); for (int i = 4; i < 8; ++i) { u[i] = cmul(u[i], base); base = cmulFancy(base, w); @@ -106,83 +219,8 @@ T2 bcast(T2 src, u32 span) { #endif -void OVERLOAD shuflBigLDS(u32 WG, local T2 *lds, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD shufl(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local T* lds = (local T*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } -} - -// Same as shufl but use ints instead of doubles to reduce LDS memory requirements. -// Lower LDS requirements should let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. -// Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly -// code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. -void OVERLOAD shufl_int(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local int* lds = (local int*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).x; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).y; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).z; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).w; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + me]; u[i] = as_double2(tmp); } - bar(); // I'm not sure why this barrier call is needed -} - -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT T2 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT T2 values). -void OVERLOAD shufl2(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - - // Partition lds memory into upper and lower halves - assert(WG == G_H); - - // Accessing lds memory as doubles is faster than T2 accesses - local T* lds = ((local T*) lds2) + (me / WG) * SMALL_HEIGHT; - - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } +void OVERLOAD shufl(u32 WG, local T2 *lds, T2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { + shufl64(WG, lds, u, n, f, numWG, sb, lowMe); } void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { @@ -209,7 +247,7 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { if (TABMUL_CHAIN) { T2 w = trig[p]; - chainMul (n, u, w, 0); + chainMul(n, u, w, 0); return; } @@ -226,8 +264,7 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { } for (u32 i = 2; i < n; ++i) { - T2 base = trig[(i-1)*WG + p]; - u[i] = cmul(u[i], base); + u[i] = cmul(u[i], trig[(i-1)*WG + p]); } return; } @@ -250,7 +287,7 @@ T2 partial_cmul(T2 u, T sine_over_cosine) { #define X2_via_FMA(a, c, b) { T2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul4_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 me) { +void preload_tabMul4_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { TrigSingle trig1 = (TrigSingle) trig; // Read 3 lines of sine/cosine values for the first fft4. Read two of the lines as a pair as AMD likes T2 global memory reads @@ -263,19 +300,20 @@ void preload_tabMul4_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 me) { } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me) { +void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { local T *lds1 = (local T *) lds; TrigSingle trig1 = (TrigSingle) trig; trig1 += 4*WG; // Skip past sine_over_cosine values // Use LDS memory to distribute preloaded trig values. if (f > 1) { + bar(WG); lds1[me] = preloads[4]; // Preloaded sine/cosine values lds1[WG+me] = preloads[5]; // Preloaded cosine values - bar(WG); } // Apply sine/cosines + bar(WG); for (u32 i = 1; i < 4; ++i) { T sine_over_cosine; if (f == 1) sine_over_cosine = preloads[i-1]; @@ -299,13 +337,11 @@ void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f preloads[2] = lds1[WG + ((me/f) & 3) * WG/4 + (2 * WG + me)/(4*f) * f/4]; preloads[3] = lds1[WG + ((me/f) & 3) * WG/4 + (3 * WG + me)/(4*f) * f/4]; preloads[1] = lds1[WG + ((me/f) & 3) * WG/4 + (1 * WG + me)/(4*f) * f/4]; - bar(WG); } } // Finish off a partial tabMul while doing next fft4 making more use of FMA. -void finish_tabMul4_fft4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me, u32 save_one_more_mul) { - local T *lds1 = (local T *) lds; +void finish_tabMul4_fft4(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingle trig1 = (TrigSingle) trig; // @@ -338,7 +374,7 @@ void finish_tabMul4_fft4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u //************************************************************************************ // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul8_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 me) { +void preload_tabMul8_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { TrigSingle trig1 = (TrigSingle) trig; // Read 7 lines of sine/cosine values for the first fft8. Read six of the lines as pairs as AMD likes T2 global memory reads @@ -353,19 +389,20 @@ void preload_tabMul8_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 me) { } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me) { +void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { local T *lds1 = (local T *) lds; TrigSingle trig1 = (TrigSingle) trig; trig1 += 8*WG; // Skip past sine_over_cosine values // Use LDS memory to distribute preloaded trig values. if (f > 1) { + bar(WG); lds1[me] = preloads[8]; // Preloaded sine/cosine values lds1[WG+me] = preloads[9]; // Preloaded cosine values - bar(WG); } // Apply sine/cosines + bar(WG); for (u32 i = 1; i < 8; ++i) { T sine_over_cosine; if (f == 1) sine_over_cosine = preloads[i-1]; @@ -394,13 +431,11 @@ void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f preloads[7] = lds1[WG + ((me/f) & 7) * WG/8 + (7 * WG + me)/(8*f) * f/8]; preloads[2] = lds1[WG + ((me/f) & 7) * WG/8 + (2 * WG + me)/(8*f) * f/8]; preloads[3] = lds1[WG + ((me/f) & 7) * WG/8 + (3 * WG + me)/(8*f) * f/8]; - bar(WG); } } // Finish off a partial tabMul while doing next fft8 making more use of FMA. -void finish_tabMul8_fft8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 me, u32 save_one_more_mul) { - local T *lds1 = (local T *) lds; +void finish_tabMul8_fft8(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingle trig1 = (TrigSingle) trig; // @@ -503,7 +538,7 @@ void OVERLOAD chainMul8(F2 *u, F2 w, u32 tailSquareBcast) { u[3] = cmulFancy(u[3], w3); w3.x += 1; - F2 base = cmulFancy (w3, w); + F2 base = cmulFancy(w3, w); for (int i = 4; i < 8; ++i) { u[i] = cmul(u[i], base); base = cmulFancy(base, w); @@ -517,55 +552,8 @@ void OVERLOAD chainMul(u32 len, F2 *u, F2 w, u32 tailSquareBcast) { if (len == 8) chainMul8(u, w, tailSquareBcast); } -void OVERLOAD shuflBigLDS(u32 WG, local F2 *lds, F2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD shufl(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f) { //GWBUG - is shufl of int2 faster (BigLDS)? - u32 me = get_local_id(0); - local F* lds = (local F*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } -} - -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT GF31 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT GF31 values). -void OVERLOAD shufl2(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - - // Partition lds memory into upper and lower halves - assert(WG == G_H); - - // Accessing lds memory as F is faster than F2 accesses //GWBUG??? - local F* lds = ((local F*) lds2) + (me / WG) * SMALL_HEIGHT; - - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } +void OVERLOAD shufl(u32 WG, local F2 *lds, F2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { + shufl32(WG, lds, u, n, f, numWG, sb, lowMe); } void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { @@ -574,7 +562,7 @@ void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN32) { - chainMul (n, u, trig[p], 0); + chainMul(n, u, trig[p], 0); return; } @@ -632,55 +620,8 @@ void OVERLOAD chainMul(u32 len, GF31 *u, GF31 w) { if (len == 8) chainMul8(u, w); } -void OVERLOAD shuflBigLDS(u32 WG, local GF31 *lds, GF31 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD shufl(u32 WG, local GF31 *lds2, GF31 *u, u32 n, u32 f) { //GWBUG - is shufl of int2 faster (BigLDS)? - u32 me = get_local_id(0); - local Z31* lds = (local Z31*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } -} - -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT GF31 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT GF31 values). -void OVERLOAD shufl2(u32 WG, local GF31 *lds2, GF31 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - - // Partition lds memory into upper and lower halves - assert(WG == G_H); - - // Accessing lds memory as Z31s is faster than GF31 accesses //GWBUG??? - local Z31* lds = ((local Z31*) lds2) + (me / WG) * SMALL_HEIGHT; - - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } +void OVERLOAD shufl(u32 WG, local GF31 *lds, GF31 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { + shufl32(WG, (local F2 *) lds, (local F2 *) u, n, f, numWG, sb, lowMe); } void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { @@ -689,7 +630,7 @@ void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN31) { - chainMul (n, u, trig[p]); + chainMul(n, u, trig[p]); return; } @@ -728,7 +669,7 @@ void OVERLOAD chainMul8(GF61 *u, GF61 w, u32 tailSquareBcast) { GF61 w2 = csq(w); u[2] = cmul(u[2], w2); - GF61 base = cmul (w2, w); //GWBUG - see FP64 version for many possible optimizations + GF61 base = cmul(w2, w); //GWBUG - see FP64 version for many possible optimizations for (int i = 3; i < 8; ++i) { u[i] = cmul(u[i], base); base = cmul(base, w); @@ -742,83 +683,8 @@ void OVERLOAD chainMul(u32 len, GF61 *u, GF61 w, u32 tailSquareBcast) { if (len == 8) chainMul8(u, w, tailSquareBcast); } -void OVERLOAD shuflBigLDS(u32 WG, local GF61 *lds, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i]; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + me]; } -} - -void OVERLOAD shufl(u32 WG, local GF61 *lds2, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local Z61* lds = (local Z61*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } -} - -// Same as shufl but use ints instead of Z61s to reduce LDS memory requirements. -// Lower LDS requirements should let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. -// Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly -// code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. -void OVERLOAD shufl_int(u32 WG, local GF61 *lds2, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - local int* lds = (local int*) lds2; - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).x; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).y; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).z; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = as_int4(u[i]).w; } - bar(); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + me]; u[i] = as_ulong2(tmp); } - bar(); // I'm not sure why this barrier call is needed -} - -// Shufl two simultaneous FFT_HEIGHTs. Needed for tailSquared where u and v are computed simultaneously in different threads. -// NOTE: It is very important for this routine to use lds memory in coordination with reverseLine2 and unreverseLine2. -// Failure to do so would result in the need for more bar() calls. Specifically, the u values are stored in the upper half -// of lds memory (first SMALL_HEIGHT GF61 values). The v values are stored in the lower half of lds memory (next SMALL_HEIGHT GF61 values). -void OVERLOAD shufl2(u32 WG, local GF61 *lds2, GF61 *u, u32 n, u32 f) { - u32 me = get_local_id(0); - - // Partition lds memory into upper and lower halves - assert(WG == G_H); - - // Accessing lds memory as Z61s is faster than GF61 accesses - local Z61* lds = ((local Z61*) lds2) + (me / WG) * SMALL_HEIGHT; - - me = me % WG; - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + me]; } - bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (me & ~mask) * n + (me & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + me]; } +void OVERLOAD shufl(u32 WG, local GF61 *lds, GF61 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { + shufl64(WG, (local T2 *) lds, (T2 *) u, n, f, numWG, sb, lowMe); } void OVERLOAD tabMul(u32 WG, TrigGF61 trig, GF61 *u, u32 n, u32 f, u32 me) { @@ -827,7 +693,7 @@ void OVERLOAD tabMul(u32 WG, TrigGF61 trig, GF61 *u, u32 n, u32 f, u32 me) { // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN61) { - chainMul (n, u, trig[p], 0); + chainMul(n, u, trig[p], 0); return; } diff --git a/src/cl/fftheight.cl b/src/cl/fftheight.cl index 69fa75b8..49323782 100644 --- a/src/cl/fftheight.cl +++ b/src/cl/fftheight.cl @@ -35,76 +35,43 @@ void OVERLOAD fft_NH(T2 *u) { #error FFT_VARIANT_H == 0 only supported by AMD GPUs #endif -void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w) { - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - w = bcast(w, s); - - chainMul(NH, u, w, 1); - - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w) { +void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { u32 WG = SMALL_HEIGHT / NH; - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(WG); } + for (u32 s = 1; s < WG; s *= NH) { fft_NH(u); w = bcast(w, s); chainMul(NH, u, w, 1); - shufl2(SMALL_HEIGHT / NH, lds, u, NH, s); + shufl(WG, lds, u, NH, s, numWG, sb, lowMe); } fft_NH(u); } #else -void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w) { - u32 me = get_local_id(0); +void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { u32 WG = SMALL_HEIGHT / NH; #if !UNROLL_H __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); + tabMul(WG, trig, u, NH, s, lowMe); + shufl(WG, lds, u, NH, s, numWG, sb, lowMe); } fft_NH(u); } #endif -void OVERLOAD new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, int callnum) { +void OVERLOAD new_fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe, int callnum) { u32 WG = SMALL_HEIGHT / NH; - u32 me = get_local_id(0); - // This line mimics shufl2 -- partition lds into halves - local T2* partitioned_lds = lds + (me / WG) * SMALL_HEIGHT / 2; - me = me % WG; + + // This line mimics shufl -- partition lds + local T2* partitioned_lds = lds; + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * SMALL_HEIGHT * sb / sizeof(T2); // Custom code for various SMALL_HEIGHT values @@ -116,27 +83,25 @@ void OVERLOAD new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, int callnum trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); // Do first fft4, partial tabMul, and shufl. fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, me); - shufl2(WG, lds, u, NH, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, me); - bar(WG); - shufl2(WG, lds, u, NH, 4); + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, me); - bar(WG); - shufl2(WG, lds, u, NH, 16); + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 16, me, 1); + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); #elif SMALL_HEIGHT == 512 && NH == 8 && FFT_VARIANT_H == 2 @@ -146,21 +111,20 @@ void OVERLOAD new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, int callnum trig += WG*8 + 2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, me); + preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); // Do first fft8, partial tabMul, and shufl. fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, me); - shufl2(WG, lds, u, NH, 1); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, partitioned_lds, trig, preloads, u, 1, me, 1); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, me); - bar(WG); - shufl2(WG, lds, u, NH, 8); + finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(WG, lds, u, NH, 8, numWG, sb, lowMe); // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, partitioned_lds, trig, preloads, u, 8, me, 1); + finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 1); #elif SMALL_HEIGHT == 1024 && NH == 4 && FFT_VARIANT_H == 2 @@ -170,44 +134,41 @@ void OVERLOAD new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, int callnum trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); // Do first fft4, partial tabMul, and shufl. fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, me); - shufl2(WG, lds, u, NH, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, me); - bar(WG); - shufl2(WG, lds, u, NH, 4); + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, me); - bar(WG); - shufl2(WG, lds, u, NH, 16); + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 16, me, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, me); - bar(WG); - shufl2(WG, lds, u, NH, 64); + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(WG, lds, u, NH, 64, numWG, sb, lowMe); // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, partitioned_lds, trig, preloads, u, 64, me, 1); + finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); #else // Old version - fft_HEIGHT2(lds, u, trig, w); + fft_HEIGHT(lds, u, trig, w, numWG, sb, lowMe); #endif } -void new_fft_HEIGHT2_1(local T2 *lds, T2 *u, Trig trig, T2 w) { new_fft_HEIGHT2(lds, u, trig, w, 1); } -void new_fft_HEIGHT2_2(local T2 *lds, T2 *u, Trig trig, T2 w) { new_fft_HEIGHT2(lds, u, trig, w, 2); } +void new_fft_HEIGHT1(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, w, numWG, sb, lowMe, 1); } +void new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, w, numWG, sb, lowMe, 2); } #endif @@ -228,41 +189,22 @@ void OVERLOAD fft_NH(F2 *u) { #endif } -void OVERLOAD fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { u32 WG = SMALL_HEIGHT / NH; #if !UNROLL_H __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); + tabMul(WG, trig, u, NH, s, lowMe); + shufl(WG, lds, u, NH, s, numWG, sb, lowMe); } fft_NH(u); } -void new_fft_HEIGHT2_1(local F2 *lds, F2 *u, TrigFP32 trig) { fft_HEIGHT2(lds, u, trig); } -void new_fft_HEIGHT2_2(local F2 *lds, F2 *u, TrigFP32 trig) { fft_HEIGHT2(lds, u, trig); } +void new_fft_HEIGHT1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } +void new_fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } #endif @@ -283,41 +225,22 @@ void OVERLOAD fft_NH(GF31 *u) { #endif } -void OVERLOAD fft_HEIGHT(local GF31 *lds, GF31 *u, TrigGF31 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local GF31 *lds, GF31 *u, TrigGF31 trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_HEIGHT(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { u32 WG = SMALL_HEIGHT / NH; #if !UNROLL_H __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); + tabMul(WG, trig, u, NH, s, lowMe); + shufl(WG, lds, u, NH, s, numWG, sb, lowMe); } fft_NH(u); } -void OVERLOAD new_fft_HEIGHT2_1(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_HEIGHT2(lds, u, trig); } -void OVERLOAD new_fft_HEIGHT2_2(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_HEIGHT2(lds, u, trig); } +void OVERLOAD new_fft_HEIGHT1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } +void OVERLOAD new_fft_HEIGHT2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } #endif @@ -338,40 +261,21 @@ void OVERLOAD fft_NH(GF61 *u) { #endif } -void OVERLOAD fft_HEIGHT(local GF61 *lds, GF61 *u, TrigGF61 trig) { - u32 me = get_local_id(0); - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - - for (u32 s = 1; s < SMALL_HEIGHT / NH; s *= NH) { - if (s > 1) { bar(); } - fft_NH(u); - tabMul(SMALL_HEIGHT / NH, trig, u, NH, s, me); - shufl(SMALL_HEIGHT / NH, lds, u, NH, s); - } - fft_NH(u); -} - -void OVERLOAD fft_HEIGHT2(local GF61 *lds, GF61 *u, TrigGF61 trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_HEIGHT(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { u32 WG = SMALL_HEIGHT / NH; #if !UNROLL_H __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WG; s *= NH) { - if (s > 1) { bar(WG); } fft_NH(u); - tabMul(WG, trig, u, NH, s, me % WG); - shufl2(WG, lds, u, NH, s); + tabMul(WG, trig, u, NH, s, lowMe); + shufl(WG, lds, u, NH, s, numWG, sb, lowMe); } fft_NH(u); } -void OVERLOAD new_fft_HEIGHT2_1(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_HEIGHT2(lds, u, trig); } -void OVERLOAD new_fft_HEIGHT2_2(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_HEIGHT2(lds, u, trig); } +void OVERLOAD new_fft_HEIGHT1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } +void OVERLOAD new_fft_HEIGHT2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } #endif diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index ae14ec53..82b3377d 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -8,11 +8,11 @@ // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT / 2]; - + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local T2 lds[lds_bytes / sizeof(T2)]; + T2 u[NH]; u32 g = get_group_id(0); - u32 me = get_local_id(0); readTailFusedLine(in, u, g, me); @@ -23,7 +23,7 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); #endif - fft_HEIGHT(lds, u, smallTrig, w); + fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); write(G_H, NH, u, out, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } @@ -39,7 +39,8 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT / 2]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local F2 lds[lds_bytes / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -57,7 +58,7 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { F2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); #endif - fft_HEIGHT(lds, u, smallTrigF2); + fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); write(G_H, NH, u, outF2, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } @@ -73,7 +74,8 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT / 2]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF31 lds[lds_bytes / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -81,12 +83,11 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 u[NH]; u32 g = get_group_id(0); - u32 me = get_local_id(0); readTailFusedLine(in31, u, g, me); - fft_HEIGHT(lds, u, smallTrig31); + fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); write(G_H, NH, u, out31, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } @@ -102,7 +103,8 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT / 2]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF61 lds[lds_bytes / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -110,12 +112,11 @@ KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 u[NH]; u32 g = get_group_id(0); - u32 me = get_local_id(0); readTailFusedLine(in61, u, g, me); - fft_HEIGHT(lds, u, smallTrig61); + fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); write(G_H, NH, u, out61, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index 2fe0d7d4..21cacf77 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -10,7 +10,7 @@ // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[WIDTH / 2]; + local T2 lds[WIDTH * SHUFL_BYTES_W / sizeof(T2)]; T2 u[NW]; u32 g = get_group_id(0); @@ -27,9 +27,9 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) u[i] = U2(in[p].x * w1, in[p].y * w2); } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -41,7 +41,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 lds[WIDTH / 2]; + local F2 lds[WIDTH * SHUFL_BYTES_W / sizeof(F2)]; F2 u[NW]; u32 g = get_group_id(0); @@ -58,9 +58,9 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ u[i] = U2(in[p].x * w1, in[p].y * w2); } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -72,7 +72,7 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { - local GF31 lds[WIDTH / 2]; + local GF31 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF31)]; GF31 u[NW]; u32 g = get_group_id(0); @@ -113,9 +113,9 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -127,7 +127,7 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { - local GF61 lds[WIDTH / 2]; + local GF61 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; GF61 u[NW]; u32 g = get_group_id(0); @@ -170,9 +170,9 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { if (weight_shift > 61) weight_shift -= 61; } - fft_WIDTH(lds, u, smallTrig); + fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); - writeCarryFusedLine(u, out, g); + writeCarryFusedLine(u, out, g, me); } @@ -184,7 +184,7 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[WIDTH / 2]; + local T2 lds[WIDTH * SHUFL_BYTES_W / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; T2 u[NW]; GF31 u31[NW]; @@ -236,11 +236,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(lds, u, smallTrig); - writeCarryFusedLine(u, out, g); - bar(); - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); + fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u, out, g, me); + + fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u31, out31, g, me); } @@ -252,7 +252,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 ldsF2[WIDTH / 2]; + local F2 ldsF2[WIDTH * SHUFL_BYTES_W / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; F2 uF2[NW]; GF31 u31[NW]; @@ -306,11 +306,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, g); - bar(); - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(uF2, outF2, g, me); + + fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u31, out31, g, me); } @@ -322,7 +322,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[WIDTH / 2]; + local GF61 lds61[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; F2 uF2[NW]; GF61 u61[NW]; @@ -376,11 +376,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG if (weight_shift > 61) weight_shift -= 61; } - fft_WIDTH(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, g); - bar(); - fft_WIDTH(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, g); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(uF2, outF2, g, me); + + fft_WIDTH(lds61, u61, smallTrig61, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u61, out61, g, me); } @@ -392,7 +392,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { - local GF61 lds61[WIDTH / 2]; + local GF61 lds61[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; GF31 u31[NW]; GF61 u61[NW]; @@ -459,11 +459,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); - bar(); - fft_WIDTH(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, g); + fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u31, out31, g, me); + + fft_WIDTH(lds61, u61, smallTrig61, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u61, out61, g, me); } @@ -475,7 +475,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[WIDTH / 2]; + local GF61 lds61[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; F2 uF2[NW]; @@ -551,14 +551,14 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - fft_WIDTH(ldsF2, uF2, smallTrigF2); - writeCarryFusedLine(uF2, outF2, g); - bar(); - fft_WIDTH(lds31, u31, smallTrig31); - writeCarryFusedLine(u31, out31, g); - bar(); - fft_WIDTH(lds61, u61, smallTrig61); - writeCarryFusedLine(u61, out61, g); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(uF2, outF2, g, me); + + fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u31, out31, g, me); + + fft_WIDTH(lds61, u61, smallTrig61, 1, SHUFL_BYTES_W, me); + writeCarryFusedLine(u61, out61, g, me); } diff --git a/src/cl/fftw.cl b/src/cl/fftw.cl index a19b26d0..89277649 100644 --- a/src/cl/fftw.cl +++ b/src/cl/fftw.cl @@ -9,13 +9,14 @@ // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[WIDTH / 2]; + local T2 lds[WIDTH * SHUFL_BYTES_W / sizeof(T2)]; T2 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); - readCarryFusedLine(in, u, g); - fft_WIDTH(lds, u, smallTrig); + readCarryFusedLine(in, u, g, me); + fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); out += WIDTH * g; write(G_W, NW, u, out, 0); } @@ -31,7 +32,7 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[WIDTH / 2]; + local F2 lds[WIDTH * SHUFL_BYTES_W / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -39,9 +40,10 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { F2 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); - readCarryFusedLine(inF2, u, g); - fft_WIDTH(lds, u, smallTrigF2); + readCarryFusedLine(inF2, u, g, me); + fft_WIDTH(lds, u, smallTrigF2, 1, SHUFL_BYTES_W, me); outF2 += WIDTH * g; write(G_W, NW, u, outF2, 0); } @@ -56,7 +58,7 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF31 KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[WIDTH / 2]; + local GF31 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -64,9 +66,10 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); - readCarryFusedLine(in31, u, g); - fft_WIDTH(lds, u, smallTrig31); + readCarryFusedLine(in31, u, g, me); + fft_WIDTH(lds, u, smallTrig31, 1, SHUFL_BYTES_W, me); out31 += WIDTH * g; write(G_W, NW, u, out31, 0); } @@ -81,7 +84,7 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF61 KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[WIDTH / 2]; + local GF61 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -89,9 +92,10 @@ KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 u[NW]; u32 g = get_group_id(0); + u32 me = get_local_id(0); - readCarryFusedLine(in61, u, g); - fft_WIDTH(lds, u, smallTrig61); + readCarryFusedLine(in61, u, g, me); + fft_WIDTH(lds, u, smallTrig61, 1, SHUFL_BYTES_W, me); out61 += WIDTH * g; write(G_W, NW, u, out61, 0); } diff --git a/src/cl/fftwidth.cl b/src/cl/fftwidth.cl index d0589ab0..03935149 100644 --- a/src/cl/fftwidth.cl +++ b/src/cl/fftwidth.cl @@ -29,39 +29,38 @@ void OVERLOAD fft_NW(T2 *u) { #error FFT_VARIANT_W == 0 only supported by AMD GPUs #endif -void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { + u32 WG = WIDTH / NW; + #if NW == 8 - T2 w = fancyTrig_N(ND / WIDTH * me); + T2 w = fancyTrig_N(ND / WIDTH * lowMe); #else - T2 w = slowTrig_N(ND / WIDTH * me, ND / NW); + T2 w = slowTrig_N(ND / WIDTH * lowMe, ND / NW); #endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } + for (u32 s = 1; s < WG; s *= NW) { fft_NW(u); w = bcast(w, s); chainMul(NW, u, w, 0); - shufl( WIDTH / NW, lds, u, NW, s); + shufl(WG, lds, u, NW, s, numWG, sb, lowMe); } fft_NW(u); } #else -void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { + u32 WG = WIDTH / NW; #if !UNROLL_W __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } + for (u32 s = 1; s < WG; s *= NW) { fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); + tabMul(WG, trig, u, NW, s, lowMe); + shufl(WG, lds, u, NW, s, numWG, sb, lowMe); } fft_NW(u); } @@ -74,9 +73,12 @@ void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig) { // To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. // The downside is sine/cosine cannot be computed with chained multiplies. -void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, int callnum) { +void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe, const u32 sb, int callnum) { u32 WG = WIDTH / NW; - u32 me = get_local_id(0); + + // This line mimics shufl -- partition lds + local T2* partitioned_lds = lds; + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * WIDTH * sb / sizeof(T2); // Custom code for various WIDTH values @@ -88,27 +90,25 @@ void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, int callnum) { trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); // Do first fft4, partial tabMul, and shufl. fft4(u); - partial_tabMul4(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 4, me); - bar(WG); - shufl(WG, lds, u, NW, 4); + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 16, me); - bar(WG); - shufl(WG, lds, u, NW, 16); + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 16, me, 1); + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); #elif WIDTH == 512 && NW == 8 && FFT_VARIANT_W == 2 @@ -118,21 +118,20 @@ void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, int callnum) { trig += WG*8; // Skip past old FFT_width trig values. // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, me); + preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); // Do first fft8, partial tabMul, and shufl. fft8(u); - partial_tabMul8(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 1, me, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, lds, trig, preloads, u, 8, me); - bar(); - shufl(WG, lds, u, NW, 8); + finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 8, me, 0); // We'd rather set save_one_more_mul to 1 + finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 #elif WIDTH == 1024 && NW == 4 && FFT_VARIANT_W == 2 @@ -142,33 +141,30 @@ void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, int callnum) { trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, me); + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); // Do first fft4, partial tabMul, and shufl. fft4(u); - partial_tabMul4(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 1, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 4, me); - bar(WG); - shufl(WG, lds, u, NW, 4); + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 4, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 16, me); - bar(WG); - shufl(WG, lds, u, NW, 16); + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 16, me, 1); - partial_tabMul4(WG, lds, trig, preloads, u, 64, me); - bar(WG); - shufl(WG, lds, u, NW, 64); + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, lds, trig, preloads, u, 64, me, 1); + finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); #elif WIDTH == 4096 && NW == 8 && FFT_VARIANT_W == 2 @@ -178,39 +174,37 @@ void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, int callnum) { trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, me); + preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); // Do first fft8, partial tabMul, and shufl. fft8(u); - partial_tabMul8(WG, lds, trig, preloads, u, 1, me); - shufl(WG, lds, u, NW, 1); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 1, me, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, lds, trig, preloads, u, 8, me); - bar(); - shufl(WG, lds, u, NW, 8); + finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 8, me, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, lds, trig, preloads, u, 64, me); - bar(); - shufl(WG, lds, u, NW, 64); + finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); // Finish third tabMul and perform final fft8. - finish_tabMul8_fft8(WG, lds, trig, preloads, u, 64, me, 0); // We'd rather set save_one_more_mul to 1 + finish_tabMul8_fft8(WG, trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 #else // Old version - fft_WIDTH(lds, u, trig); + fft_WIDTH(lds, u, trig, numWG, sb, lowMe); #endif } // There are two version of new_fft_WIDTH in case we want to try saving some trig values from new_fft_WIDTH1 in LDS memory for later use in new_fft_WIDTH2. -void OVERLOAD new_fft_WIDTH1(local T2 *lds, T2 *u, Trig trig) { new_fft_WIDTH(lds, u, trig, 1); } -void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig) { new_fft_WIDTH(lds, u, trig, 2); } +void OVERLOAD new_fft_WIDTH1(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, lowMe, sb, 1); } +void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, lowMe, sb, 2); } #endif @@ -231,23 +225,22 @@ void OVERLOAD fft_NW(F2 *u) { #endif } -void OVERLOAD fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { + u32 WG = WIDTH / NW; #if !UNROLL_W __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } + for (u32 s = 1; s < WG; s *= NW) { fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); + tabMul(WG, trig, u, NW, s, lowMe); + shufl(WG, lds, u, NW, s, numWG, sb, lowMe); } fft_NW(u); } -void OVERLOAD new_fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig) { fft_WIDTH(lds, u, trig); } -void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig) { fft_WIDTH(lds, u, trig); } +void OVERLOAD new_fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } +void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } #endif @@ -268,23 +261,22 @@ void OVERLOAD fft_NW(GF31 *u) { #endif } -void OVERLOAD fft_WIDTH(local GF31 *lds, GF31 *u, TrigGF31 trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_WIDTH(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { + u32 WG = WIDTH / NW; #if !UNROLL_W __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } + for (u32 s = 1; s < WG; s *= NW) { fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); + tabMul(WG, trig, u, NW, s, lowMe); + shufl(WG, lds, u, NW, s, numWG, sb, lowMe); } fft_NW(u); } -void OVERLOAD new_fft_WIDTH1(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_WIDTH(lds, u, trig); } -void OVERLOAD new_fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig) { fft_WIDTH(lds, u, trig); } +void OVERLOAD new_fft_WIDTH1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } +void OVERLOAD new_fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } #endif @@ -305,22 +297,21 @@ void OVERLOAD fft_NW(GF61 *u) { #endif } -void OVERLOAD fft_WIDTH(local GF61 *lds, GF61 *u, TrigGF61 trig) { - u32 me = get_local_id(0); +void OVERLOAD fft_WIDTH(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { + u32 WG = WIDTH / NW; #if !UNROLL_W __attribute__((opencl_unroll_hint(1))) #endif - for (u32 s = 1; s < WIDTH / NW; s *= NW) { - if (s > 1) { bar(); } + for (u32 s = 1; s < WG; s *= NW) { fft_NW(u); - tabMul(WIDTH / NW, trig, u, NW, s, me); - shufl(WIDTH / NW, lds, u, NW, s); + tabMul(WG, trig, u, NW, s, lowMe); + shufl(WG, lds, u, NW, s, numWG, sb, lowMe); } fft_NW(u); } -void OVERLOAD new_fft_WIDTH1(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_WIDTH(lds, u, trig); } -void OVERLOAD new_fft_WIDTH2(local GF61 *lds, GF61 *u, TrigGF61 trig) { fft_WIDTH(lds, u, trig); } +void OVERLOAD new_fft_WIDTH1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } +void OVERLOAD new_fft_WIDTH2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } #endif diff --git a/src/cl/math.cl b/src/cl/math.cl index 479238c6..5ef7d515 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -171,7 +171,7 @@ i32 optional_sub(i32 a, const i32 b) { // Optionally subtract a value if first arg is greater than value. i32 optional_mod(i32 a, const i32 b) { -#if 0 //HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (not sure why) +#if ENABLE_OPTIONAL_MOD && HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (not sure why) __asm("{.reg .pred %%p;\n\t" " setp.ge.s32 %%p, %0, %1;\n\t" // a > b " @%%p sub.s32 %0, %0, %1;}" // if (a > b) a = a - b @@ -207,7 +207,7 @@ u64 OVERLOAD mad32(u32 a, u32 b, u64 c) { } u128 OVERLOAD mad64(u64 a, u64 b, u64 c) { -#if 0 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why +#if ENABLE_MAD64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why u64 reslo, reshi; __asm("mad.lo.cc.u64 %0, %2, %3, %4;\n\t" "madc.hi.u64 %1, %2, %3, 0;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b), "l"(u128_lo64(c))); @@ -236,7 +236,7 @@ u128 OVERLOAD mad64(u64 a, u64 b, u64 c) { } u128 OVERLOAD mad64(u64 a, u64 b, u128 c) { -#if 0 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why +#if ENABLE_MAD64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why u64 reslo, reshi; __asm("mad.lo.cc.u64 %0, %2, %3, %4;\n\t" "madc.hi.u64 %1, %2, %3, %5;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b), "l"(u128_lo64(c)), "l"(u128_hi64(c))); diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 263937f3..36fcac56 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -54,13 +54,13 @@ // u[i] i ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...SMALL_HEIGHT-1 (multiples of one) -void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line) { +void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; - out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + (u32) get_local_id(0); // One pad every line + a big pad every SMALL_HEIGHT lines + out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } #else - out += line * WIDTH + (u32) get_local_id(0); + out += line * WIDTH + me; for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } #endif } @@ -311,8 +311,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) } // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { - u32 me = get_local_id(0); +void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { u32 SIZEY = OUT_WG / OUT_SIZEX; #if PAD_SIZE > 0 @@ -381,13 +380,13 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { #if FFT_FP32 || NTT_GF31 -void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line) { +void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; - out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + (u32) get_local_id(0); // One pad every line + a big pad every SMALL_HEIGHT lines + out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } #else - out += line * WIDTH + (u32) get_local_id(0); + out += line * WIDTH + me; for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } #endif } @@ -537,8 +536,7 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) #endif } -void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { - u32 me = get_local_id(0); +void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 SIZEY = OUT_WG / OUT_SIZEX; #if PAD_SIZE > 0 // Adjust in pointer based on the x value used in writeMiddleOutLine @@ -595,8 +593,8 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { // Since F2 and GF31 are the same size we can simply call the floats based code -void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line) { - writeCarryFusedLine((F2 *) u, (P(F2)) out, line); +void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { + writeCarryFusedLine((F2 *) u, (P(F2)) out, line, me); } void OVERLOAD readMiddleInLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { @@ -623,8 +621,8 @@ void OVERLOAD writeMiddleOutLine (P(GF31) out, GF31 *u, u32 chunk_y, u32 chunk_x writeMiddleOutLine ((P(F2)) out, (F2 *) u, chunk_y, chunk_x); } -void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line) { - readCarryFusedLine((CP(F2)) in, (F2 *) u, line); +void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { + readCarryFusedLine((CP(F2)) in, (F2 *) u, line, me); } #endif @@ -638,8 +636,8 @@ void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line) { // Since T2 and GF61 are the same size we can simply call the doubles based code -void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line) { - writeCarryFusedLine((T2 *) u, (P(T2)) out, line); +void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { + writeCarryFusedLine((T2 *) u, (P(T2)) out, line, me); } void OVERLOAD readMiddleInLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { @@ -666,8 +664,8 @@ void OVERLOAD writeMiddleOutLine (P(GF61) out, GF61 *u, u32 chunk_y, u32 chunk_x writeMiddleOutLine ((P(T2)) out, (T2 *) u, chunk_y, chunk_x); } -void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { - readCarryFusedLine((CP(T2)) in, (T2 *) u, line); +void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { + readCarryFusedLine((CP(T2)) in, (T2 *) u, line, me); } #endif @@ -778,8 +776,7 @@ void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { // line ranges 0...BIG_HEIGHT-1 (multiples of one) // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); @@ -787,8 +784,7 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line) { } // Write a line from carryFused. This data will be read by fftMiddleIn. -void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); @@ -899,8 +895,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) // line ranges 0...BIG_HEIGHT-1 (multiples of one) // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); @@ -908,8 +903,7 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line) { } // Write a line from carryFused. This data will be read by fftMiddleIn. -void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line) { - u32 me = get_local_id(0); // Multiples of BIG_HEIGHT +void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); @@ -990,12 +984,12 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 y, u32 x) // Since F2 and GF31 are the same size we can simply call the floats based code -void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line) { - readCarryFusedLine((CP(F2)) in, (F2 *) u, line); +void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { + readCarryFusedLine((CP(F2)) in, (F2 *) u, line, me); } -void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line) { - writeCarryFusedLine((F2 *) u, (P(F2)) out, line); +void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { + writeCarryFusedLine((F2 *) u, (P(F2)) out, line, me); } void OVERLOAD readMiddleInLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { @@ -1033,12 +1027,12 @@ void OVERLOAD writeMiddleOutLine (P(GF31) out, GF31 *u, u32 y, u32 x) { // Since T2 and GF61 are the same size we can simply call the doubles based code -void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line) { - readCarryFusedLine((CP(T2)) in, (T2 *) u, line); +void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { + readCarryFusedLine((CP(T2)) in, (T2 *) u, line, me); } -void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line) { - writeCarryFusedLine((T2 *) u, (P(T2)) out, line); +void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { + writeCarryFusedLine((T2 *) u, (P(T2)) out, line, me); } void OVERLOAD readMiddleInLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 1cdd5db0..3e2299f8 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -49,7 +49,8 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local T2 lds[lds_bytes / sizeof(T2)]; T2 u[NH], v[NH]; T2 p[NH], q[NH]; @@ -65,7 +66,9 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { readTailFusedLine(in, u, line1, me); readTailFusedLine(in, v, line2, me); -#if NH == 8 +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); #else T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); @@ -74,19 +77,15 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig, w); - bar(); - fft_HEIGHT(lds, v, smallTrig, w); + fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); #else readTailFusedLine(a, p, line1, me); readTailFusedLine(a, q, line2, me); - fft_HEIGHT(lds, u, smallTrig, w); - bar(); - fft_HEIGHT(lds, v, smallTrig, w); - bar(); - fft_HEIGHT(lds, p, smallTrig, w); - bar(); - fft_HEIGHT(lds, q, smallTrig, w); + fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, p, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, q, smallTrig, w, 1, SHUFL_BYTES_H, me); #endif T2 trig = slowTrig_N(line1 + me * H, ND / NH); @@ -109,10 +108,8 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig, w); - bar(); - fft_HEIGHT(lds, u, smallTrig, w); + fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out, memline2, me); writeTailFusedLine(u, out, memline1, me); } @@ -164,7 +161,8 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local F2 lds[lds_bytes / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; CP(F2) aF2 = (CP(F2)) a; @@ -188,19 +186,15 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, aF2, memline1 * SMALL_HEIGHT); read(G_H, NH, q, aF2, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrigF2); - bar(); - fft_HEIGHT(lds, v, smallTrigF2); + fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); #else readTailFusedLine(aF2, p, line1, me); readTailFusedLine(aF2, q, line2, me); - fft_HEIGHT(lds, u, smallTrigF2); - bar(); - fft_HEIGHT(lds, v, smallTrigF2); - bar(); - fft_HEIGHT(lds, p, smallTrigF2); - bar(); - fft_HEIGHT(lds, q, smallTrigF2); + fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, p, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, q, smallTrigF2, 1, SHUFL_BYTES_H, me); #endif F2 trig = slowTrig_N(line1 + me * H, ND / NH); @@ -223,10 +217,8 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrigF2); - bar(); - fft_HEIGHT(lds, u, smallTrigF2); + fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, outF2, memline2, me); writeTailFusedLine(u, outF2, memline1, me); } @@ -278,7 +270,8 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar } KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF31 lds[lds_bytes / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); @@ -302,19 +295,15 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a31, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a31, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig31); - bar(); - fft_HEIGHT(lds, v, smallTrig31); + fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); #else readTailFusedLine(a31, p, line1, me); readTailFusedLine(a31, q, line2, me); - fft_HEIGHT(lds, u, smallTrig31); - bar(); - fft_HEIGHT(lds, v, smallTrig31); - bar(); - fft_HEIGHT(lds, p, smallTrig31); - bar(); - fft_HEIGHT(lds, q, smallTrig31); + fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, p, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, q, smallTrig31, 1, SHUFL_BYTES_H, me); #endif // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) @@ -354,10 +343,8 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig31); - bar(); - fft_HEIGHT(lds, u, smallTrig31); + fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out31, memline2, me); writeTailFusedLine(u, out31, memline1, me); } @@ -407,7 +394,8 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar } KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF61 lds[lds_bytes / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); @@ -431,19 +419,15 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a61, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a61, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig61); - bar(); - fft_HEIGHT(lds, v, smallTrig61); + fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); #else readTailFusedLine(a61, p, line1, me); readTailFusedLine(a61, q, line2, me); - fft_HEIGHT(lds, u, smallTrig61); - bar(); - fft_HEIGHT(lds, v, smallTrig61); - bar(); - fft_HEIGHT(lds, p, smallTrig61); - bar(); - fft_HEIGHT(lds, q, smallTrig61); + fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, p, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, q, smallTrig61, 1, SHUFL_BYTES_H, me); #endif // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) @@ -483,10 +467,8 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig61); - bar(); - fft_HEIGHT(lds, u, smallTrig61); + fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out61, memline2, me); writeTailFusedLine(u, out61, memline1, me); } diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index bf960f1c..d9edc1f7 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -54,7 +54,8 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT / 2]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local T2 lds[lds_bytes / sizeof(T2)]; T2 u[NH]; u32 H = ND / SMALL_HEIGHT; @@ -66,7 +67,9 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { u32 me = get_local_id(0); readTailFusedLine(in, u, line, me); -#if NH == 8 +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); #else T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); @@ -74,20 +77,20 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { T2 trig = slowTrig_N(line + me * H, ND / NH); - fft_HEIGHT(lds, u, smallTrig, w); + fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrig, w); + fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), me); } #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local T2 lds[lds_bytes / sizeof(T2)]; T2 u[NH], v[NH]; @@ -107,22 +110,17 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in, u, line1, me); readTailFusedLine(in, v, line2, me); -#if NH == 8 +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); #else T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); #endif -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig + zerohack, w); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrig + zerohack, w); -#else - fft_HEIGHT(lds, u, smallTrig, w); - bar(); - fft_HEIGHT(lds, v, smallTrig, w); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT(lds + zerohack, u, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds + zerohack, v, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS == 2 @@ -169,10 +167,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig, w); - bar(); - fft_HEIGHT(lds, u, smallTrig, w); + fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out, memline2, me); writeTailFusedLine(u, out, memline1, me); @@ -202,7 +198,8 @@ void OVERLOAD pairSq2_special(T2 *u, T2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local T2 lds[lds_bytes * 2 / sizeof(T2)]; T2 u[NH]; @@ -227,18 +224,16 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // Read lines u and v readTailFusedLine(in, u, line, lowMe); -#if NH == 8 +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 T2 w = fancyTrig_N(H * lowMe); #else T2 w = slowTrig_N(H * lowMe, ND / NH); #endif -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrig + zerohack, w); -#else - new_fft_HEIGHT2_1(lds, u, smallTrig, w); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + new_fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 2, SHUFL_BYTES_H, lowMe); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS == 2 @@ -263,8 +258,6 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { T2 trig = NTLOAD(smallTrig[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -276,15 +269,12 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); - - new_fft_HEIGHT2_2(lds, u, smallTrig, w); + new_fft_HEIGHT2(lds, u, smallTrig, w, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), lowMe); @@ -342,7 +332,8 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT / 2]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local F2 lds[lds_bytes / sizeof(F2)]; F2 u[NH]; u32 H = ND / SMALL_HEIGHT; @@ -360,20 +351,20 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { F2 trig = slowTrig_N(line + me * H, ND / NH); - fft_HEIGHT(lds, u, smallTrigF2); + fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrigF2); + fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), me); } #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local F2 lds[lds_bytes / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -397,16 +388,9 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(inF2, u, line1, me); readTailFusedLine(inF2, v, line2, me); -#if ZEROHACK_H - u32 zerohack = get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrigF2 + zerohack); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrigF2 + zerohack); -#else - fft_HEIGHT(lds, u, smallTrigF2); - bar(); - fft_HEIGHT(lds, v, smallTrigF2); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT(lds + zerohack, u, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds + zerohack, v, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS32 == 2 @@ -453,10 +437,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrigF2); - bar(); - fft_HEIGHT(lds, u, smallTrigF2); + fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, outF2, memline2, me); writeTailFusedLine(u, outF2, memline1, me); @@ -486,7 +468,8 @@ void OVERLOAD pairSq2_special(F2 *u, F2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local F2 lds[lds_bytes * 2 / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -515,12 +498,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // Read lines u and v readTailFusedLine(inF2, u, line, lowMe); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrigF2 + zerohack); -#else - new_fft_HEIGHT2_1(lds, u, smallTrigF2); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + new_fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 2, SHUFL_BYTES_H, lowMe); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS32 == 2 @@ -545,8 +524,6 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { F2 trig = NTLOAD(smallTrigF2[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -558,15 +535,12 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); - - new_fft_HEIGHT2_2(lds, u, smallTrigF2); + new_fft_HEIGHT2(lds, u, smallTrigF2, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), lowMe); @@ -627,7 +601,8 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT / 2]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF31 lds[lds_bytes / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -663,19 +638,20 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - fft_HEIGHT(lds, u, smallTrig31); + fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrig31); + + fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), me); } #if SINGLE_WIDE KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF31 lds[lds_bytes / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -699,16 +675,9 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in31, u, line1, me); readTailFusedLine(in31, v, line2, me); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig31 + zerohack); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrig31 + zerohack); -#else - fft_HEIGHT(lds, u, smallTrig31); - bar(); - fft_HEIGHT(lds, v, smallTrig31); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT(lds + zerohack, u, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds + zerohack, v, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS31 >= 1 @@ -751,10 +720,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig31); - bar(); - fft_HEIGHT(lds, u, smallTrig31); + fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out31, memline2, me); writeTailFusedLine(u, out31, memline1, me); @@ -783,7 +750,8 @@ void OVERLOAD pairSq2_special(GF31 *u, GF31 base_squared) { } KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF31 lds[lds_bytes * 2 / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -812,12 +780,8 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // Read lines u and v readTailFusedLine(in31, u, line, lowMe); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrig31 + zerohack); -#else - new_fft_HEIGHT2_1(lds, u, smallTrig31); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + new_fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 2, SHUFL_BYTES_H, lowMe); // Do a little bit of memory access and a little bit of math. Good on a Radeon VII. #if TAIL_TRIGS31 >= 1 @@ -838,8 +802,6 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 trig = NTLOAD(smallTrig31[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -851,15 +813,12 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); - - new_fft_HEIGHT2_2(lds, u, smallTrig31); + new_fft_HEIGHT2(lds, u, smallTrig31, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), lowMe); @@ -920,7 +879,8 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT / 2]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF61 lds[lds_bytes / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -956,19 +916,20 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - fft_HEIGHT(lds, u, smallTrig61); + fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - bar(); - fft_HEIGHT(lds, u, smallTrig61); + + fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), me); } #if SINGLE_WIDE KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF61 lds[lds_bytes / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -992,16 +953,9 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in61, u, line1, me); readTailFusedLine(in61, v, line2, me); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig61 + zerohack); - bar(); - fft_HEIGHT(lds + zerohack, v, smallTrig61 + zerohack); -#else - fft_HEIGHT(lds, u, smallTrig61); - bar(); - fft_HEIGHT(lds, v, smallTrig61); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + fft_HEIGHT(lds + zerohack, u, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds + zerohack, v, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS61 >= 1 @@ -1044,10 +998,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } - bar(); - fft_HEIGHT(lds, v, smallTrig61); - bar(); - fft_HEIGHT(lds, u, smallTrig61); + fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out61, memline2, me); writeTailFusedLine(u, out61, memline1, me); @@ -1076,7 +1028,8 @@ void OVERLOAD pairSq2_special(GF61 *u, GF61 base_squared) { } KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[SMALL_HEIGHT]; + const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; + local GF61 lds[lds_bytes * 2 / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -1105,12 +1058,8 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // Read lines u and v readTailFusedLine(in61, u, line, lowMe); -#if ZEROHACK_H - u32 zerohack = (u32) get_group_id(0) / 131072; - new_fft_HEIGHT2_1(lds + zerohack, u, smallTrig61 + zerohack); -#else - new_fft_HEIGHT2_1(lds, u, smallTrig61); -#endif + u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; + new_fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 2, SHUFL_BYTES_H, lowMe); // Do a little bit of memory access and a little bit of math. Good on a Radeon VII. #if TAIL_TRIGS61 >= 1 @@ -1131,8 +1080,6 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 trig = NTLOAD(smallTrig61[height_trigs + line_u*G_H*2 + me]); #endif - bar(G_H); - #if SINGLE_KERNEL // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. if (line_u == 0) { @@ -1144,15 +1091,12 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #else if (1) { #endif - revCrossLine(G_H, lds, u + NH/2, NH/2, isSecondHalf); + revCrossLine(lds, u); pairSq(NH/2, u, u + NH/2, trig, false); - bar(G_H); - revCrossLine(G_H, lds, u + NH/2, NH/2, !isSecondHalf); + revCrossLine(lds, u); } - bar(G_H); - - new_fft_HEIGHT2_2(lds, u, smallTrig61); + new_fft_HEIGHT2(lds, u, smallTrig61, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), lowMe); diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index 01710cf1..9234f858 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -30,84 +30,181 @@ #define SINGLE_WIDE TAIL_KERNELS < 2 // Old single-wide tailSquare vs. new double-wide tailSquare #define SINGLE_KERNEL (TAIL_KERNELS & 1) == 0 // TailSquare uses a single kernel vs. two kernels -#if FFT_FP64 +// 64-bit implementations of reverse routines -void OVERLOAD reverse(u32 WG, local T2 *lds, T2 *u, bool bump) { +#if FFT_FP64 | NTT_GF61 + +void OVERLOAD reverse(u32 WG, local T2 *lds2, T2 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; - bar(); - + if (SHUFL_BYTES_H >= 8) { + local T2 *lds = lds2; + bar(WG); #if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; + lds[revMe + 0 * WG] = u[3]; + lds[revMe + 1 * WG] = u[2]; + lds[revMe + 2 * WG] = u[1]; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; #elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error + lds[revMe + 0 * WG] = u[1]; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; #endif + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + } - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + else if (SHUFL_BYTES_H == 4) { + local T *lds = (local T *) lds2; + bar(WG); +#if NH == 8 + lds[revMe + 0 * WG] = u[3].x; + lds[revMe + 1 * WG] = u[2].x; + lds[revMe + 2 * WG] = u[1].x; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0].x; +#elif NH == 4 + lds[revMe + 0 * WG] = u[1].x; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0].x; +#endif + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i].x = lds[i * WG + me]; } + bar(WG); +#if NH == 8 + lds[revMe + 0 * WG] = u[3].y; + lds[revMe + 1 * WG] = u[2].y; + lds[revMe + 2 * WG] = u[1].y; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0].y; +#elif NH == 4 + lds[revMe + 0 * WG] = u[1].y; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0].y; +#endif + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i].y = lds[i * WG + me]; } + } } -void OVERLOAD reverseLine(u32 WG, local T2 *lds2, T2 *u) { +void OVERLOAD reverseLine(u32 WG, local T2 *lds, T2 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; - local T2 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } - - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } -} - -// This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local T2* lds2, T2 *u, u32 n, bool writeSecondHalf) { - u32 me = get_local_id(0); - u32 lowMe = me % WG; - - u32 revLowMe = WG - 1 - lowMe; - - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } + if (SHUFL_BYTES_H == 16) { + local T2 *ldsOut = lds + revMe; + local T2 *ldsIn = lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i] = ldsIn[WG * i]; } + } - bar(); // we need a full bar because we're crossing halves + else if (SHUFL_BYTES_H == 8) { + local T *ldsOut = (local T *) lds + revMe; + local T *ldsIn = (local T *) lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[WG * i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].y = ldsIn[WG * i]; } + } - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } + else if (SHUFL_BYTES_H == 4) { + local int *ldsOut = (local int *) lds + revMe; + local int *ldsIn = (local int *) lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).x; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.x = ldsIn[WG * i]; u[i] = as_double2(tmp); } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).y; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.y = ldsIn[WG * i]; u[i] = as_double2(tmp); } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).z; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.z = ldsIn[WG * i]; u[i] = as_double2(tmp); } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).w; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.w = ldsIn[WG * i]; u[i] = as_double2(tmp); } + } } // -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// These versions are for the kernel(s) that use a double-wide workgroup (u in half the workgroup, v in the other half) // -void OVERLOAD reverse2(local T2 *lds, T2 *u) { +void OVERLOAD reverse2(local T2 *lds2, T2 *u) { u32 me = get_local_id(0); - - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; + u32 lowMe = me % G_H; + + if (SHUFL_BYTES_H >= 8) { + local T2 *lds = lds2; + if (me >= G_H) lds += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T2); + // For NH=8, u[0] to u[3] are left unchanged. Write to lds: + // u[7]rev u[6]rev u[5]rev u[4]rev + // v[7]rev v[6]rev v[5]rev v[4]rev + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i]; } + // For NH=8, read from lds into u[i]: + // u[4] = u[7]rev v[7]rev + // u[5] = u[6]rev v[6]rev + // u[6] = u[5]rev v[5]rev + // u[7] = u[4]rev v[4]rev + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * G_H + lowMe]; } + } + + else if (SHUFL_BYTES_H == 4) { + local T *lds = (local T *) lds2; + if (me >= G_H) lds += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T); + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i].x; } + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].x = lds[i * G_H + lowMe]; } + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i].y; } + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].y = lds[i * G_H + lowMe]; } } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } } +// This is used to reverse the second part of a line, and cross the reversed parts between the halves. +void OVERLOAD revCrossLine(local T2* lds2, T2 *u) { + u32 me = get_local_id(0); + u32 lowMe = me % G_H; + u32 revLowMe = G_H - 1 - lowMe; + + if (SHUFL_BYTES_H >= 8) { + local T2 *ldsOut = lds2; + local T2 *ldsIn = lds2; + if (me < G_H) ldsOut += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T2); // Crossing LDS halves + else ldsIn += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T2); // Staying within LDS halves (just like shufl) + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } + bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[G_H * i + lowMe]; } + } + + else if (SHUFL_BYTES_H == 4) { + local T *ldsOut = (local T *) lds2; + local T *ldsIn = (local T *) lds2; + if (me < G_H) ldsOut += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T); + else ldsIn += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T); + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].x; } + bar(); // we need a full bar because we just crossed halves + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].x = ldsIn[G_H * i + lowMe]; } + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].y; } + bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].y = ldsIn[G_H * i + lowMe]; } + } +} + +#if 0 // Unused + // Somewhat similar to reverseLine. // The u values are in threads < G_H, the v values to reverse in threads >= G_H. // Whereas reverseLine leaves u values alone. This reverseLine moves u values around @@ -119,7 +216,7 @@ void OVERLOAD reverse2(local T2 *lds, T2 *u) { void OVERLOAD reverseLine2(local T2 *lds, T2 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an +// NOTE: It is important that this routine use lds memory in coordination with shufl. Failure to do so would require an // unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT T2 values). // The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT T2 values). @@ -191,89 +288,107 @@ void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { #endif +#endif + /**************************************************************************/ /* Similar to above, but for an FFT based on FP32 */ /**************************************************************************/ -#if FFT_FP32 +#if FFT_FP32 | NTT_GF31 void OVERLOAD reverse(u32 WG, local F2 *lds, F2 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; - bar(); - + if (SHUFL_BYTES_H >= 4) { + bar(WG); #if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; + lds[revMe + 0 * WG] = u[3]; + lds[revMe + 1 * WG] = u[2]; + lds[revMe + 2 * WG] = u[1]; + lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; #elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error + lds[revMe + 0 * WG] = u[1]; + lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; #endif - - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + bar(WG); + for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + } } -void OVERLOAD reverseLine(u32 WG, local F2 *lds2, F2 *u) { +void OVERLOAD reverseLine(u32 WG, local F2 *lds, F2 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; - local F2 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } - - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } -} - -// This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local F2* lds2, F2 *u, u32 n, bool writeSecondHalf) { - u32 me = get_local_id(0); - u32 lowMe = me % WG; - - u32 revLowMe = WG - 1 - lowMe; - - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } - - bar(); // we need a full bar because we're crossing halves + if (SHUFL_BYTES_H >= 8) { + local F2 *ldsOut = lds + revMe; + local F2 *ldsIn = lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i] = ldsIn[WG * i]; } + } - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } + else if (SHUFL_BYTES_H == 4) { + local F *ldsOut = (local F *) lds + revMe; + local F *ldsIn = (local F *) lds + me; + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[WG * i]; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < NH; ++i) { u[i].y = ldsIn[WG * i]; } + } } // -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// These versions are for the kernel(s) that use a double-wide workgroup (u in half the workgroup, v in the other half) // void OVERLOAD reverse2(local F2 *lds, F2 *u) { u32 me = get_local_id(0); + u32 lowMe = me % G_H; + + if (SHUFL_BYTES_H >= 4) { + if (me >= G_H) lds += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(F2); + // For NH=8, u[0] to u[3] are left unchanged. Write to lds: + // u[7]rev u[6]rev u[5]rev u[4]rev + // v[7]rev v[6]rev v[5]rev v[4]rev + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i]; } + // For NH=8, read from lds into u[i]: + // u[4] = u[7]rev v[7]rev + // u[5] = u[6]rev v[6]rev + // u[6] = u[5]rev v[5]rev + // u[7] = u[4]rev v[4]rev + bar(G_H); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * G_H + lowMe]; } + } +} - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; +// This is used to reverse the second part of a line, and cross the reversed parts between the halves. +void OVERLOAD revCrossLine(local F2* lds2, F2 *u) { + u32 me = get_local_id(0); + u32 lowMe = me % G_H; + u32 revLowMe = G_H - 1 - lowMe; + + if (SHUFL_BYTES_H >= 4) { + local F2 *ldsOut = lds2; + local F2 *ldsIn = lds2; + if (me < G_H) ldsOut += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(F2); + else ldsIn += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(F2); + bar(); // we need a full bar because we're crossing halves + for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } + bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[G_H * i + lowMe]; } } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } } +#if 0 // Unused + // Somewhat similar to reverseLine. // The u values are in threads < G_H, the v values to reverse in threads >= G_H. // Whereas reverseLine leaves u values alone. This reverseLine moves u values around @@ -357,6 +472,8 @@ void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { #endif +#endif + /**************************************************************************/ /* Similar to above, but for an NTT based on GF(M31^2) */ @@ -365,161 +482,32 @@ void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { #if NTT_GF31 void OVERLOAD reverse(u32 WG, local GF31 *lds, GF31 *u, bool bump) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me + bump; - - bar(); - -#if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; -#elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error -#endif - - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + reverse(WG, (local F2 *) lds, (F2 *) u, bump); } -void OVERLOAD reverseLine(u32 WG, local GF31 *lds2, GF31 *u) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me; - - local GF31 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } - - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } +void OVERLOAD reverseLine(u32 WG, local GF31 *lds, GF31 *u) { + reverseLine(WG, (local F2 *) lds, (F2 *) u); } -// This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local GF31* lds2, GF31 *u, u32 n, bool writeSecondHalf) { - u32 me = get_local_id(0); - u32 lowMe = me % WG; - - u32 revLowMe = WG - 1 - lowMe; - - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } - - bar(); // we need a full bar because we're crossing halves - - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } -} - -// -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) -// - void OVERLOAD reverse2(local GF31 *lds, GF31 *u) { - u32 me = get_local_id(0); - - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; - } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } + reverse2((local F2 *) lds, (F2 *) u); } -// Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. -// Whereas reverseLine leaves u values alone. This reverseLine moves u values around -// so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: -// u[0] u[1] // Returned in u[0] -// u[2] u[3] // Returned in u[1] -// v[3]rev v[2]rev // Returned in u[2] -// v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local GF31 *lds, GF31 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an -// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT GF31 values). -// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT GF31 values). - - if (G_H > WAVEFRONT) bar(); - -// For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H - -#if 1 - local GF31 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } +void OVERLOAD revCrossLine(local GF31* lds, GF31 *u) { + revCrossLine((local F2 *) lds, (F2 *) u); +} - lds += me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } -#else - local Z61 *ldsOut = (local Z61 *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } +#if 0 // Unused - local ZF61 *ldsIn = (local T *) lds + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } -#endif +void OVERLOAD reverseLine2(local GF31 *lds, GF31 *u) { + reverseLine2((local F2 *) lds, (F2 *) u); } -// Undo a reverseLine2 void OVERLOAD unreverseLine2(local GF31 *lds, GF31 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially -// writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of -// lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. - -#if 1 - local GF31 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } - -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } -#else - local Z61 *ldsOut = (local T *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } + unreverseLine2((local F2 *) lds, (F2 *) u); +} -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local Z61 *ldsIn = (local T *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } #endif -} #endif @@ -531,160 +519,31 @@ void OVERLOAD unreverseLine2(local GF31 *lds, GF31 *u) { #if NTT_GF61 void OVERLOAD reverse(u32 WG, local GF61 *lds, GF61 *u, bool bump) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me + bump; - - bar(); - -#if NH == 8 - lds[revMe + 0 * WG] = u[3]; - lds[revMe + 1 * WG] = u[2]; - lds[revMe + 2 * WG] = u[1]; - lds[bump ? ((revMe + 3 * WG) % (4 * WG)) : (revMe + 3 * WG)] = u[0]; -#elif NH == 4 - lds[revMe + 0 * WG] = u[1]; - lds[bump ? ((revMe + WG) % (2 * WG)) : (revMe + WG)] = u[0]; -#else -#error -#endif - - bar(); - for (i32 i = 0; i < NH/2; ++i) { u[i] = lds[i * WG + me]; } + reverse(WG, (local T2 *) lds, (T2 *) u, bump); } -void OVERLOAD reverseLine(u32 WG, local GF61 *lds2, GF61 *u) { - u32 me = get_local_id(0); - u32 revMe = WG - 1 - me; - - local GF61 *lds = lds2 + revMe; - bar(); - for (u32 i = 0; i < NH; ++i) { lds[WG * (NH - 1 - i)] = u[i]; } - - lds = lds2 + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[WG * i]; } +void OVERLOAD reverseLine(u32 WG, local GF61 *lds, GF61 *u) { + reverseLine(WG, (local T2 *) lds, (T2 *) u); } -// This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(u32 WG, local GF61* lds2, GF61 *u, u32 n, bool writeSecondHalf) { - u32 me = get_local_id(0); - u32 lowMe = me % WG; - - u32 revLowMe = WG - 1 - lowMe; - - for (u32 i = 0; i < n; ++i) { lds2[WG * n * writeSecondHalf + WG * (n - 1 - i) + revLowMe] = u[i]; } - - bar(); // we need a full bar because we're crossing halves - - for (u32 i = 0; i < n; ++i) { u[i] = lds2[WG * n * !writeSecondHalf + WG * i + lowMe]; } -} - -// -// These versions are for the kernel(s) that uses a double-wide workgroup (u in half the workgroup, v in the other half) -// - void OVERLOAD reverse2(local GF61 *lds, GF61 *u) { - u32 me = get_local_id(0); - - // For NH=8, u[0] to u[3] are left unchanged. Write to lds: - // u[7]rev u[6]rev - // u[5]rev u[4]rev - // v[7]rev v[6]rev - // v[5]rev v[4]rev - bar(); - for (u32 i = 0; i < NH / 2; ++i) { - u32 j = (i * G_H + me % G_H); - lds[me < G_H ? ((NH/2)*G_H - j) % ((NH/2)*G_H) : NH*G_H-1 - j] = u[NH/2 + i]; - } - // For NH=8, read from lds into u[i]: - // u[4] = u[7]rev v[7]rev - // u[5] = u[6]rev v[6]rev - // u[6] = u[5]rev v[5]rev - // u[7] = u[4]rev v[4]rev - bar(); - lds += me % G_H + (me / G_H) * NH/2 * G_H; - for (u32 i = 0; i < NH / 2; ++i) { u[NH/2 + i] = lds[i * G_H]; } + reverse2((local T2 *) lds, (T2 *) u); } -// Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. -// Whereas reverseLine leaves u values alone. This reverseLine moves u values around -// so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: -// u[0] u[1] // Returned in u[0] -// u[2] u[3] // Returned in u[1] -// v[3]rev v[2]rev // Returned in u[2] -// v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local GF61 *lds, GF61 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an -// unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT GF61 values). -// The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT GF61 values). - - if (G_H > WAVEFRONT) bar(); - -// For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H - -#if 1 - local GF61 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } +void OVERLOAD revCrossLine(local GF61* lds, GF61 *u) { + revCrossLine((local T2 *) lds, (T2 *) u); +} - lds += me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } -#else - local Z61 *ldsOut = (local Z61 *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } +#if 0 // Unused - local ZF61 *ldsIn = (local T *) lds + me; - bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } -#endif +void OVERLOAD reverseLine2(local GF61 *lds, GF61 *u) { + reverseLine2((local T2 *) lds, (T2 *) u); } -// Undo a reverseLine2 void OVERLOAD unreverseLine2(local GF61 *lds, GF61 *u) { - u32 me = get_local_id(0); - -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially -// writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of -// lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. - -#if 1 - local GF61 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } - -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } -#else - local Z61 *ldsOut = (local T *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } + unreverseLine2((local T2 *) lds, (T2 *) u); +} -// For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local Z61 *ldsIn = (local T *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; - bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } #endif -} #endif diff --git a/src/tune.cpp b/src/tune.cpp index 4d613d0f..ae376054 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -911,7 +911,7 @@ void Tune::tune() { } // Find best BIGLIT setting - if (time_FFTs) { + if (0 && time_FFTs) { // Deprecated FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_biglit = 0; @@ -954,7 +954,7 @@ void Tune::tune() { config.write("\n -log 1000000\n"); } if (args->workers < 2) { - config.write("\n# Running two workers sometimes gives better throughput. Autoprimenet will need to create up a second worktodo file."); + config.write("\n# Running two workers sometimes gives better throughput. AutoPrimenet will need to create a second worktodo file."); config.write("\n# -workers 2\n"); config.write("\n# Changing TAIL_KERNELS to 3 when running two workers may be better."); config.write("\n# -use TAIL_KERNELS=3\n"); From 08036f356c6b6a0fa8c28d37880426ed649e939c Mon Sep 17 00:00:00 2001 From: george Date: Thu, 12 Mar 2026 02:00:30 +0000 Subject: [PATCH 009/214] Testing indicates WMUL=2 should dbe the default. RTX4xxx and RTX5xxx GPUs benefit from L2STORE and LULOAD. Added support for those options officially. Since FAST_BARRIER seems to now work on nVidia, the option is now tuned. --- src/Gpu.cpp | 4 +++- src/cl/base.cl | 8 +++++++ src/tune.cpp | 63 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 84cc4434..6d00c0fd 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -228,7 +228,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Default value for -use options that must also be parsed in C++ code tail_single_wide = 0, tail_single_kernel = 1; // Default tailSquare is double-wide in one kernel in_place = 0; // Default is not in-place - wmul = 1; // Default is carryFused processes one workgroup at a time + wmul = 2; // Default is carryFused processes two lines at a time pad_size = isAmdGpu(id) ? 256 : 0; // Default is 256 bytes for AMD, 0 for others // Validate -use options @@ -263,6 +263,8 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "TABMUL_CHAIN32", "TABMUL_CHAIN61", "MODM31", + "ENABLE_L2STORE", + "ENABLE_LULOAD", "WMUL" }); if (!isValid) { diff --git a/src/cl/base.cl b/src/cl/base.cl index f252cfe1..1765fdfe 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -189,6 +189,14 @@ G_H "group height" == SMALL_HEIGHT / NH #define ZEROHACK_H 1 #endif +#if !defined(ENABLE_L2STORE) +#define ENABLE_L2STORE 1 +#endif + +#if !defined(ENABLE_LULOAD) +#define ENABLE_LULOAD 1 +#endif + // Expected defines: EXP the exponent. // WIDTH, SMALL_HEIGHT, MIDDLE. diff --git a/src/tune.cpp b/src/tune.cpp index ae376054..76474bcb 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -343,6 +343,7 @@ void Tune::tune() { // There are some options and variants that are different based on GPU manufacturer bool AMDGPU = isAmdGpu(q->context->deviceId()); + bool NVIDIAGPU = isNvidiaGpu(q->context->deviceId()); bool tune_config = 1; bool time_FFTs = 0; @@ -601,7 +602,7 @@ void Tune::tune() { } // Find best FAST_BARRIER setting - if (AMDGPU) { + if (1 /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_fast_barrier = 0; @@ -910,6 +911,66 @@ void Tune::tune() { args->flags["ZEROHACK_H"] = to_string(best_zerohack_h); } + // Find best WMUL setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_wmul = 0; + u32 current_wmul = args->value("WMUL", 2); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 wmul : {1, 2, 4}) { + args->flags["WMUL"] = to_string(wmul); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using WMUL=%u is %6.1f\n", fft.spec().c_str(), wmul, cost); + if (wmul == current_wmul) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_wmul = wmul; } + } + log("Best WMUL is %u. Default WMUL is 2.\n", best_wmul); + configsUpdate(current_cost, best_cost, 0.003, "WMUL", best_wmul, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["WMUL"] = to_string(best_wmul); + } + + // Find best ENABLE_L2STORE setting + if (NVIDIAGPU) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_enable_l2store = 0; + u32 current_enable_l2store = args->value("ENABLE_L2STORE", 2); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 enable_l2store : {0, 1}) { + args->flags["ENABLE_L2STORE"] = to_string(enable_l2store); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using ENABLE_L2STORE=%u is %6.1f\n", fft.spec().c_str(), enable_l2store, cost); + if (enable_l2store == current_enable_l2store) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_enable_l2store = enable_l2store; } + } + log("Best ENABLE_L2STORE is %u. Default ENABLE_L2STORE is 1.\n", best_enable_l2store); + configsUpdate(current_cost, best_cost, 0.003, "ENABLE_L2STORE", best_enable_l2store, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["ENABLE_L2STORE"] = to_string(best_enable_l2store); + } + + // Find best ENABLE_LULOAD setting + if (NVIDIAGPU) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_enable_luload = 0; + u32 current_enable_luload = args->value("ENABLE_LULOAD", 2); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 enable_luload : {0, 1}) { + args->flags["ENABLE_LULOAD"] = to_string(enable_luload); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using ENABLE_LULOAD=%u is %6.1f\n", fft.spec().c_str(), enable_luload, cost); + if (enable_luload == current_enable_luload) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_enable_luload = enable_luload; } + } + log("Best ENABLE_LULOAD is %u. Default ENABLE_LULOAD is 1.\n", best_enable_luload); + configsUpdate(current_cost, best_cost, 0.003, "ENABLE_LULOAD", best_enable_luload, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["ENABLE_LULOAD"] = to_string(best_enable_luload); + } + // Find best BIGLIT setting if (0 && time_FFTs) { // Deprecated FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; From 1b0a8238f6761ee7dc7d97d36121b6900fdf8b9d Mon Sep 17 00:00:00 2001 From: george Date: Thu, 12 Mar 2026 08:01:26 +0000 Subject: [PATCH 010/214] Fixed bug in unused PTX code --- src/cl/math.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index 5ef7d515..28401a7b 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -210,7 +210,7 @@ u128 OVERLOAD mad64(u64 a, u64 b, u64 c) { #if ENABLE_MAD64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why u64 reslo, reshi; __asm("mad.lo.cc.u64 %0, %2, %3, %4;\n\t" - "madc.hi.u64 %1, %2, %3, 0;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b), "l"(u128_lo64(c))); + "madc.hi.u64 %1, %2, %3, 0;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b), "l"(c)); return make_u128(reshi, reslo); #elif HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Faster on TitanV. No difference on mobile 4070. Much cleaner PTX code generated. uint2 a2 = as_uint2(a); From fa7c9ec3036a50d184f8a5949ccfc56ad2c961af Mon Sep 17 00:00:00 2001 From: george Date: Fri, 13 Mar 2026 17:54:54 +0000 Subject: [PATCH 011/214] Fixed bug where WMUL default was 2 in Gpu.cpp and 1 in carryfused.cl --- src/cl/carryfused.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index ba48c8bb..312923dc 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -18,7 +18,7 @@ void spin() { // Increasing WMUL to 2 will reduce carryShuttle activity. This led to a 1% speedup on Titan V. Testing on other GPUs is needed. #ifndef WMUL -#define WMUL 1 +#define WMUL 2 #endif #if AMDGPU From b34571c36f29131d7308625074f8ad439d4de90a Mon Sep 17 00:00:00 2001 From: george Date: Sat, 14 Mar 2026 19:23:32 +0000 Subject: [PATCH 012/214] Change lds_bytes to a #define. Some OpenCL compilers did not like a const u32 definition. --- src/cl/carryfused.cl | 36 ++++++++++++++---------------------- src/cl/ffthin.cl | 15 +++++++-------- src/cl/tailmul.cl | 15 +++++++-------- src/cl/tailsquare.cl | 39 +++++++++++++++------------------------ 4 files changed, 43 insertions(+), 62 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 312923dc..ccbf931e 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -16,6 +16,9 @@ void spin() { #endif } +// LDS bytes used by shufl for each line processed in fft_WIDTH +#define LDS_BYTES (WIDTH * SHUFL_BYTES_W) + // Increasing WMUL to 2 will reduce carryShuttle activity. This led to a 1% speedup on Titan V. Testing on other GPUs is needed. #ifndef WMUL #define WMUL 2 @@ -33,8 +36,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) // If WMUL is one, there is no shuffling of carries if (WMUL == 1) return; - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes used by shufl for each WMUL workgroup - const u32 lds_i64s = lds_bytes / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL workgroup + const u32 lds_i64s = LDS_BYTES / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL line local i64 *lds = (local i64 *) lds2; // Handle nasty case where we are writing 8-byte quantities but SHUFL_BYTES_W is only 4 bytes @@ -87,8 +89,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) // If WMUL is one, there is no shuffling of carries if (WMUL == 1) return; - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes used by shufl for each WMUL workgroup - const u32 lds_i32s = lds_bytes / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL workgroup + const u32 lds_i32s = LDS_BYTES / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL line local i32 *lds = (local i32 *) lds2; lds += (me / G_W) * lds_i32s + lowMe; // This WMUL workgroup's LDS area @@ -109,8 +110,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local T2 lds[WMUL * lds_bytes / sizeof(T2)]; + local T2 lds[WMUL * LDS_BYTES / sizeof(T2)]; T2 u[NW]; @@ -298,8 +298,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local F2 lds[WMUL * lds_bytes / sizeof(F2)]; + local F2 lds[WMUL * LDS_BYTES / sizeof(F2)]; F2 u[NW]; @@ -481,8 +480,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF31 lds[WMUL * lds_bytes / sizeof(GF31)]; + local GF31 lds[WMUL * LDS_BYTES / sizeof(GF31)]; GF31 u[NW]; @@ -690,8 +688,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds[WMUL * LDS_BYTES / sizeof(GF61)]; GF61 u[NW]; @@ -906,8 +903,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local T2 lds[WMUL * lds_bytes / sizeof(T2)]; + local T2 lds[WMUL * LDS_BYTES / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; T2 u[NW]; @@ -1143,8 +1139,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local F2 ldsF2[WMUL * lds_bytes / sizeof(F2)]; + local F2 ldsF2[WMUL * LDS_BYTES / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; F2 uF2[NW]; @@ -1383,8 +1378,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; F2 uF2[NW]; @@ -1622,8 +1616,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; GF31 u31[NW]; @@ -1870,8 +1863,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index 82b3377d..233aa92f 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -4,12 +4,14 @@ #include "math.cl" #include "fftheight.cl" +// LDS bytes used by shufl for each line processed in fft_HEIGHT +#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) + #if FFT_FP64 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH]; u32 g = get_group_id(0); @@ -39,8 +41,7 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -74,8 +75,7 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -103,8 +103,7 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 3e2299f8..14682112 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -5,6 +5,9 @@ #include "trig.cl" #include "fftheight.cl" +// LDS bytes used by shufl for each line processed in fft_HEIGHT +#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) + #if FFT_FP64 // Handle the final multiplication step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -49,8 +52,7 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH], v[NH]; T2 p[NH], q[NH]; @@ -161,8 +163,7 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; CP(F2) aF2 = (CP(F2)) a; @@ -270,8 +271,7 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar } KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); @@ -394,8 +394,7 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar } KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index d9edc1f7..c4625378 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -4,6 +4,9 @@ #include "trig.cl" #include "fftheight.cl" +// LDS bytes used by shufl for each line processed in fft_HEIGHT +#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) + #if FFT_FP64 // Handle the final squaring step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -54,8 +57,7 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH]; u32 H = ND / SMALL_HEIGHT; @@ -89,8 +91,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH], v[NH]; @@ -198,8 +199,7 @@ void OVERLOAD pairSq2_special(T2 *u, T2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes * 2 / sizeof(T2)]; + local T2 lds[2 * LDS_BYTES / sizeof(T2)]; T2 u[NH]; @@ -332,8 +332,7 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; F2 u[NH]; u32 H = ND / SMALL_HEIGHT; @@ -363,8 +362,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -468,8 +466,7 @@ void OVERLOAD pairSq2_special(F2 *u, F2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes * 2 / sizeof(F2)]; + local F2 lds[2 * LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -601,8 +598,7 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -650,8 +646,7 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -750,8 +745,7 @@ void OVERLOAD pairSq2_special(GF31 *u, GF31 base_squared) { } KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes * 2 / sizeof(GF31)]; + local GF31 lds[2 * LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -879,8 +873,7 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -928,8 +921,7 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -1028,8 +1020,7 @@ void OVERLOAD pairSq2_special(GF61 *u, GF61 base_squared) { } KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes * 2 / sizeof(GF61)]; + local GF61 lds[2 * LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); From 76486bdc9b5ba0ae8ce69cbfd69520f28d258d19 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 23 Mar 2026 19:07:38 +0000 Subject: [PATCH 013/214] Added load/store macros for many nvidia options. Added ability to connect PRPLL's loads and stores to any of the load/store possibilities. --- src/cl/base.cl | 412 ++++++++++++++++++++++++++++++++++++------- src/cl/carryfused.cl | 90 +++++----- src/cl/fft-middle.cl | 32 ++-- src/cl/fftbase.cl | 46 ++--- src/cl/middle.cl | 96 +++++----- src/cl/tailmul.cl | 35 ++-- src/cl/tailsquare.cl | 107 +++++------ 7 files changed, 546 insertions(+), 272 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 1765fdfe..75ccabf8 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -189,14 +189,6 @@ G_H "group height" == SMALL_HEIGHT / NH #define ZEROHACK_H 1 #endif -#if !defined(ENABLE_L2STORE) -#define ENABLE_L2STORE 1 -#endif - -#if !defined(ENABLE_LULOAD) -#define ENABLE_LULOAD 1 -#endif - // Expected defines: EXP the exponent. // WIDTH, SMALL_HEIGHT, MIDDLE. @@ -267,54 +259,383 @@ ulong2 OVERLOAD U2(ulong a, ulong b) { return (ulong2) (a, b); } #define P(x) global x * restrict #define CP(x) const P(x) -// Macros for non-temporal load and store. The theory behind only non-temporal reads (option 2) is that with alternating buffers, -// read buffers will not be needed for quite a while, but write buffers will be needed soon. -#if NONTEMPORAL == 1 && defined(__has_builtin) && __has_builtin(__builtin_nontemporal_load) && __has_builtin(__builtin_nontemporal_store) -#define NTLOAD(mem) __builtin_nontemporal_load(&(mem)) -#define NTSTORE(mem,val) __builtin_nontemporal_store(val, &(mem)) -#elif NONTEMPORAL == 2 && defined(__has_builtin) && __has_builtin(__builtin_nontemporal_load) -#define NTLOAD(mem) __builtin_nontemporal_load(&(mem)) -#define NTSTORE(mem,val) (mem) = val +#define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void + + +// For reasons unknown, loading trig values into nVidia's constant cache has terrible performance +#if AMDGPU +typedef constant const T2* Trig; +typedef constant const T* TrigSingle; +typedef constant const F2* TrigFP32; +typedef constant const GF31* TrigGF31; +typedef constant const GF61* TrigGF61; +#else +typedef global const T2* Trig; +typedef global const T* TrigSingle; +typedef global const F2* TrigFP32; +typedef global const GF31* TrigGF31; +typedef global const GF61* TrigGF61; +#endif +// However, caching weights in nVidia's constant cache improves performance. +// Even better is to not pollute the constant cache with weights that are used only once. +// This requires two typedefs depending on how we want to use the BigTab pointer. +// For AMD we can declare BigTab as constant or global - it doesn't really matter. +typedef constant const double2* ConstBigTab; +typedef constant const float2* ConstBigTabFP32; +#if AMDGPU +typedef constant const double2* BigTab; +typedef constant const float2* BigTabFP32; +#else +typedef global const double2* BigTab; +typedef global const float2* BigTabFP32; +#endif + +// +// nVidia GPUs have lots of different caching options for loads and stores. +// AMD GPUs have have far fewer options for loads and stores. +// These routines let us try the different options. +// + +// Basic load and store. Presumably stored in all caches using a standard LRU algorithm. + +#define LOAD(mem) *(mem) +#define STORE(mem,val) *(mem) = val + +// Non-temporal load and store. + +#if defined(__has_builtin) && __has_builtin(__builtin_nontemporal_load) +#define NTLOAD(mem) __builtin_nontemporal_load(mem) +#else +#define NTLOAD LOAD +#endif + +#if defined(__has_builtin) && __has_builtin(__builtin_nontemporal_store) +#define NTSTORE(mem,val) __builtin_nontemporal_store(val, mem) +#else +#define NTSTORE STORE +#endif + +// Routines for loading data from memory into the L2 cache but not the L1 cache. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD L2LOAD(CP(T2) mem) { + T2 retval; + __asm("ld.global.cg.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD L2LOAD(TrigSingle mem) { + T retval; + __asm("ld.global.cg.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD L2LOAD(CP(F2) mem) { + F2 retval; + __asm("ld.global.cg.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +i64 OVERLOAD L2LOAD(i64 *mem) { + i64 retval; + __asm("ld.global.cg.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD L2LOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.cg.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD L2LOAD(i32 *mem) { + i32 retval; + __asm("ld.global.cg.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD L2LOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.cg.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; +} #else -#define NTLOAD(mem) (mem) -#define NTSTORE(mem,val) (mem) = val +#define L2LOAD LOAD #endif // Routines for storing to L2 cache bypassing L1 cache. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +void OVERLOAD L2STORE(P(T2) mem, T2 val) { + __asm("st.global.cg.v2.f64 [%0], {%1, %2};" : : "l"(mem), "d"(val.x), "d"(val.y)); +} +void OVERLOAD L2STORE(P(F2) mem, F2 val) { + __asm("st.global.cg.v2.f32 [%0], {%1, %2};" : : "l"(mem), "f"(val.x), "f"(val.y)); +} void OVERLOAD L2STORE(i64 *mem, i64 val) { -#if ENABLE_L2STORE && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher __asm("st.global.cg.b64 [%0], %1;" : : "l"(mem), "l"(val)); -#else - *mem = val; -#endif } void OVERLOAD L2STORE(i32 *mem, i32 val) { -#if ENABLE_L2STORE && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher __asm("st.global.cg.b32 [%0], %1;" : : "l"(mem), "r"(val)); +} #else - *mem = val; +#define L2STORE STORE #endif + +// Routines for loading data from memory into the L1 and L2 caches, but cache line is marked evict first. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD EFLOAD(CP(T2) mem) { + T2 retval; + __asm("ld.global.cs.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD EFLOAD(TrigSingle mem) { + T retval; + __asm("ld.global.cs.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD EFLOAD(CP(F2) mem) { + F2 retval; + __asm("ld.global.cs.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +i64 OVERLOAD EFLOAD(i64 *mem) { + i64 retval; + __asm("ld.global.cs.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD EFLOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.cs.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD EFLOAD(i32 *mem) { + i32 retval; + __asm("ld.global.cs.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD EFLOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.cs.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; } +#else +#define EFLOAD LOAD +#endif + +// Routines for storing to L1 and L2 caches with cache line marked evict first. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +void OVERLOAD EFSTORE(P(T2) mem, T2 val) { + __asm("st.global.cs.v2.f64 [%0], {%1, %2};" : : "l"(mem), "d"(val.x), "d"(val.y)); +} +void OVERLOAD EFSTORE(P(F2) mem, F2 val) { + __asm("st.global.cs.v2.f32 [%0], {%1, %2};" : : "l"(mem), "f"(val.x), "f"(val.y)); +} +void OVERLOAD EFSTORE(i64 *mem, i64 val) { + __asm("st.global.cs.b64 [%0], %1;" : : "l"(mem), "l"(val)); +} +void OVERLOAD EFSTORE(i32 *mem, i32 val) { + __asm("st.global.cs.b32 [%0], %1;" : : "l"(mem), "r"(val)); +} +#else +#define EFSTORE STORE +#endif // Routines for loading a value and marking it for "last use". + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD LULOAD(Trig mem) { + T2 retval; + __asm("ld.global.lu.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD LULOAD(TrigSingle mem) { + T retval; + __asm("ld.global.lu.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD LULOAD(TrigFP32 mem) { + F2 retval; + __asm("ld.global.lu.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} i64 OVERLOAD LULOAD(i64 *mem) { -#if ENABLE_LULOAD && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher i64 retval; __asm("ld.global.lu.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); return retval; -#else - return *mem; -#endif +} +GF61 OVERLOAD LULOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.lu.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; } i32 OVERLOAD LULOAD(i32 *mem) { -#if ENABLE_LULOAD && HAS_PTX >= 200 // Cache hints requires sm_20 support or higher i32 retval; __asm("ld.global.lu.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); return retval; +} +GF31 OVERLOAD LULOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.lu.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; +} #else - return *mem; +#define LULOAD LOAD #endif + +// Routines for loading a read-only and placing it in the non-coherent texture cache. + +#if HAS_PTX >= 500 // Texture cache requires sm_50 support or higher +T2 OVERLOAD NCLOAD(Trig mem) { + T2 retval; + __asm("ld.global.nc.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; } +T OVERLOAD NCLOAD(TrigSingle mem) { + T retval; + __asm("ld.global.nc.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD NCLOAD(TrigFP32 mem) { + F2 retval; + __asm("ld.global.nc.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +i64 OVERLOAD NCLOAD(i64 *mem) { + i64 retval; + __asm("ld.global.nc.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD NCLOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.nc.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD NCLOAD(i32 *mem) { + i32 retval; + __asm("ld.global.nc.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD NCLOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.lu.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; +} +#else +#define NCLOAD LOAD +#endif + +// +// These macros map various types of data accesses to one of the load/store routines above +// + +// Routines for loading/storing FFT data. Lots of data, kernels read it once, write it once. If possible, data should not be written to L1 cache. +// If L2 cache is "small", we should look for ways to prioritize keeping data that is re-used in the L2 cache. + +#define FFTLOAD_TYPE LOADS % 10 +#define CSLOAD_TYPE (LOADS / 10) % 10 +#define TFLOAD_TYPE (LOADS / 100) % 10 +#define TSLOAD_TYPE (LOADS / 1000) % 10 +#define TOLOAD_TYPE (LOADS / 10000) % 10 + +#define FFTSTORE_TYPE STORES % 10 +#define CSSTORE_TYPE (STORES / 10) % 10 + +#if FFTLOAD_TYPE == 1 +#define FFTLOAD NTLOAD +#elif FFTLOAD_TYPE == 2 +#define FFTLOAD L2LOAD +#elif FFTLOAD_TYPE == 3 +#define FFTLOAD EFLOAD +#elif FFTLOAD_TYPE == 4 +#define FFTLOAD LULOAD +#else +#define FFTLOAD LOAD +#endif + +#if FFTSTORE_TYPE == 1 +#define FFTSTORE NTSTORE +#elif FFTSTORE_TYPE == 2 +#define FFTSTORE L2STORE +#elif FFTSTORE_TYPE == 3 +#define FFTSTORE EFSTORE +#else +#define FFTSTORE STORE +#endif + +// Routines for loading/storing carryShuttle data. CarryFused writes it once, and reads it once. The data is never used again. +// If possible, data should not be written to L1 cache and not written to memory after it is read. + +#if CSLOAD_TYPE == 1 +#define CSLOAD NTLOAD +#elif CSLOAD_TYPE == 2 +#define CSLOAD L2LOAD +#elif CSLOAD_TYPE == 3 +#define CSLOAD EFLOAD +#elif CSLOAD_TYPE == 4 +#define CSLOAD LULOAD +#else +#define CSLOAD LOAD +#endif + +#if CSSTORE_TYPE == 1 +#define CSSTORE NTSTORE +#elif CSSTORE_TYPE == 2 +#define CSSTORE L2STORE +#elif CSSTORE_TYPE == 3 +#define CSSTORE EFSTORE +#else +#define CSSTORE STORE +#endif + +// Routines for loading trig data that is frequently reused. If possible, data should saved in L1 and L2 caches and perhaps marked evict last. +// TF stands for "Trig Frequently reused". It is highly unlikely that any option other than the default LOAD makes sense. + +#if TFLOAD_TYPE == 1 +#define TFLOAD NTLOAD +#elif TFLOAD_TYPE == 2 +#define TFLOAD L2LOAD +#elif TFLOAD_TYPE == 3 +#define TFLOAD EFLOAD +#elif TFLOAD_TYPE == 4 +#define TFLOAD LULOAD +#elif TFLOAD_TYPE == 5 +#define TFLOAD NCLOAD +#else +#define TFLOAD LOAD +#endif + +// Routines for loading trig data that is used once but is smaller than a cache line. The rest of the cache line will be needed soon. +// If possible, data should saved in L1(?) and L2 caches and perhaps marked evict first. +// TS stands for "Trig Several reuses". + +#if TSLOAD_TYPE == 1 +#define TSLOAD NTLOAD +#elif TSLOAD_TYPE == 2 +#define TSLOAD L2LOAD +#elif TSLOAD_TYPE == 3 +#define TSLOAD EFLOAD +#elif TSLOAD_TYPE == 4 +#define TSLOAD LULOAD +#elif TSLOAD_TYPE == 5 +#define TSLOAD NCLOAD +#else +#define TSLOAD LOAD +#endif + +// Routines for loading trig data that is used once and is a cache line or larger. +// If possible, data should saved in L2 caches if the L2 cache is very large. +// TO stands for "Trig used Once". + +#if TOLOAD_TYPE == 1 +#define TOLOAD NTLOAD +#elif TOLOAD_TYPE == 2 +#define TOLOAD L2LOAD +#elif TOLOAD_TYPE == 3 +#define TOLOAD EFLOAD +#elif TOLOAD_TYPE == 4 +#define TOLOAD LULOAD +#elif TOLOAD_TYPE == 5 +#define TOLOAD NCLOAD +#else +#define TOLOAD LOAD +#endif // Prefetch macros. Unused at present, I tried using them in fftMiddleInGF61 on a 5080 with no benefit. void PREFETCHL1(const __global void *addr) { @@ -328,36 +649,6 @@ void PREFETCHL2(const __global void *addr) { #endif } -// For reasons unknown, loading trig values into nVidia's constant cache has terrible performance -#if AMDGPU -typedef constant const T2* Trig; -typedef constant const T* TrigSingle; -typedef constant const F2* TrigFP32; -typedef constant const GF31* TrigGF31; -typedef constant const GF61* TrigGF61; -#else -typedef global const T2* Trig; -typedef global const T* TrigSingle; -typedef global const F2* TrigFP32; -typedef global const GF31* TrigGF31; -typedef global const GF61* TrigGF61; -#endif -// However, caching weights in nVidia's constant cache improves performance. -// Even better is to not pollute the constant cache with weights that are used only once. -// This requires two typedefs depending on how we want to use the BigTab pointer. -// For AMD we can declare BigTab as constant or global - it doesn't really matter. -typedef constant const double2* ConstBigTab; -typedef constant const float2* ConstBigTabFP32; -#if AMDGPU -typedef constant const double2* BigTab; -typedef constant const float2* BigTabFP32; -#else -typedef global const double2* BigTab; -typedef global const float2* BigTabFP32; -#endif - -#define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void - #if FFT_FP64 void OVERLOAD read(u32 WG, u32 N, T2 *u, const global T2 *in, u32 base) { in += base + (u32) get_local_id(0); @@ -441,3 +732,4 @@ void OVERLOAD bar(const u32 WG) { // A half-barrier is only needed when half-a-workgroup needs a barrier. // This is used e.g. by the double-wide tailSquare, where LDS is split between the halves. void halfBar() { if (get_enqueued_local_size(0) / 2 > WAVEFRONT) { bar(); } } + diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 312923dc..7e64f179 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -16,6 +16,9 @@ void spin() { #endif } +// LDS bytes used by shufl for each line processed in fft_WIDTH +#define LDS_BYTES (WIDTH * SHUFL_BYTES_W) + // Increasing WMUL to 2 will reduce carryShuttle activity. This led to a 1% speedup on Titan V. Testing on other GPUs is needed. #ifndef WMUL #define WMUL 2 @@ -33,8 +36,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) // If WMUL is one, there is no shuffling of carries if (WMUL == 1) return; - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes used by shufl for each WMUL workgroup - const u32 lds_i64s = lds_bytes / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL workgroup + const u32 lds_i64s = LDS_BYTES / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL line local i64 *lds = (local i64 *) lds2; // Handle nasty case where we are writing 8-byte quantities but SHUFL_BYTES_W is only 4 bytes @@ -87,8 +89,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) // If WMUL is one, there is no shuffling of carries if (WMUL == 1) return; - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes used by shufl for each WMUL workgroup - const u32 lds_i32s = lds_bytes / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL workgroup + const u32 lds_i32s = LDS_BYTES / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL line local i32 *lds = (local i32 *) lds2; lds += (me / G_W) * lds_i32s + lowMe; // This WMUL workgroup's LDS area @@ -109,8 +110,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local T2 lds[WMUL * lds_bytes / sizeof(T2)]; + local T2 lds[WMUL * LDS_BYTES / sizeof(T2)]; T2 u[NW]; @@ -192,7 +192,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -254,7 +254,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -264,7 +264,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -298,8 +298,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local F2 lds[WMUL * lds_bytes / sizeof(F2)]; + local F2 lds[WMUL * LDS_BYTES / sizeof(F2)]; F2 u[NW]; @@ -376,7 +375,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -438,7 +437,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -448,7 +447,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -481,8 +480,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF31 lds[WMUL * lds_bytes / sizeof(GF31)]; + local GF31 lds[WMUL * LDS_BYTES / sizeof(GF31)]; GF31 u[NW]; @@ -585,7 +583,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -638,7 +636,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -648,7 +646,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -690,8 +688,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds[WMUL * LDS_BYTES / sizeof(GF61)]; GF61 u[NW]; @@ -799,7 +796,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -853,7 +850,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -863,7 +860,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -906,8 +903,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local T2 lds[WMUL * lds_bytes / sizeof(T2)]; + local T2 lds[WMUL * LDS_BYTES / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; T2 u[NW]; @@ -1024,7 +1020,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -1086,7 +1082,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -1096,7 +1092,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -1143,8 +1139,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local F2 ldsF2[WMUL * lds_bytes / sizeof(F2)]; + local F2 ldsF2[WMUL * LDS_BYTES / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; F2 uF2[NW]; @@ -1264,7 +1259,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -1326,7 +1321,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -1336,7 +1331,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -1383,8 +1378,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; F2 uF2[NW]; @@ -1504,7 +1498,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -1566,7 +1560,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -1576,7 +1570,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -1622,8 +1616,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; GF31 u31[NW]; @@ -1753,7 +1746,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -1807,7 +1800,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -1817,7 +1810,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { @@ -1870,8 +1863,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - const u32 lds_bytes = WIDTH * SHUFL_BYTES_W; // LDS bytes needed for each WMUL workgroup - local GF61 lds61[WMUL * lds_bytes / sizeof(GF61)]; + local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; @@ -2015,7 +2007,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. if (gr < H / WMUL && me >= (WMUL-1) * G_W) { - for (i32 i = 0; i < NW; ++i) { L2STORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } + for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready #if OLD_FENCE @@ -2077,7 +2069,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. if (gr < H / WMUL) { for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess(me, i)]); } } else { @@ -2087,7 +2079,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { - carry[i] = LULOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); + carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } if (me == 0) { diff --git a/src/cl/fft-middle.cl b/src/cl/fft-middle.cl index 31db8bfc..03203a89 100644 --- a/src/cl/fft-middle.cl +++ b/src/cl/fft-middle.cl @@ -100,7 +100,7 @@ void OVERLOAD middleMul(T2 *u, u32 s, Trig trig) { if (MIDDLE == 1) return; if (WIDTH == SMALL_HEIGHT) trig += SMALL_HEIGHT; // In this case we can share the MiddleMul2 trig table. Skip over the MiddleMul trig table. - T2 w = trig[s]; // s / BIG_HEIGHT + T2 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT if (MIDDLE < SHARP_MIDDLE) { WADD(1, w); @@ -191,8 +191,8 @@ void OVERLOAD middleMul2(T2 *u, u32 x, u32 y, double factor, Trig trig) { return; } - trig += SMALL_HEIGHT; // Skip over the MiddleMul trig table - T2 w = trig[x]; // x / (MIDDLE * WIDTH) + trig += SMALL_HEIGHT; // Skip over the MiddleMul trig table + T2 w = TFLOAD(&trig[x]); // x / (MIDDLE * WIDTH) if (MIDDLE < SHARP_MIDDLE) { T2 base = slowTrig_N(x * y + x * SMALL_HEIGHT, ND / MIDDLE * 2) * factor; @@ -208,7 +208,7 @@ void OVERLOAD middleMul2(T2 *u, u32 x, u32 y, double factor, Trig trig) { Trig trig2 = trig + WIDTH; // Skip over the fist MiddleMul2 trig table u32 desired_root = x * y; - T2 base = cmulFancy(trig2[desired_root % SMALL_HEIGHT], trig[desired_root / SMALL_HEIGHT]) * factor; //Optimization to do: put multiply by factor in trig2 table + T2 base = cmulFancy(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig[desired_root / SMALL_HEIGHT])) * factor; //Optimization to do: put multiply by factor in trig2 table WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { @@ -396,8 +396,8 @@ void OVERLOAD middleMul(F2 *u, u32 s, TrigFP32 trig) { if (MIDDLE == 1) return; if (WIDTH == SMALL_HEIGHT) trig += SMALL_HEIGHT; // In this case we can share the MiddleMul2 trig table. Skip over the MiddleMul trig table. - F2 w = trig[s]; // s / BIG_HEIGHT - + F2 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT + if (MIDDLE < SHARP_MIDDLE) { WADD(1, w); #if MM_CHAIN == 0 @@ -488,7 +488,7 @@ void OVERLOAD middleMul2(F2 *u, u32 x, u32 y, float factor, TrigFP32 trig) { } trig += SMALL_HEIGHT; // Skip over the MiddleMul trig table - F2 w = trig[x]; // x / (MIDDLE * WIDTH) + F2 w = TFLOAD(&trig[x]); // x / (MIDDLE * WIDTH) if (MIDDLE < SHARP_MIDDLE) { F2 base = slowTrig_N(x * y + x * SMALL_HEIGHT, ND / MIDDLE * 2) * factor; @@ -504,7 +504,7 @@ void OVERLOAD middleMul2(F2 *u, u32 x, u32 y, float factor, TrigFP32 trig) { TrigFP32 trig2 = trig + WIDTH; // Skip over the fist MiddleMul2 trig table u32 desired_root = x * y; - F2 base = cmulFancy(trig2[desired_root % SMALL_HEIGHT], trig[desired_root / SMALL_HEIGHT]) * factor; //Optimization to do: put multiply by factor in trig2 table + F2 base = cmulFancy(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig[desired_root / SMALL_HEIGHT])) * factor; //Optimization to do: put multiply by factor in trig2 table WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { @@ -668,13 +668,13 @@ void OVERLOAD middleMul(GF31 *u, u32 s, TrigGF31 trig) { #if !MIDDLE_CHAIN // Read all trig values from memory for (u32 k = 1; k < MIDDLE; ++k) { - WADD(k, trig[s]); + WADD(k, TFLOAD(&trig[s])); s += SMALL_HEIGHT; } #else - GF31 w = trig[s]; // s / BIG_HEIGHT + GF31 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT WADD(1, w); if (MIDDLE == 2) return; @@ -705,9 +705,9 @@ void OVERLOAD middleMul2(GF31 *u, u32 x, u32 y, TrigGF31 trig) { // The first trig table can be shared with MiddleMul trig table if WIDTH = HEIGHT. if (WIDTH == SMALL_HEIGHT) trig1 = trig; - GF31 w = trig1[x]; // x / (MIDDLE * WIDTH) + GF31 w = TFLOAD(&trig1[x]); // x / (MIDDLE * WIDTH) u32 desired_root = x * y; - GF31 base = cmul(trig2[desired_root % SMALL_HEIGHT], trig1[desired_root / SMALL_HEIGHT]); + GF31 base = cmul(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig1[desired_root / SMALL_HEIGHT])); WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { @@ -788,13 +788,13 @@ void OVERLOAD middleMul(GF61 *u, u32 s, TrigGF61 trig) { #if !MIDDLE_CHAIN // Read all trig values from memory for (u32 k = 1; k < MIDDLE; ++k) { - WADD(k, trig[s]); + WADD(k, TFLOAD(&trig[s])); s += SMALL_HEIGHT; } #else - GF61 w = trig[s]; // s / BIG_HEIGHT + GF61 w = TFLOAD(&trig[s]); // s / BIG_HEIGHT WADD(1, w); if (MIDDLE == 2) return; @@ -825,9 +825,9 @@ void OVERLOAD middleMul2(GF61 *u, u32 x, u32 y, TrigGF61 trig) { // The first trig table can be shared with MiddleMul trig table if WIDTH = HEIGHT. if (WIDTH == SMALL_HEIGHT) trig1 = trig; - GF61 w = trig1[x]; // x / (MIDDLE * WIDTH) + GF61 w = TFLOAD(&trig1[x]); // x / (MIDDLE * WIDTH) u32 desired_root = x * y; - GF61 base = cmul(trig2[desired_root % SMALL_HEIGHT], trig1[desired_root / SMALL_HEIGHT]); + GF61 base = cmul(TFLOAD(&trig2[desired_root % SMALL_HEIGHT]), TFLOAD(&trig1[desired_root / SMALL_HEIGHT])); WADD(0, base); for (u32 k = 1; k < MIDDLE; ++k) { diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 9d2fde10..88cd3380 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -246,7 +246,7 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { // Apparently, chained Fancy muls at these short n=4 and n=8 lengths are very accurate. if (TABMUL_CHAIN) { - T2 w = trig[p]; + T2 w = TFLOAD(&trig[p]); chainMul(n, u, w, 0); return; } @@ -255,7 +255,7 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { // Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. if (!TABMUL_CHAIN) { - T2 w = trig[p]; + T2 w = TFLOAD(&trig[p]); if (n >= 8) { u[1] = cmulFancy(u[1], w); @@ -264,7 +264,7 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { } for (u32 i = 2; i < n; ++i) { - u[i] = cmul(u[i], trig[(i-1)*WG + p]); + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } @@ -292,11 +292,11 @@ void preload_tabMul4_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 // Read 3 lines of sine/cosine values for the first fft4. Read two of the lines as a pair as AMD likes T2 global memory reads Trig trig2 = (Trig) trig1; - T2 sine_over_cosines = trig2[me]; + T2 sine_over_cosines = TFLOAD(&trig2[me]); preloads[0] = sine_over_cosines.x; preloads[1] = sine_over_cosines.y; // Read 3rd line - preloads[2] = trig1[2*WG + me]; + preloads[2] = TFLOAD(&trig1[2*WG + me]); } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. @@ -326,7 +326,7 @@ void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f // Read pairs of lines to make AMD happy with T2 global memory loads for (u32 i = 0; i < 4; i += 2) { Trig trig2 = (Trig) (trig1 + i*WG); - T2 cosines = trig2[me]; + T2 cosines = TFLOAD(&trig2[me]); preloads[i] = cosines.x; preloads[i+1] = cosines.y; } @@ -357,8 +357,8 @@ void finish_tabMul4_fft4(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG // Preload one line of sine/cosines and one line of cosines for later tabMuls. We'll later broadcast these values as needed using LDS. if (f == 1) { - preloads[4] = trig1[3*WG + me]; // Sine/cosines for later tabMuls - preloads[5] = trig1[4*WG + 4*WG + me]; // Cosines for later tabMuls + preloads[4] = TFLOAD(&trig1[3*WG + me]); // Sine/cosines for later tabMuls + preloads[5] = TFLOAD(&trig1[4*WG + 4*WG + me]); // Cosines for later tabMuls } // Do the last level of fft4 applying cosine1 @@ -380,12 +380,12 @@ void preload_tabMul8_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 // Read 7 lines of sine/cosine values for the first fft8. Read six of the lines as pairs as AMD likes T2 global memory reads for (u32 i = 1; i < 7; i += 2) { Trig trig2 = (Trig) (trig1 + (i-1)*WG); - T2 sine_over_cosines = trig2[me]; + T2 sine_over_cosines = TFLOAD(&trig2[me]); preloads[i-1] = sine_over_cosines.x; preloads[i] = sine_over_cosines.y; } // Read 7th line - preloads[6] = trig1[6*WG + me]; + preloads[6] = TFLOAD(&trig1[6*WG + me]); } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. @@ -415,7 +415,7 @@ void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f // Read pairs of lines to make AMD happy with T2 global memory loads for (u32 i = 0; i < 8; i += 2) { Trig trig2 = (Trig) (trig1 + i*WG); - T2 cosines = trig2[me]; + T2 cosines = TFLOAD(&trig2[me]); preloads[i] = cosines.x; preloads[i+1] = cosines.y; } @@ -455,8 +455,8 @@ void finish_tabMul8_fft8(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. if (f == 1) { - preloads[8] = trig1[7*WG + me]; // Sine/cosines for second tabMul - preloads[9] = trig1[8*WG + 8*WG + me]; // Cosines for second tabMul + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul } // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3/cosine1 @@ -486,8 +486,8 @@ void finish_tabMul8_fft8(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. if (f == 1) { - preloads[8] = trig1[7*WG + me]; // Sine/cosines for second tabMul - preloads[9] = trig1[8*WG + 8*WG + me]; // Cosines for second tabMul + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul } // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3 @@ -562,7 +562,7 @@ void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN32) { - chainMul(n, u, trig[p], 0); + chainMul(n, u, TFLOAD(&trig[p]), 0); return; } @@ -570,12 +570,12 @@ void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { if (!TABMUL_CHAIN32) { if (n >= 8) { - u[1] = cmulFancy(u[1], trig[p]); + u[1] = cmulFancy(u[1], TFLOAD(&trig[p])); } else { - u[1] = cmul(u[1], trig[p]); + u[1] = cmul(u[1], TFLOAD(&trig[p])); } for (u32 i = 2; i < n; ++i) { - u[i] = cmul(u[i], trig[(i-1)*WG + p]); + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } @@ -630,7 +630,7 @@ void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN31) { - chainMul(n, u, trig[p]); + chainMul(n, u, TFLOAD(&trig[p])); return; } @@ -638,7 +638,7 @@ void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { if (!TABMUL_CHAIN31) { for (u32 i = 1; i < n; ++i) { - u[i] = cmul(u[i], trig[(i-1)*WG + p]); + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } @@ -693,7 +693,7 @@ void OVERLOAD tabMul(u32 WG, TrigGF61 trig, GF61 *u, u32 n, u32 f, u32 me) { // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN61) { - chainMul(n, u, trig[p], 0); + chainMul(n, u, TFLOAD(&trig[p]), 0); return; } @@ -701,7 +701,7 @@ void OVERLOAD tabMul(u32 WG, TrigGF61 trig, GF61 *u, u32 n, u32 f, u32 me) { if (!TABMUL_CHAIN61) { for (u32 i = 1; i < n; ++i) { - u[i] = cmul(u[i], trig[(i-1)*WG + p]); + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 36fcac56..41809b48 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -58,10 +58,10 @@ void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #else out += line * WIDTH + me; - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #endif } @@ -71,10 +71,10 @@ void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { // Rather than having u[i] also increment by one, we choose a larger pad increment u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; in += y * WIDTH + y * PAD_SIZE + (y / SMALL_HEIGHT) * BIG_PAD_SIZE + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } #else in += y * WIDTH + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT * WIDTH]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT * WIDTH]); } #endif } @@ -108,7 +108,7 @@ void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = SMALL_HEIGHT / (IN_WG / IN_SIZEX) * (MIDDLE * IN_WG + PAD_SIZE) // = SMALL_HEIGHT * MIDDLE * IN_SIZEX + SMALL_HEIGHT / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #else @@ -118,7 +118,7 @@ void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * SMALL_HEIGHT / (IN_WG / IN_SIZEX) * IN_WG // = MIDDLE * SMALL_HEIGHT * IN_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #endif } @@ -151,7 +151,7 @@ void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { for (i32 i = 0; i < NH; ++i) { // u32 fftMiddleIn_y = i * G_H + me; // The fftMiddleIn y value // u32 chunk_y = fftMiddleIn_y / SIZEY; // The fftMiddleIn chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } @@ -177,7 +177,7 @@ void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { for (i32 i = 0; i < NH; ++i) { u32 fftMiddleIn_y = i * G_H + me; // The fftMiddleIn y value u32 chunk_y = fftMiddleIn_y / SIZEY; // The fftMiddleIn chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } @@ -208,10 +208,10 @@ void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { #else out += line * (SMALL_HEIGHT + PAD_SIZE) + me; // Pad every output line #endif - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #else // No padding out += line * SMALL_HEIGHT + me; - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #endif } @@ -225,10 +225,10 @@ void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { #else in += y * MIDDLE * (SMALL_HEIGHT + PAD_SIZE) + x; #endif - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT + PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT + PAD_SIZE)]); } #else // No rotation, might be better on nVidia cards in += y * MIDDLE * SMALL_HEIGHT + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT]); } #endif } @@ -295,7 +295,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = WIDTH / (OUT_WG / OUT_SIZEX) * (MIDDLE * OUT_WG + PAD_SIZE) // = WIDTH * MIDDLE * OUT_SIZEX + WIDTH / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #else @@ -305,7 +305,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * WIDTH / (OUT_WG / OUT_SIZEX) * OUT_WG // = MIDDLE * WIDTH * OUT_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #endif } @@ -337,7 +337,7 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { for (i32 i = 0; i < NW; ++i) { // u32 fftMiddleOut_y = i * G_W + me; // The fftMiddleOut y value // u32 chunk_y = fftMiddleOut_y / SIZEY; // The fftMiddleOut chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } @@ -363,7 +363,7 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { for (i32 i = 0; i < NW; ++i) { // u32 fftMiddleOut_y = i * G_W + me; // The fftMiddleOut y value // u32 chunk_y = fftMiddleOut_y / SIZEY; // The fftMiddleOut chunk_y value - u[i] = NTLOAD(in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } @@ -384,10 +384,10 @@ void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #else out += line * WIDTH + me; - for (u32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W], u[i]); } + for (u32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W], u[i]); } #endif } @@ -397,10 +397,10 @@ void OVERLOAD readMiddleInLine(F2 *u, CP(F2) in, u32 y, u32 x) { // Rather than having u[i] also increment by one, we choose a larger pad increment u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; in += y * WIDTH + y * PAD_SIZE + (y / SMALL_HEIGHT) * BIG_PAD_SIZE + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT * (WIDTH + PAD_SIZE) + BIG_PAD_SIZE)]); } #else in += y * WIDTH + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT * WIDTH]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT * WIDTH]); } #endif } @@ -417,7 +417,7 @@ void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = SMALL_HEIGHT / (IN_WG / IN_SIZEX) * (MIDDLE * IN_WG + PAD_SIZE) // = SMALL_HEIGHT * MIDDLE * IN_SIZEX + SMALL_HEIGHT / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #else // Output data such that readCarryFused lines are packed tightly together. No padding. out += chunk_y * MIDDLE * IN_WG + // Write y chunks after middles @@ -425,7 +425,7 @@ void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * SMALL_HEIGHT / (IN_WG / IN_SIZEX) * IN_WG // = MIDDLE * SMALL_HEIGHT * IN_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * IN_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * IN_WG], u[i]); } #endif } @@ -452,7 +452,7 @@ void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 fftMiddleIn_y_incr = G_H; // The increment to next fftMiddleIn y value u32 chunk_y_incr = fftMiddleIn_y_incr / SIZEY; // The increment to next fftMiddleIn chunk_y value for (i32 i = 0; i < NH; ++i) { - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } #else // Read data that was not rotated or padded @@ -474,7 +474,7 @@ void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { for (i32 i = 0; i < NH; ++i) { u32 fftMiddleIn_y = i * G_H + me; // The fftMiddleIn y value u32 chunk_y = fftMiddleIn_y / SIZEY; // The fftMiddleIn chunk_y value - u[i] = NTLOAD(in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * IN_WG)]); // Adjust in pointer the same way writeMiddleInLine did chunk_y += chunk_y_incr; } #endif @@ -488,10 +488,10 @@ void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { #else out += line * (SMALL_HEIGHT + PAD_SIZE) + me; // Pad every output line #endif - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #else // No padding out += line * SMALL_HEIGHT + me; - for (u32 i = 0; i < NH; ++i) { NTSTORE(out[i * G_H], u[i]); } + for (u32 i = 0; i < NH; ++i) { FFTSTORE(&out[i * G_H], u[i]); } #endif } @@ -505,10 +505,10 @@ void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { #else in += y * MIDDLE * (SMALL_HEIGHT + PAD_SIZE) + x; #endif - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * (SMALL_HEIGHT + PAD_SIZE)]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * (SMALL_HEIGHT + PAD_SIZE)]); } #else // No rotation, might be better on nVidia cards in += y * MIDDLE * SMALL_HEIGHT + x; - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SMALL_HEIGHT]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SMALL_HEIGHT]); } #endif } @@ -524,7 +524,7 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = WIDTH / (OUT_WG / OUT_SIZEX) * (MIDDLE * OUT_WG + PAD_SIZE) // = WIDTH * MIDDLE * OUT_SIZEX + WIDTH / SIZEY * PAD_SIZE // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #else // Output data such that readCarryFused lines are packed tightly together. No padding. out += chunk_y * MIDDLE * OUT_WG + // Write y chunks after middles @@ -532,7 +532,7 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // = MIDDLE * WIDTH / (OUT_WG / OUT_SIZEX) * OUT_WG // = MIDDLE * WIDTH * OUT_SIZEX // Write each u[i] sequentially - for (int i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * OUT_WG], u[i]); } + for (int i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * OUT_WG], u[i]); } #endif } @@ -556,7 +556,7 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 fftMiddleOut_y_incr = G_W; // The increment to next fftMiddleOut y value u32 chunk_y_incr = fftMiddleOut_y_incr / SIZEY; // The increment to next fftMiddleOut chunk_y value for (i32 i = 0; i < NW; ++i) { - u[i] = NTLOAD(in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * (MIDDLE * OUT_WG + PAD_SIZE)]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } #else // Read data that was not rotated or padded @@ -576,7 +576,7 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 fftMiddleOut_y_incr = G_W; // The increment to next fftMiddleOut y value u32 chunk_y_incr = fftMiddleOut_y_incr / SIZEY; // The increment to next fftMiddleOut chunk_y value for (i32 i = 0; i < NW; ++i) { - u[i] = NTLOAD(in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did + u[i] = FFTLOAD(&in[chunk_y * MIDDLE * OUT_WG]); // Adjust in pointer the same way writeMiddleOutLine did chunk_y += chunk_y_incr; } #endif @@ -780,7 +780,7 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); - for (u32 i = 0; i < NW; ++i) { u[i] = NTLOAD(in[i * G_W / 16 * SIZEW]); } + for (u32 i = 0; i < NW; ++i) { u[i] = FFTLOAD(&in[i * G_W / 16 * SIZEW]); } } // Write a line from carryFused. This data will be read by fftMiddleIn. @@ -788,7 +788,7 @@ void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { // me i u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); - for (i32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W / 16 * SIZEW], u[i]); } + for (i32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W / 16 * SIZEW], u[i]); } } //**************************************************************************************** @@ -801,14 +801,14 @@ void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { // me i void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { in += (x / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM]); } } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 y, u32 x) { out += (x / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM], u[i]); } + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } } //**************************************************************************************** @@ -824,14 +824,14 @@ void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT in += (width / 16 * SIZEW) + (middle * SIZEM) + (width % 16 * SIZEBLK) + (me % 16); - for (i32 i = 0; i < NH; ++i) { u[i] = NTLOAD(in[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } + for (i32 i = 0; i < NH; ++i) { u[i] = FFTLOAD(&in[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } } void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT out += (width / 16 * SIZEW) + (middle * SIZEM) + (width % 16 * SIZEBLK) + (me % 16); - for (i32 i = 0; i < NH; ++i) { NTSTORE(out[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } + for (i32 i = 0; i < NH; ++i) { FFTSTORE(&out[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } } //**************************************************************************************** @@ -844,14 +844,14 @@ void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { in += (y / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM]); } } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) { out += (y / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM], u[i]); } + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } } #endif @@ -899,7 +899,7 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); - for (u32 i = 0; i < NW; ++i) { u[i] = NTLOAD(in[i * G_W / 16 * SIZEW32]); } + for (u32 i = 0; i < NW; ++i) { u[i] = FFTLOAD(&in[i * G_W / 16 * SIZEW32]); } } // Write a line from carryFused. This data will be read by fftMiddleIn. @@ -907,7 +907,7 @@ void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { // me u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); - for (i32 i = 0; i < NW; ++i) { NTSTORE(out[i * G_W / 16 * SIZEW32], u[i]); } + for (i32 i = 0; i < NW; ++i) { FFTSTORE(&out[i * G_W / 16 * SIZEW32], u[i]); } } //**************************************************************************************** @@ -920,14 +920,14 @@ void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { // me void OVERLOAD readMiddleInLine(F2 *u, CP(F2) in, u32 y, u32 x) { in += (x / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM32]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM32]); } } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 y, u32 x) { out += (x / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, y / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM32], u[i]); } + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } } //**************************************************************************************** @@ -943,14 +943,14 @@ void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT in += (width / 16 * SIZEW32) + (middle * SIZEM32) + (width % 16 * SIZEBLK32) + (me % 16); - for (i32 i = 0; i < NH; ++i) { u[i] = NTLOAD(in[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } + for (i32 i = 0; i < NH; ++i) { u[i] = FFTLOAD(&in[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } } void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT out += (width / 16 * SIZEW32) + (middle * SIZEM32) + (width % 16 * SIZEBLK32) + (me % 16); - for (i32 i = 0; i < NH; ++i) { NTSTORE(out[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } + for (i32 i = 0; i < NH; ++i) { FFTSTORE(&out[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16], u[i]); } } //**************************************************************************************** @@ -963,14 +963,14 @@ void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { in += (y / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { u[i] = NTLOAD(in[i * SIZEM32]); } + for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM32]); } } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 y, u32 x) { out += (y / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, x / 16) * 16) + (x % 16); - for (i32 i = 0; i < MIDDLE; ++i) { NTSTORE(out[i * SIZEM32], u[i]); } + for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } } #endif diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 3e2299f8..51ac9c11 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -5,6 +5,9 @@ #include "trig.cl" #include "fftheight.cl" +// LDS bytes used by shufl for each line processed in fft_HEIGHT +#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) + #if FFT_FP64 // Handle the final multiplication step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -49,8 +52,7 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH], v[NH]; T2 p[NH], q[NH]; @@ -161,8 +163,7 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; CP(F2) aF2 = (CP(F2)) a; @@ -270,8 +271,7 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar } KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); @@ -310,18 +310,18 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS31 >= 1 - GF31 trig = smallTrig31[height_trigs + me]; // Trig values for line zero, should be cached + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); // Trig values for line zero, should be cached #if SINGLE_WIDE - GF31 mult = smallTrig31[height_trigs + G_H + line1]; + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line1]); #else - GF31 mult = smallTrig31[height_trigs + G_H + line1 * 2]; + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line1 * 2]); #endif trig = cmul(trig, mult); #else #if SINGLE_WIDE - GF31 trig = NTLOAD(smallTrig31[height_trigs + line1*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line1*G_H + me]); #else - GF31 trig = NTLOAD(smallTrig31[height_trigs + line1*2*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line1*2*G_H + me]); #endif #endif @@ -394,8 +394,7 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar } KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); @@ -434,18 +433,18 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS61 >= 1 - GF61 trig = smallTrig61[height_trigs + me]; // Trig values for line zero, should be cached + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); // Trig values for line zero, should be cached #if SINGLE_WIDE - GF61 mult = smallTrig61[height_trigs + G_H + line1]; + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line1]); #else - GF61 mult = smallTrig61[height_trigs + G_H + line1 * 2]; + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line1 * 2]); #endif trig = cmul(trig, mult); #else #if SINGLE_WIDE - GF61 trig = NTLOAD(smallTrig61[height_trigs + line1*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line1*G_H + me]); #else - GF61 trig = NTLOAD(smallTrig61[height_trigs + line1*2*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line1*2*G_H + me]); #endif #endif diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index d9edc1f7..e30b8012 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -4,6 +4,9 @@ #include "trig.cl" #include "fftheight.cl" +// LDS bytes used by shufl for each line processed in fft_HEIGHT +#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) + #if FFT_FP64 // Handle the final squaring step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -54,8 +57,7 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH]; u32 H = ND / SMALL_HEIGHT; @@ -89,8 +91,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH], v[NH]; @@ -132,8 +133,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached T2 per line - T2 trig = smallTrig[height_trigs + me]; // Trig values for line zero, should be cached - T2 mult = smallTrig[height_trigs + G_H + line1]; // Line multiplier + T2 trig = TFLOAD(&smallTrig[height_trigs + me]); // Trig values for line zero, should be cached + T2 mult = TSLOAD(&smallTrig[height_trigs + G_H + line1]); // Line multiplier trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -142,7 +143,7 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values - T2 trig = NTLOAD(smallTrig[height_trigs + line1*G_H + me]); + T2 trig = TOLOAD(&smallTrig[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL @@ -198,8 +199,7 @@ void OVERLOAD pairSq2_special(T2 *u, T2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local T2 lds[lds_bytes * 2 / sizeof(T2)]; + local T2 lds[2 * LDS_BYTES / sizeof(T2)]; T2 u[NH]; @@ -245,8 +245,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached T2 per line - T2 trig = smallTrig[height_trigs + lowMe]; // Trig values for line zero, should be cached - T2 mult = smallTrig[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + T2 trig = TFLOAD(&smallTrig[height_trigs + lowMe]); // Trig values for line zero, should be cached + T2 mult = TSLOAD(&smallTrig[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -255,7 +255,7 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values - T2 trig = NTLOAD(smallTrig[height_trigs + line_u*G_H*2 + me]); + T2 trig = TOLOAD(&smallTrig[height_trigs + line_u*G_H*2 + me]); #endif #if SINGLE_KERNEL @@ -332,8 +332,7 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; F2 u[NH]; u32 H = ND / SMALL_HEIGHT; @@ -363,8 +362,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -402,8 +400,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached F2 per line - F2 trig = smallTrigF2[height_trigs + me]; // Trig values for line zero, should be cached - F2 mult = smallTrigF2[height_trigs + G_H + line1]; // Line multiplier + F2 trig = TFLOAD(&smallTrigF2[height_trigs + me]); // Trig values for line zero, should be cached + F2 mult = TSLOAD(&smallTrigF2[height_trigs + G_H + line1]); // Line multiplier trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -412,7 +410,7 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - F2 trig = NTLOAD(smallTrigF2[height_trigs + line1*G_H + me]); + F2 trig = TOLOAD(&smallTrigF2[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL @@ -468,8 +466,7 @@ void OVERLOAD pairSq2_special(F2 *u, F2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local F2 lds[lds_bytes * 2 / sizeof(F2)]; + local F2 lds[2 * LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -511,8 +508,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached F2 per line - F2 trig = smallTrigF2[height_trigs + lowMe]; // Trig values for line zero, should be cached - F2 mult = smallTrigF2[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + F2 trig = TFLOAD(&smallTrigF2[height_trigs + lowMe]); // Trig values for line zero, should be cached + F2 mult = TSLOAD(&smallTrigF2[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmulFancy(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -521,7 +518,7 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - F2 trig = NTLOAD(smallTrigF2[height_trigs + line_u*G_H*2 + me]); + F2 trig = TOLOAD(&smallTrigF2[height_trigs + line_u*G_H*2 + me]); #endif #if SINGLE_KERNEL @@ -601,8 +598,7 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -623,18 +619,18 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS31 >= 1 - GF31 trig = smallTrig31[height_trigs + me]; + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); #if SINGLE_WIDE - GF31 mult = smallTrig31[height_trigs + G_H + line]; + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line]); #else - GF31 mult = smallTrig31[height_trigs + G_H + which]; + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + which]); #endif trig = cmul(trig, mult); #else #if SINGLE_WIDE - GF31 trig = NTLOAD(smallTrig31[height_trigs + line*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line*G_H + me]); #else - GF31 trig = NTLOAD(smallTrig31[height_trigs + which*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + which*G_H + me]); #endif #endif @@ -650,8 +646,7 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -685,8 +680,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF31 per line - GF31 trig = smallTrig31[height_trigs + me]; // Trig values for line zero, should be cached - GF31 mult = smallTrig31[height_trigs + G_H + line1]; // Line multiplier + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); // Trig values for line zero, should be cached + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line1]); // Line multiplier trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -695,7 +690,7 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF31 trig = NTLOAD(smallTrig31[height_trigs + line1*G_H + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL @@ -750,8 +745,7 @@ void OVERLOAD pairSq2_special(GF31 *u, GF31 base_squared) { } KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF31 lds[lds_bytes * 2 / sizeof(GF31)]; + local GF31 lds[2 * LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -789,8 +783,8 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF31 per line - GF31 trig = smallTrig31[height_trigs + lowMe]; // Trig values for line zero, should be cached - GF31 mult = smallTrig31[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + GF31 trig = TFLOAD(&smallTrig31[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -799,7 +793,7 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF31 trig = NTLOAD(smallTrig31[height_trigs + line_u*G_H*2 + me]); + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line_u*G_H*2 + me]); #endif #if SINGLE_KERNEL @@ -879,8 +873,7 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -901,18 +894,18 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS61 >= 1 - GF61 trig = smallTrig61[height_trigs + me]; + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); #if SINGLE_WIDE - GF61 mult = smallTrig61[height_trigs + G_H + line]; + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line]); #else - GF61 mult = smallTrig61[height_trigs + G_H + which]; + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + which]); #endif trig = cmul(trig, mult); #else #if SINGLE_WIDE - GF61 trig = NTLOAD(smallTrig61[height_trigs + line*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line*G_H + me]); #else - GF61 trig = NTLOAD(smallTrig61[height_trigs + which*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + which*G_H + me]); #endif #endif @@ -928,8 +921,7 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -963,8 +955,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF61 per line - GF61 trig = smallTrig61[height_trigs + me]; // Trig values for line zero, should be cached - GF61 mult = smallTrig61[height_trigs + G_H + line1]; // Line multiplier + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); // Trig values for line zero, should be cached + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line1]); // Line multiplier trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -973,7 +965,7 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF61 trig = NTLOAD(smallTrig61[height_trigs + line1*G_H + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line1*G_H + me]); #endif #if SINGLE_KERNEL @@ -1028,8 +1020,7 @@ void OVERLOAD pairSq2_special(GF61 *u, GF61 base_squared) { } KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - const u32 lds_bytes = SMALL_HEIGHT * SHUFL_BYTES_H; - local GF61 lds[lds_bytes * 2 / sizeof(GF61)]; + local GF61 lds[2 * LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -1067,8 +1058,8 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read a hopefully cached line of data and one non-cached GF61 per line - GF61 trig = smallTrig61[height_trigs + lowMe]; // Trig values for line zero, should be cached - GF61 mult = smallTrig61[height_trigs + G_H + line_u*2 + isSecondHalf]; // Two multipliers. One for line u, one for line v. + GF61 trig = TFLOAD(&smallTrig61[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. trig = cmul(trig, mult); // On consumer-grade GPUs, it is likely beneficial to read all trig values. @@ -1077,7 +1068,7 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. u32 height_trigs = SMALL_HEIGHT*1; // Read pre-computed trig values - GF61 trig = NTLOAD(smallTrig61[height_trigs + line_u*G_H*2 + me]); + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line_u*G_H*2 + me]); #endif #if SINGLE_KERNEL From a2db643eb40c50691d3034625008c825e26f44bd Mon Sep 17 00:00:00 2001 From: george Date: Mon, 23 Mar 2026 19:19:07 +0000 Subject: [PATCH 014/214] Added CUDA support courtesy of "Sherpa" and Frey. Started some changess needed to get MSVC to compile PRPLL (CUDA support on Windows will require MSVC). --- Makefile | 44 +- README.md | 13 +- README.txt | 4 + src/Args.cpp | 19 +- src/Args.h | 9 +- src/Gpu.cpp | 61 +- src/Gpu.h | 2 +- src/KernelCompiler.cpp | 7 + src/clwrap.h | 11 + src/cuda/clwrap_cuda.cpp | 1099 ++++++++++++++++++++++++++++++++++++ src/cuda/cudawrap.cpp | 505 +++++++++++++++++ src/cuda/cudawrap.h | 138 +++++ src/cuda/opencl_compat.cuh | 365 ++++++++++++ src/cuda/tinycuda.h | 317 +++++++++++ src/tune.cpp | 286 +++++++--- 15 files changed, 2755 insertions(+), 125 deletions(-) create mode 100644 src/cuda/clwrap_cuda.cpp create mode 100644 src/cuda/cudawrap.cpp create mode 100644 src/cuda/cudawrap.h create mode 100644 src/cuda/opencl_compat.cuh create mode 100644 src/cuda/tinycuda.h diff --git a/Makefile b/Makefile index b1c021e0..1ba62799 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Use "make DEBUG=1" for a debug build +# Use "make CUDA=1" for a CUDA build, use "make DEBUG=1" for a debug build # The build artifacts are put in the "build-release" subfolder (or "build-debug" for a debug build). @@ -19,32 +19,39 @@ else CXX ?= g++ endif -ifneq ($(findstring MINGW, $(HOST_OS)), MINGW) -COMMON_FLAGS = -Wall -std=c++20 -static-libstdc++ -static-libgcc +ifeq ($(CUDA), 1) + BIN=build-cuda + CUDASRCS1 = clwrap_cuda.cpp cudawrap.cpp + CUDAFLAGS = -DCUDA_BACKEND -Isrc/cuda -I/usr/local/cuda/include + CUDAOBJS = $(CUDASRCS1:%.cpp=$(BIN)/%.o) + OPENCL_LIBS = -L/usr/local/cuda/lib64 -lcuda -lnvrtc else -# For mingw-64 use this: -COMMON_FLAGS = -Wall -std=c++20 -static-libstdc++ -static-libgcc -static + BIN=build-release + CUDAFLAGS = + CUDAOBJS = + ifeq ($(HOST_OS), Darwin) + OPENCL_LIBS = -framework OpenCL + else + OPENCL_LIBS = -lOpenCL + endif endif -# -fext-numeric-literals -ifeq ($(HOST_OS), Darwin) -OPENCL_LIBS = -framework OpenCL +ifneq ($(findstring MINGW, $(HOST_OS)), MINGW) + COMMON_FLAGS = -Wall $(CUDAFLAGS) -std=c++20 -static-libstdc++ -static-libgcc else -OPENCL_LIBS = -lOpenCL +# For mingw-64 use this: + COMMON_FLAGS = -Wall $(CUDAFLAGS) -std=c++20 -static-libstdc++ -static-libgcc -static endif - +# -fext-numeric-literals ifeq ($(DEBUG), 1) BIN=build-debug - CXXFLAGS = -g $(COMMON_FLAGS) STRIP= else -BIN=build-release - CXXFLAGS = -O3 -DNDEBUG $(COMMON_FLAGS) STRIP=-s @@ -56,7 +63,7 @@ SRCS2 = test.cpp # SRCS=$(addprefix src/, $(SRCS1)) -OBJS = $(SRCS1:%.cpp=$(BIN)/%.o) +OBJS = $(CUDAOBJS) $(SRCS1:%.cpp=$(BIN)/%.o) DEPDIR := $(BIN)/.d $(shell mkdir -p $(DEPDIR) >/dev/null) DEPFLAGS = -MT $@ -MMD -MP -MF $(DEPDIR)/$*.Td @@ -80,16 +87,19 @@ $(BIN)/prpll-amd: ${OBJS} $(CXX) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) -lamdocl64 -L/opt/rocm/lib ${STRIP} clean: - rm -rf build-debug build-release + rm -rf build-debug build-release build-cuda $(BIN)/%.o : src/%.cpp $(DEPDIR)/%.d $(COMPILE.cc) $(OUTPUT_OPTION) $< $(POSTCOMPILE) +$(BIN)/%.o : src/cuda/%.cpp $(DEPDIR)/%.d + $(COMPILE.cc) $(OUTPUT_OPTION) $< + $(POSTCOMPILE) -# src/bundle.cpp is just a wrapping of the OpenCL sources (*.cl) as a C string. +# src/bundle.cpp is just a wrapping of the OpenCL sources (*.cl) as a C string (as well as the CUDA OpenCL translation code) -src/bundle.cpp: genbundle.sh src/cl/*.cl +src/bundle.cpp: genbundle.sh src/cuda/*.cuh src/cl/*.cl bash genbundle.sh $^ > src/bundle.cpp $(DEPDIR)/%.d: ; diff --git a/README.md b/README.md index 5ae916af..ba6b9ada 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ PRPLL implements two primality tests for Mersenne numbers: PRP ("PRobable Prime") and LL ("Lucas-Lehmer") as the name suggests. -PRPLL is an OpenCL (GPU) program for primality testing Mersenne numbers. +PRPLL is an OpenCL (GPU) and CUDA program for primality testing Mersenne numbers. ## Build @@ -32,12 +32,21 @@ Invoke `make` in the source directory. See `prpll -h` for the command line options. +## License + +This project is licensed under the **GNU General Public License v3.0** - see [LICENSE](LICENSE) for details. + + +## Credits + +[PRPLL](https://github.com/preda/gpuowl) was originally authored (as gpuowl) by **Mihai Preda**. **George Woltman** authored optimizations and NTT code contributions. The CUDA backend was authored by **"Sherpa"** in honor of John Allen Frey. + + ## Why LL For Mersenne primes search, the PRP test is by far preferred over LL, such that LL is not used anymore for search. But LL is still used to verify a prime found by PRP (which is a very rare occurence). - ### Lucas-Lehmer (LL) This is a test that proves whether a Mersenne number is prime or not, but without providing a factor in the case where it is not prime. The Lucas-Lehmer test is very simple to describe: iterate the function f(x)=(x^2 - 2) modulo M(p) starting with the number 4. If diff --git a/README.txt b/README.txt index 64b8dcd9..825fb6c3 100644 --- a/README.txt +++ b/README.txt @@ -9,6 +9,7 @@ In the gpuowl project directory (where the file Makefile is located) run make. This will produce a file "prpll" in the build-debug or build-release subdirectory. Use "make" to do a release build in the "build-release" subdirectory. +Use "make CUDA=1" to produce a CUDA version in the "build-cuda" subdirectory. Use "make DEBUG=1" to produce a debug build in the "build-debug" subdirectory. Use "make exe" for a Windows build. Use "make clean" to remove the "build-debug" and "build-release" directories. @@ -28,3 +29,6 @@ Run you need to fix your OpenCL installation first. 2. run "prpll -h", and verify that it displays a list of devices towards the end. + +3. run "prpll -tune", to tune various PRPLL options for your GPU. + diff --git a/src/Args.cpp b/src/Args.cpp index e66e1cc0..cece2fa6 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -2,7 +2,6 @@ #include "Args.h" #include "File.h" -#include "FFTConfig.h" #include "clwrap.h" #include "gpuid.h" #include "Proof.h" @@ -99,14 +98,13 @@ bool Args::hasFlag(const string& key) const { return flags.find(key) != flags.en void Args::printHelp() { printf(R"( PRPLL is "PRobable Prime and Lucas-Lehmer Categorizer", AKA "Purple-cat" -PRPLL is under active development and not ready for production use. -PRPLL is an OpenCL (GPU) program for primality testing Mersenne numbers (of the form 2^n - 1). +PRPLL is an OpenCL/CUDA (GPU) program for primality testing Mersenne numbers (of the form 2^n - 1). To check that OpenCL is installed correctly use the command "clinfo". If clinfo does not find any devices or otherwise fails, this program will not run. -This program is tested on Linux/ROCm (AMD GPUs); it may also run on Windows and on Nvidia GPUs. +This program is tested on Linux/ROCm (AMD GPUs); it also runs on Windows and on Nvidia GPUs. For information about Mersenne primes search see https://www.mersenne.org/ @@ -168,8 +166,8 @@ named "config.txt" in the prpll run directory. -roe : measure the Round-Off Error (Z) for more iterations (slow) -use : comma separated list of defines for configuring gpuowl.cl, such as: - -use FAST_BARRIER: on AMD Radeon VII and older AMD GPUs, use a faster barrier(). Do not use - this option on Nvidia GPUs or on RDNA AMD GPUs where it produces errors + -use FAST_BARRIER: on AMD Radeon VII and older AMD GPUs, use a faster barrier(). This option + may not work on Nvidia GPUs or on RDNA AMD GPUs where it produces errors (which are nevertheless detected). -use NO_ASM : do not use __asm() blocks (inline assembly) -use STATS= : enable carry statistics collection & logging, for the kernel according to : @@ -200,11 +198,14 @@ named "config.txt" in the prpll run directory. -tune : Looks for best settings to include in config.txt. Times many FFTs to find fastest one to test exponents -- written to tune.txt. An -fft can be given on the command line to limit which FFTs are timed. Options are not required. If present, the options are a comma separated list from below. - noconfig - Skip timings to find best config.txt settings - fp64 - Tune for settings that affect FP64 FFTs. Time FP64 FFTs for tune.txt. + noconfig - Skip timings to find best config.txt settings. + inplace - Skip timings for not-in-place FFTs and NTTs. All nVidia GPUs seem to prefer in-place FFTs and NTTs. + fp64 - Tune for settings that affect FP64 FFTs. Time FP64 FFTs for tune.txt. ntt - Tune for settings that affect integer NTTs. Time integer NTTs for tune.txt. + nofp32 - Do not tune for settings that affect FP32 FFTs. Some openCL compilers have trouble with FP32. minexp= - Time FFTs to find the best one for exponents greater than . maxexp= - Time FFTs to find the best one for exponents less than . + quick= - Higher values equals a quicker, potentially less accurate tune. Val ranges from 1 to 10. -device : select the GPU at position N in the list of devices -uid : select the GPU with the given UID (on ROCm/AMDGPU, Linux) -pci : select the GPU with the given PCI BDF, e.g. "0c:00.0" @@ -380,7 +381,7 @@ void Args::parse(const string& line) { else if (key == "-dir") { dir = s; } else if (key == "-carry") { if (s == "short" || s == "long") { - carry = s == "short" ? CARRY_SHORT : CARRY_LONG; + carry = s == "short" ? CARRY_32 : CARRY_64; } else { log("-carry expects short|long\n"); throw "-carry expects short|long"; diff --git a/src/Args.h b/src/Args.h index d7273afe..99d3a87d 100644 --- a/src/Args.h +++ b/src/Args.h @@ -3,6 +3,7 @@ #pragma once #include "common.h" +#include "FFTConfig.h" #include #include @@ -21,10 +22,8 @@ class Args { static vector splitUses(std::string ss); static std::string mergeArgs(int argc, char **argv); - enum {CARRY_AUTO = 0, CARRY_SHORT, CARRY_LONG}; - explicit Args(bool silent = false) : silent{silent} {} - + void parse(const string& line); void setDefaults(); bool uses(const std::string& key) const { return flags.find(key) != flags.end(); } @@ -56,7 +55,7 @@ class Args { std::map> perFftConfig; int device = 0; - + bool safeMath = true; bool clean = true; bool verbose = false; @@ -72,7 +71,7 @@ class Args { bool keepProof = false; - int carry = CARRY_AUTO; + enum CARRY_KIND carry = CARRY_AUTO; u32 workers = 1; u32 blockSize = 1000; u32 logStep = 20000; diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 6d00c0fd..5d30c848 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -228,7 +228,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Default value for -use options that must also be parsed in C++ code tail_single_wide = 0, tail_single_kernel = 1; // Default tailSquare is double-wide in one kernel in_place = 0; // Default is not in-place - wmul = 2; // Default is carryFused processes two lines at a time + wmul = 2; // Default is carryFused processes two lines at a time pad_size = isAmdGpu(id) ? 256 : 0; // Default is 256 bytes for AMD, 0 for others // Validate -use options @@ -248,7 +248,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "DEBUG", "CARRY64", "BIGLIT", // Deprecated - "NONTEMPORAL", + "NONTEMPORAL", // Deprecated "INPLACE", "PAD", "MIDDLE_IN_LDS_TRANSPOSE", @@ -263,8 +263,8 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "TABMUL_CHAIN32", "TABMUL_CHAIN61", "MODM31", - "ENABLE_L2STORE", - "ENABLE_LULOAD", + "LOADS","STORES", + "CFBLKS","MIBLKS","MOBLKS","TSBLKS", // CUDA - experimental "WMUL" }); if (!isValid) { @@ -283,6 +283,13 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< if (k == "PAD") pad_size = atoi(v.c_str()); } + // Maximum WMUL is 32KB / (WIDTH * SHUFL_BYTES_W) + { + u32 shufl_bytes_w = args.value("SHUFL_BYTES_W", 8); + u32 max_wmul = 32768 / (fft.shape.width * shufl_bytes_w); + if (wmul > max_wmul) wmul = max_wmul; + } + string defines = toDefine(config); if (doLog) { log("config: %s\n", defines.c_str()); } @@ -505,40 +512,53 @@ Gpu::~Gpu() { #define ROE_SIZE 100000 #define CARRY_SIZE 100000 +#if CUDA_BACKEND +#define CARRYFUSED_BLOCKS(x) args.value("CFBLKS", 0) == 0 ? x : args.value("CFBLKS", 0) == 1 ? x " -DCUDA_MIN_BLOCKS=3" : x " -maxregcount 84" +#define MIDDLEIN_BLOCKS args.value("MIBLKS", 0) == 0 ? "" : args.value("MIBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount 84" +#define MIDDLEOUT_BLOCKS args.value("MOBLKS", 0) == 0 ? "" : args.value("MOBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount 84" +#define TAILSQUARE_BLOCKS args.value("TSBLKS", 0) == 0 ? "" : args.value("TSBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount 84" +#else +#define CARRYFUSED_BLOCKS(x) "" +#define MIDDLEIN_BLOCKS "" +#define MIDDLEOUT_BLOCKS "" +#define TAILSQUARE_BLOCKS "" +#endif + Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : queue(q), background{shared.background}, args{*shared.args}, E(E), N(fft.shape.size()), - fft(fft), + fft(fft), WIDTH(fft.shape.width), SMALL_H(fft.shape.height), BIG_H(SMALL_H * fft.shape.middle), hN(N / 2), nW(fft.shape.nW()), nH(fft.shape.nH()), - useLongCarry{args.carry == Args::CARRY_LONG}, + useLongCarry{args.carry == CARRY_64}, compiler{args, queue->context, clDefines(args, queue->context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size, wmul)}, #define K(name, ...) name(#name, &compiler, profile.make(#name), queue, __VA_ARGS__) - K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H)), + K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), MIDDLEIN_BLOCKS), K(kfftHin, "ffthin.cl", "fftHin", hN / nH), K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2), K(ktailSquare, "tailsquare.cl", "tailSquare", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2), // Single-wide tailSquare with one kernel + hN / nH / 2, // Single-wide tailSquare with one kernel + TAILSQUARE_BLOCKS), K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2), K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H)), + K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), MIDDLEOUT_BLOCKS), K(kfftW, "fftw.cl", "fftW", hN / nW), - K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H)), + K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), MIDDLEIN_BLOCKS), K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH), - K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2), + K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2, TAILSQUARE_BLOCKS), K(ktailSquareGF31, "tailsquare.cl", "tailSquareGF31", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel @@ -546,20 +566,21 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& hN / nH / 2), // Single-wide tailSquare with one kernel K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2), K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H)), + K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), MIDDLEOUT_BLOCKS), K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW), - K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H)), + K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), MIDDLEIN_BLOCKS), K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH), K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2), K(ktailSquareGF61, "tailsquare.cl", "tailSquareGF61", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2), // Single-wide tailSquare with one kernel + hN / nH / 2, // Single-wide tailSquare with one kernel + TAILSQUARE_BLOCKS), K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2), K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H)), + K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), MIDDLEOUT_BLOCKS), K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW), K(kfftP, "fftp.cl", "fftP", hN / nW), @@ -568,11 +589,11 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1"), K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1 -DROE=1"), K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, "-DLL=1"), - K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW), - K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DROE=1"), - K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DMUL3=1"), - K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DMUL3=1 -DROE=1"), - K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, "-DLL=1"), + K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("")), + K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DROE=1")), + K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DMUL3=1")), + K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DMUL3=1 -DROE=1")), + K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DLL=1")), K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN), diff --git a/src/Gpu.h b/src/Gpu.h index b184c4d6..32c0a407 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -21,7 +21,7 @@ #include struct PRPResult; -struct Task; +class Task; class Signal; class ProofSet; diff --git a/src/KernelCompiler.cpp b/src/KernelCompiler.cpp index cbf50c16..59165081 100644 --- a/src/KernelCompiler.cpp +++ b/src/KernelCompiler.cpp @@ -65,9 +65,16 @@ Program KernelCompiler::compile(const string& fileName, const string& extraArgs) if (!dump.empty()) { args += " -save-temps="s + dump + "/" + fileName; } +#ifdef CUDA_BACKEND int err = clCompileProgram(p1.get(), 1, &deviceId, args.c_str(), clSources.size(), (const cl_program*) (clSources.data()), getClFileNames().data(), nullptr, nullptr); +#else + // Skip first file (opencl_compat.cuh) if this is a standard openCL application rather than a CUDA translation + int err = clCompileProgram(p1.get(), 1, &deviceId, args.c_str(), + clSources.size()-1, (const cl_program*) (clSources.data()+1), getClFileNames().data()+1, + nullptr, nullptr); +#endif if (string mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } if (err != CL_SUCCESS) { log("Compiling '%s' error %s (args %s)\n", fileName.c_str(), errMes(err).c_str(), args.c_str()); diff --git a/src/clwrap.h b/src/clwrap.h index 8b9bf9e4..f5cf78fd 100644 --- a/src/clwrap.h +++ b/src/clwrap.h @@ -2,7 +2,11 @@ #pragma once +#ifdef CUDA_BACKEND +#include "tinycuda.h" +#else #include "tinycl.h" +#endif #include #include @@ -123,3 +127,10 @@ std::array getEventNanos(cl_event event); u32 getEventInfo(cl_event event); cl_context getQueueContext(cl_command_queue q); + +#ifdef CUDA_BACKEND +// Set L2 cache persistence for multiple read-only buffers on the given stream. +// Computes the address span covering all buffers and sets a single access policy window. +// Buffers that are nullptr or zero-size are skipped. +void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers); +#endif diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp new file mode 100644 index 00000000..0f24cb63 --- /dev/null +++ b/src/cuda/clwrap_cuda.cpp @@ -0,0 +1,1099 @@ +// CUDA Driver API implementation of OpenCL API functions. +// This replaces clwrap.cpp when building with the native CUDA backend, +// mapping all cl* calls to cu* equivalents via the CUDA Driver API. + +#include "tinycuda.h" +#include "cudawrap.h" // For NvrtcProgram::preprocessOpenCL and compile + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +using namespace std; + +// Track allocated cl_mem objects so clSetKernelArg can distinguish buffer args from scalars. +// In OpenCL, buffer args are passed as &memobj where memobj is cl_mem (a pointer to _cl_mem). +// We need to convert these to CUdeviceptr for CUDA kernel launch. +static unordered_set g_allocatedBuffers; + +// Global CUDA context — set once by clCreateContext, used to ensure current before CUDA calls +static CUcontext g_cudaContext = nullptr; + +static void ensureContextCurrent() { + if (g_cudaContext) { + cuCtxSetCurrent(g_cudaContext); + } +} + +// Global state for CUDA initialization +static bool g_cudaInitialized = false; +static void ensureCudaInit() { + if (!g_cudaInitialized) { + CUresult err = cuInit(0); + if (err != CUDA_SUCCESS) { + fprintf(stderr, "cuInit failed: %d\n", (int)err); + } + g_cudaInitialized = true; + } +} + +// Global device list (allocated once, never freed) +static vector<_cl_device_id> g_devices; +static bool g_devicesEnumerated = false; + +static void enumerateDevices() { + if (g_devicesEnumerated) return; + ensureCudaInit(); + int count = 0; + cuDeviceGetCount(&count); + g_devices.resize(count); + for (int i = 0; i < count; i++) { + cuDeviceGet(&g_devices[i].dev, i); + } + g_devicesEnumerated = true; +} + +// ---- OpenCL API implementations ---- + +extern "C" { + +unsigned clGetPlatformIDs(unsigned num, cl_platform_id* platforms, unsigned* numRet) { + // CUDA has no "platforms" concept — just return 1 dummy + if (numRet) *numRet = 1; + if (platforms && num >= 1) platforms[0] = nullptr; + return CL_SUCCESS; +} + +int clGetDeviceIDs(cl_platform_id, cl_device_type, unsigned num, cl_device_id* devices, unsigned* numRet) { + enumerateDevices(); + unsigned n = g_devices.size(); + if (numRet) *numRet = n; + if (devices) { + for (unsigned i = 0; i < min(num, n); i++) { + devices[i] = &g_devices[i]; + } + } + return n > 0 ? CL_SUCCESS : CL_DEVICE_NOT_FOUND; +} + +cl_context clCreateContext(const intptr_t*, unsigned nDevices, const cl_device_id* devices, + void (*)(const char*, const void*, size_t, void*), void*, int* err) { + if (!devices || nDevices == 0) { if (err) *err = CL_INVALID_DEVICE; return nullptr; } + auto* ctx = new _cl_context; + ctx->dev = devices[0]->dev; +#if CUDA_VERSION >= 13000 + CUctxCreateParams params{}; + CUresult r = cuCtxCreate_v4(&ctx->ctx, ¶ms, 0, ctx->dev); +#else + CUresult r = cuCtxCreate(&ctx->ctx, 0, ctx->dev); +#endif + if (r != CUDA_SUCCESS) { + delete ctx; + if (err) *err = CL_OUT_OF_RESOURCES; + return nullptr; + } + g_cudaContext = ctx->ctx; // Track for ensureContextCurrent() + + // L2 persistence: no benefit measured for this workload. + // cuCtxSetLimit(CU_LIMIT_PERSISTING_L2_CACHE_SIZE, 16 * 1024 * 1024); + + if (err) *err = CL_SUCCESS; + return ctx; +} + +int clReleaseContext(cl_context ctx) { + if (ctx) { + cuCtxDestroy(ctx->ctx); + delete ctx; + } + return CL_SUCCESS; +} + +int clReleaseProgram(cl_program p) { + if (p) { + // NOTE: Do NOT unload the module here. PRPLL's loadAux() gets a kernel from + // the program, then releases the program. The kernel's CUfunction remains valid + // only while the CUmodule is loaded. In OpenCL, clCreateKernel retains the + // program. In our CUDA shim, we simply never unload modules — they persist for + // the process lifetime. This is safe because PRPLL creates a fixed set of kernels + // at startup and uses them until exit. + // if (p->moduleLoaded) cuModuleUnload(p->module); + delete p; + } + return CL_SUCCESS; +} + +int clReleaseCommandQueue(cl_command_queue q) { + if (q) { + cuStreamDestroy(q->stream); + delete q; + } + return CL_SUCCESS; +} + +// ---- Program compilation (NVRTC) ---- + +cl_program clCreateProgramWithSource(cl_context ctx, unsigned count, const char** strings, + const size_t* lengths, int* err) { + auto* prog = new _cl_program; + for (unsigned i = 0; i < count; i++) { + if (lengths && lengths[i]) { + prog->source.append(strings[i], lengths[i]); + } else { + prog->source.append(strings[i]); + } + } + if (err) *err = CL_SUCCESS; + return prog; +} + +cl_program clCreateProgramWithBinary(cl_context ctx, unsigned nDevices, const cl_device_id*, + const size_t* lengths, const unsigned char** binaries, + int* binaryStatus, int* err) { + // "Binary" in CUDA land = PTX string + auto* prog = new _cl_program; + if (lengths && binaries && lengths[0] > 0) { + prog->ptx.assign((const char*)binaries[0], lengths[0]); + prog->compiled = true; + // Load the module (JIT-compile PTX to SASS) + ensureContextCurrent(); + CUresult r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); + if (r == CUDA_SUCCESS) { + prog->moduleLoaded = true; + if (binaryStatus) binaryStatus[0] = CL_SUCCESS; + } else { + fprintf(stderr, "cuModuleLoadData from cache failed: %d, PTX size=%zu\n", (int)r, lengths[0]); + prog->compiled = false; + if (binaryStatus) binaryStatus[0] = CL_INVALID_BINARY; + if (err) { *err = CL_INVALID_BINARY; return prog; } + } + } + if (err) *err = CL_SUCCESS; + return prog; +} + +// Build log storage (per-program) +static string g_lastBuildLog; + +int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* devices, const char* options, + unsigned numHeaders, const cl_program* headers, const char* const* headerNames, + void (*)(cl_program, void*), void*) { + if (!prog) return CL_INVALID_PROGRAM; + + // Get device arch for NVRTC + CUdevice dev = devices ? devices[0]->dev : g_devices[0].dev; + int major = 0, minor = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev); + cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev); + + char archOpt[32]; + snprintf(archOpt, sizeof(archOpt), "--gpu-architecture=sm_%d%d", major, minor); + + // Parse OpenCL options string into NVRTC options + // Convert OpenCL build options to NVRTC equivalents: + // -cl-std=CL2.0 → -std=c++17 + // -cl-finite-math-only → --fmad=true (enable FMA contraction, the safe subset) + // -Dfoo=bar → -Dfoo=bar (pass through) + vector nvrtcOpts; + nvrtcOpts.push_back(archOpt); + nvrtcOpts.push_back("-default-device"); + nvrtcOpts.push_back("-std=c++17"); + nvrtcOpts.push_back("-w"); // Suppress NVRTC macro redefinition warnings + + // FMA contraction: OpenCL uses -cl-finite-math-only + #pragma OPENCL FP_CONTRACT ON + // to allow the compiler to contract a*b+c into FMA instructions. NVRTC's --fmad=true + // is the safe equivalent — it ONLY enables FMA contraction without the dangerous + // parts of -use_fast_math (no flush-to-zero, no reduced-precision division/sqrt). + // This is critical for FFT performance: every butterfly is multiply-add pairs. + nvrtcOpts.push_back("--fmad=true"); + + // NOTE: --restrict (all kernel pointers are __restrict__) was tested but causes GPU read + // errors — some PRPLL kernels use in-place operations where in/out buffers alias. + // Do NOT enable globally. The compiler still auto-uses __ldg() for const pointers on sm_35+. + + // Debug: dump full options string + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + static bool dumpedOpts = false; + if (dumpPrefix && !dumpedOpts && options) { + dumpedOpts = true; + fprintf(stderr, "clCompileProgram options: [%s]\n", options); + FILE* optLog = fopen("kernel_regs.log", "a"); + if (optLog) { fprintf(optLog, "clCompileProgram options: [%s]\n", options); fclose(optLog); } + } + } + + if (options) { + istringstream iss(options); + string tok; + while (iss >> tok) { + if (tok.substr(0, 2) == "-D") { + // Fix AMD-only FFT variants for NVIDIA: variant_W=0 and variant_H=0 require + // AMD builtins (__builtin_amdgcn_ds_bpermute etc). Replace with variant 2. + // FFT_VARIANT is a 3-digit number WMH: e.g. 000, 101, 202 + if (tok.find("FFT_VARIANT=") != string::npos) { + size_t eqPos = tok.find('='); + string valStr = tok.substr(eqPos + 1); + // Strip trailing 'u' suffix + if (!valStr.empty() && valStr.back() == 'u') valStr.pop_back(); + int val = atoi(valStr.c_str()); + int vW = val / 100; + int vM = (val % 100) / 10; + int vH = val % 10; + if (vW == 0) vW = 2; // AMD BCAST → NVIDIA generic + if (vH == 0) vH = 2; + int newVal = vW * 100 + vM * 10 + vH; + tok = "-DFFT_VARIANT=" + to_string(newVal) + "u"; + } + nvrtcOpts.push_back(tok); + } else if (tok == "-cl-finite-math-only" || tok == "-cl-fast-relaxed-math") { + // FMA contraction already enabled above via --fmad=true. + // Do NOT use -use_fast_math here — it enables flush-to-zero and + // reduced-precision division/sqrt which breaks tailMul accuracy. + } + // Skip other -cl-* options (not applicable to NVRTC) + } + } + + // Build NVRTC headers from the cl_program header array + vector> nvrtcHeaders; + + // Add all OpenCL source headers + for (unsigned i = 0; i < numHeaders; i++) { + // First header: opencl_compat.cuh (inject as virtual NVRTC header) + if (i == 0) { + nvrtcHeaders.push_back({"opencl_compat.cuh", headers[0]->source}); + } + // Remaining headers need preprocessing + else if (headers[i] && headerNames[i]) { + // Preprocess OpenCL source for CUDA compatibility + string processedSrc = NvrtcProgram::preprocessOpenCL(headers[i]->source); + // Debug: verify KERNEL macro replacement + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + if (dumpPrefix && string(headerNames[i]) == "base.cl") { + auto pos = processedSrc.find("KERNEL"); + if (pos != string::npos) { + string ctx = processedSrc.substr(pos > 20 ? pos-20 : 0, 120); + fprintf(stderr, "base.cl KERNEL context: [%s]\n", ctx.c_str()); + } + } + } + nvrtcHeaders.push_back({headerNames[i], processedSrc}); + } + } + + // Preprocess the main source + string processedSource = NvrtcProgram::preprocessOpenCL(prog->source); + + // Prepend opencl_compat.cuh include if not already there + if (processedSource.find("opencl_compat.cuh") == string::npos) { + processedSource = "#include \"opencl_compat.cuh\"\n" + processedSource; + } + + // Debug: dump preprocessed source when PRPLL_DUMP_PTX is set + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + if (dumpPrefix) { + static int srcCount = 0; + char fname[512]; + snprintf(fname, sizeof(fname), "%s_src_%d.cu", dumpPrefix, srcCount++); + FILE* f = fopen(fname, "w"); + if (f) { + fwrite(processedSource.c_str(), 1, processedSource.size(), f); + fclose(f); + fprintf(stderr, "Source dumped to %s (%zu bytes)\n", fname, processedSource.size()); + } + } + } + + // Store preprocessed source for __launch_bounds__ parsing in clCreateKernel + prog->preprocessedSource = processedSource; + for (auto& [name, src] : nvrtcHeaders) { + prog->preprocessedSource += "\n"; + prog->preprocessedSource += src; + } + + // Debug: dump NVRTC options when dumping PTX + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + static bool dumpedOnce = false; + if (dumpPrefix && !dumpedOnce) { + dumpedOnce = true; + fprintf(stderr, "NVRTC options (%zu):\n", nvrtcOpts.size()); + for (auto& o : nvrtcOpts) fprintf(stderr, " %s\n", o.c_str()); + } + } + + try { + prog->ptx = NvrtcProgram::compile(processedSource, "prpll_kernel.cu", nvrtcOpts, nvrtcHeaders); + prog->compiled = true; + g_lastBuildLog.clear(); + return CL_SUCCESS; + } catch (const exception& e) { + g_lastBuildLog = e.what(); + prog->compiled = false; + fprintf(stderr, "NVRTC COMPILE FAILED: %s\n", e.what()); + // Dump the full preprocessed source for debugging + { + char fname[64]; + static int failCount = 0; + snprintf(fname, sizeof(fname), "prpll_fail_%d.cu", failCount++); + FILE* f = fopen(fname, "w"); + if (f) { + fprintf(f, "// === FAILED Main source ===\n%s\n", processedSource.c_str()); + for (auto& [name, src] : nvrtcHeaders) { + fprintf(f, "\n// === Header: %s (%zu bytes) ===\n%s\n", name.c_str(), src.size(), src.c_str()); + } + fclose(f); + fprintf(stderr, "Dumped failed source to %s\n", fname); + } + } + return CL_COMPILE_PROGRAM_FAILURE; + } +} + +cl_program clLinkProgram(cl_context ctx, unsigned nDevices, const cl_device_id*, + const char* options, unsigned nProgs, const cl_program* progs, + void (*)(cl_program, void*), void*, int* err) { + ensureContextCurrent(); + // In CUDA, compilation produces PTX directly — no separate link step needed. + // Just load the PTX as a CUmodule. + if (!progs || nProgs == 0 || !progs[0] || !progs[0]->compiled) { + if (err) *err = CL_LINK_PROGRAM_FAILURE; + return nullptr; + } + + auto* linked = new _cl_program; + linked->ptx = progs[0]->ptx; + linked->compiled = true; + // Carry preprocessed source through for KERNEL(N) parsing in clCreateKernel + for (unsigned i = 0; i < nProgs; ++i) { + if (progs[i] && !progs[i]->preprocessedSource.empty()) { + linked->preprocessedSource += progs[i]->preprocessedSource; + linked->preprocessedSource += "\n"; + } + } + + // Use cuModuleLoadDataEx with error log to see JIT errors + char jitErrorLog[8192] = {}; + char jitInfoLog[4096] = {}; + CUjit_option jitOpts[] = { + CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES, CU_JIT_ERROR_LOG_BUFFER, + CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES, CU_JIT_INFO_LOG_BUFFER + }; + void* jitOptVals[] = { + (void*)(size_t)sizeof(jitErrorLog), (void*)jitErrorLog, + (void*)(size_t)sizeof(jitInfoLog), (void*)jitInfoLog + }; + CUresult r = cuModuleLoadDataEx(&linked->module, linked->ptx.c_str(), 4, jitOpts, jitOptVals); + if (r != CUDA_SUCCESS) { + const char* errName = nullptr; + cuGetErrorName(r, &errName); + fprintf(stderr, "cuModuleLoadData FAILED: %s (%d)\n", errName ? errName : "?", (int)r); + if (jitErrorLog[0]) fprintf(stderr, "JIT error log: %s\n", jitErrorLog); + if (jitInfoLog[0]) fprintf(stderr, "JIT info log: %s\n", jitInfoLog); + // Dump first 2000 chars of PTX for debugging + fprintf(stderr, "PTX size: %zu bytes\n", linked->ptx.size()); + // Dump full PTX to file + { + FILE* ptxFile = fopen("failed_ptx.ptx", "w"); + if (ptxFile) { + fwrite(linked->ptx.c_str(), 1, linked->ptx.size(), ptxFile); + fclose(ptxFile); + fprintf(stderr, "Dumped failed PTX to failed_ptx.ptx\n"); + } + } + delete linked; + if (err) *err = CL_LINK_PROGRAM_FAILURE; + return nullptr; + } + linked->moduleLoaded = true; + + // Dump PTX to file when PRPLL_DUMP_PTX is set (e.g., PRPLL_DUMP_PTX=kernel) + // Creates files like kernel_0.ptx, kernel_1.ptx, etc. + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + if (dumpPrefix) { + static int ptxCount = 0; + char fname[512]; + snprintf(fname, sizeof(fname), "%s_%d.ptx", dumpPrefix, ptxCount++); + FILE* f = fopen(fname, "w"); + if (f) { + fwrite(linked->ptx.c_str(), 1, linked->ptx.size(), f); + fclose(f); + fprintf(stderr, "PTX dumped to %s (%zu bytes)\n", fname, linked->ptx.size()); + } + } + } + + if (err) *err = CL_SUCCESS; + return linked; +} + +int clBuildProgram(cl_program prog, unsigned nDevices, const cl_device_id* devices, + const char* options, void (*)(cl_program, void*), void*) { + // If the program was loaded from binary (cached PTX) and already has a module, + // skip recompilation — the module is already JIT'd and ready. + if (prog && prog->moduleLoaded) { + return CL_SUCCESS; + } + + // clBuildProgram = compile + link in one step + int err = clCompileProgram(prog, nDevices, devices, options, 0, nullptr, nullptr, nullptr, nullptr); + if (err != CL_SUCCESS) return err; + + CUresult r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); + if (r != CUDA_SUCCESS) return CL_BUILD_PROGRAM_FAILURE; + prog->moduleLoaded = true; + return CL_SUCCESS; +} + +int clGetProgramBuildInfo(cl_program prog, cl_device_id, cl_program_build_info info, + size_t size, void* value, size_t* sizeRet) { + if (info == CL_PROGRAM_BUILD_LOG) { + size_t len = g_lastBuildLog.size() + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) { + memcpy(value, g_lastBuildLog.c_str(), len); + } + } + return CL_SUCCESS; +} + +int clGetProgramInfo(cl_program prog, cl_program_info info, size_t size, void* value, size_t* sizeRet) { + if (!prog) return CL_INVALID_PROGRAM; + if (info == CL_PROGRAM_BINARY_SIZES) { + size_t ptxSize = prog->ptx.size(); + if (sizeRet) *sizeRet = sizeof(size_t); + if (value && size >= sizeof(size_t)) memcpy(value, &ptxSize, sizeof(size_t)); + } else if (info == CL_PROGRAM_BINARIES) { + if (sizeRet) *sizeRet = sizeof(unsigned char*); + if (value && size >= sizeof(unsigned char*)) { + unsigned char** ptrs = (unsigned char**)value; + if (ptrs[0]) memcpy(ptrs[0], prog->ptx.data(), prog->ptx.size()); + } + } + return CL_SUCCESS; +} + +// ---- Kernel ---- + +cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { + ensureContextCurrent(); + if (!prog || !prog->moduleLoaded) { + if (err) *err = CL_INVALID_PROGRAM; + return nullptr; + } + auto* k = new _cl_kernel; + k->name = name; + k->parentModule = prog->module; + CUresult r = cuModuleGetFunction(&k->func, prog->module, name); + if (r != CUDA_SUCCESS) { + fprintf(stderr, "cuModuleGetFunction('%s') failed: %d, moduleLoaded=%d, module=%p\n", + name, (int)r, prog->moduleLoaded, (void*)prog->module); + delete k; + if (err) *err = CL_INVALID_KERNEL_NAME; + return nullptr; + } + + // Shared memory carveout: default adaptive carveout is optimal for mixed kernel workloads. + + // Log register and shared memory usage per kernel when PRPLL_DUMP_PTX is set + { + static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); + if (dumpPrefix) { + int numRegs = 0, shmem = 0, maxThreads = 0; + cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); + cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); + cuFuncGetAttribute(&maxThreads, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, k->func); + fprintf(stderr, " %-25s: %3d regs, %5d shmem, maxThreads=%d\n", name, numRegs, shmem, maxThreads); + // Also write to file since WSL2+CUDA swallows stderr + FILE* regLog = fopen("kernel_regs.log", "a"); + if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, maxThreads=%d\n", name, numRegs, shmem, maxThreads); fclose(regLog); } + } + } + + // Parse .maxntid from PTX to get __launch_bounds__ value. + // PTX pattern: .visible .entry (...)\n.maxntid N, 1, 1 + k->reqWorkGroupSize = 256; // fallback + { + const string& ptx = prog->ptx; + string entryPattern = ".entry " + string(name) + "("; + size_t pos = ptx.find(entryPattern); + if (pos != string::npos) { + // Found the kernel entry. Now find .maxntid before the next .entry or opening brace + size_t searchEnd = ptx.find(".entry ", pos + 1); + if (searchEnd == string::npos) searchEnd = ptx.size(); + string maxntidPattern = ".maxntid "; + size_t mpos = ptx.find(maxntidPattern, pos); + if (mpos != string::npos && mpos < searchEnd) { + int val = atoi(ptx.c_str() + mpos + maxntidPattern.size()); + if (val > 0) { + k->reqWorkGroupSize = val; + } + } + } + } + + if (err) *err = CL_SUCCESS; + return k; +} + +int clReleaseKernel(cl_kernel k) { + delete k; + return CL_SUCCESS; +} + +int clSetKernelArg(cl_kernel k, unsigned pos, size_t size, const void* value) { + if (!k) return CL_INVALID_KERNEL; + + // Detect cl_mem buffer arguments and convert to CUdeviceptr. + // In OpenCL, buffer args are set with clSetKernelArg(k, i, sizeof(cl_mem), &memobj). + // sizeof(cl_mem) == sizeof(void*) == 8 on 64-bit. The value at 'value' is a cl_mem pointer. + // We need to store the CUdeviceptr (GPU address) instead of the cl_mem (host pointer). + if (size == sizeof(cl_mem) && value) { + cl_mem mem = *(cl_mem*)value; + if (mem && g_allocatedBuffers.count(mem)) { + CUdeviceptr devPtr = mem->ptr; + k->setArg(pos, sizeof(CUdeviceptr), &devPtr); + return CL_SUCCESS; + } + // NULL cl_mem → pass a null device pointer + if (!mem) { + CUdeviceptr devPtr = 0; + k->setArg(pos, sizeof(CUdeviceptr), &devPtr); + return CL_SUCCESS; + } + } + + k->setArg(pos, size, value); + return CL_SUCCESS; +} + +// ---- Buffer ---- + +cl_mem clCreateBuffer(cl_context ctx, cl_mem_flags flags, size_t size, void* hostPtr, int* err) { + auto* buf = new _cl_mem; + buf->size = size; + ensureContextCurrent(); + CUresult r = cuMemAlloc(&buf->ptr, size); + if (r != CUDA_SUCCESS) { + delete buf; + if (err) *err = CL_MEM_OBJECT_ALLOCATION_FAILURE; + return nullptr; + } + // Handle CL_MEM_COPY_HOST_PTR + if ((flags & CL_MEM_COPY_HOST_PTR) && hostPtr) { + cuMemcpyHtoD(buf->ptr, hostPtr, size); + } + g_allocatedBuffers.insert(buf); + if (err) *err = CL_SUCCESS; + return buf; +} + +int clReleaseMemObject(cl_mem buf) { + if (buf) { + ensureContextCurrent(); + g_allocatedBuffers.erase(buf); + cuMemFree(buf->ptr); + delete buf; + } + return CL_SUCCESS; +} + +// ---- Command Queue ---- + +cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id dev, + const cl_queue_properties* props, int* err) { + auto* q = new _cl_command_queue; + q->context = ctx; + q->profiling = false; + + // Check for profiling flag + if (props) { + for (int i = 0; props[i]; i += 2) { + if (props[i] == CL_QUEUE_PROPERTIES && (props[i+1] & CL_QUEUE_PROFILING_ENABLE)) { + q->profiling = true; + } + } + } + + // Make sure context is current + cuCtxSetCurrent(ctx->ctx); + CUresult r = cuStreamCreate(&q->stream, CU_STREAM_NON_BLOCKING); + if (r != CUDA_SUCCESS) { + delete q; + if (err) *err = CL_OUT_OF_RESOURCES; + return nullptr; + } + if (err) *err = CL_SUCCESS; + return q; +} + +// ---- Enqueue operations ---- + +int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, + const size_t* globalOffset, const size_t* globalSize, + const size_t* localSize, unsigned nWaits, + const cl_event* waits, cl_event* event) { + if (!q || !k) return CL_INVALID_VALUE; + ensureContextCurrent(); + + size_t gs = globalSize[0]; + size_t ls = localSize ? localSize[0] : 256; + size_t numBlocks = (gs + ls - 1) / ls; + + // Build args array + void* argPtrs[_cl_kernel::MAX_ARGS]; + k->buildArgPointers(argPtrs); + + // Env-gated kernel profiling (PRPLL_PROFILE=1) — takes priority over event profiling + static bool doProfile = (getenv("PRPLL_PROFILE") != nullptr); + + // Event handling (skipped when env profiler is active) + if (!doProfile && event && q->profiling) { + auto* ev = new _cl_event; + cuEventCreate(&ev->start, CU_EVENT_DEFAULT); + cuEventCreate(&ev->end, CU_EVENT_DEFAULT); + ev->hasTimings = true; + ev->commandType = CL_COMMAND_NDRANGE_KERNEL; + cuEventRecord(ev->start, q->stream); + CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + cuEventRecord(ev->end, q->stream); + *event = ev; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; + } + if (doProfile) { + static std::map kTime; + static std::map kCount; + static std::map kRegs; + static std::map kShmem; + static int totalLaunches = 0; + static CUevent pStart = nullptr, pEnd = nullptr; + if (!pStart) { cuEventCreate(&pStart, CU_EVENT_DEFAULT); cuEventCreate(&pEnd, CU_EVENT_DEFAULT); } + + cuEventRecord(pStart, q->stream); + CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + cuEventRecord(pEnd, q->stream); + cuEventSynchronize(pEnd); + float ms = 0; + cuEventElapsedTime(&ms, pStart, pEnd); + kTime[k->name] += ms; + kCount[k->name]++; + if (!kRegs.count(k->name)) { + int regs = 0, shmem = 0; + cuFuncGetAttribute(®s, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); + cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); + kRegs[k->name] = regs; + kShmem[k->name] = shmem; + } + totalLaunches++; + + if (totalLaunches % 10000 == 0) { + fprintf(stderr, "\n=== NTT KERNEL PROFILE (%d launches) ===\n", totalLaunches); + std::vector> sorted; + double totalMs = 0; + for (auto& [n, t] : kTime) { sorted.push_back({t, n}); totalMs += t; } + std::sort(sorted.rbegin(), sorted.rend()); + for (auto& [t, n] : sorted) { + fprintf(stderr, " %6.1f ms (%5.1f%%) %5d calls avg %.3f ms regs=%d shmem=%d %s\n", + t, 100.0*t/totalMs, kCount[n], t/kCount[n], kRegs[n], kShmem[n], n.c_str()); + } + fprintf(stderr, " TOTAL: %.1f ms\n===\n\n", totalMs); + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; + } + + CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + if (r != CUDA_SUCCESS) { + const char* errName = nullptr; + cuGetErrorName(r, &errName); + fprintf(stderr, "cuLaunchKernel FAILED for '%s': %s (%d)\n", k->name.c_str(), errName ? errName : "?", (int)r); + } + + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueReadBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, + size_t offset, size_t size, void* ptr, + unsigned nWaits, const cl_event* waits, cl_event* event) { + // Must use stream-ordered copy because the stream was created with CU_STREAM_NON_BLOCKING, + // which means cuMemcpyDtoH (NULL stream) won't wait for pending kernels on this stream. + CUresult r = cuMemcpyDtoHAsync(ptr, buf->ptr + offset, size, q->stream); + if (r == CUDA_SUCCESS && blocking) { + r = cuStreamSynchronize(q->stream); + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueWriteBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, + size_t offset, size_t size, const void* ptr, + unsigned nWaits, const cl_event* waits, cl_event* event) { + // Must use stream-ordered copy (same reason as clEnqueueReadBuffer above) + CUresult r = cuMemcpyHtoDAsync(buf->ptr + offset, ptr, size, q->stream); + if (r == CUDA_SUCCESS && blocking) { + r = cuStreamSynchronize(q->stream); + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueCopyBuffer(cl_command_queue q, cl_mem src, cl_mem dst, + size_t srcOffset, size_t dstOffset, size_t size, + unsigned nWaits, const cl_event* waits, cl_event* event) { + CUresult r = cuMemcpyDtoDAsync(dst->ptr + dstOffset, src->ptr + srcOffset, size, q->stream); + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueFillBuffer(cl_command_queue q, cl_mem buf, const void* pattern, + size_t patternSize, size_t offset, size_t size, + unsigned nWaits, const cl_event* waits, cl_event* event) { + CUresult r; + if (patternSize == 1) { + unsigned char val; + memcpy(&val, pattern, 1); + r = cuMemsetD8Async(buf->ptr + offset, val, size, q->stream); + } else if (patternSize == 4) { + unsigned int val; + memcpy(&val, pattern, 4); + r = cuMemsetD32Async(buf->ptr + offset, val, size / 4, q->stream); + } else { + // For other pattern sizes, fall back to memset 0 (common case is zero-fill) + r = cuMemsetD8Async(buf->ptr + offset, 0, size, q->stream); + } + if (event) *event = nullptr; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clEnqueueMarkerWithWaitList(cl_command_queue q, unsigned nWaits, + const cl_event* waits, cl_event* event) { + if (event) { + auto* ev = new _cl_event; + cuEventCreate(&ev->end, CU_EVENT_DEFAULT); + cuEventRecord(ev->end, q->stream); + ev->commandType = CL_COMMAND_MARKER; + *event = ev; + } + return CL_SUCCESS; +} + +int clFlush(cl_command_queue q) { + // CUDA streams auto-flush; no-op + return CL_SUCCESS; +} + +int clFinish(cl_command_queue q) { + if (q) cuStreamSynchronize(q->stream); + return CL_SUCCESS; +} + +// ---- Events ---- + +int clReleaseEvent(cl_event ev) { + delete ev; + return CL_SUCCESS; +} + +int clWaitForEvents(unsigned n, const cl_event* events) { + for (unsigned i = 0; i < n; i++) { + if (events[i] && events[i]->end) { + cuEventSynchronize(events[i]->end); + } + } + return CL_SUCCESS; +} + +int clGetEventInfo(cl_event ev, cl_event_info info, size_t size, void* value, size_t* sizeRet) { + if (!ev) return CL_INVALID_VALUE; + if (info == CL_EVENT_COMMAND_EXECUTION_STATUS) { + int status = CL_COMPLETE; + if (ev->end) { + CUresult r = cuEventQuery(ev->end); + if (r == CUDA_ERROR_NOT_READY) status = CL_RUNNING; + } + if (sizeRet) *sizeRet = sizeof(int); + if (value && size >= sizeof(int)) memcpy(value, &status, sizeof(int)); + } else if (info == CL_EVENT_COMMAND_TYPE) { + u32 type = ev->commandType; + if (sizeRet) *sizeRet = sizeof(u32); + if (value && size >= sizeof(u32)) memcpy(value, &type, sizeof(u32)); + } + return CL_SUCCESS; +} + +int clGetEventProfilingInfo(cl_event ev, cl_profiling_info info, size_t size, void* value, size_t* sizeRet) { + if (!ev || !ev->hasTimings) return CL_PROFILING_INFO_NOT_AVAILABLE; + + // CUDA events give elapsed time between two events, not absolute timestamps. + // We fake absolute timestamps by using a base time. + u64 timestamp = 0; + if (info == CL_PROFILING_COMMAND_START || info == CL_PROFILING_COMMAND_SUBMIT || + info == CL_PROFILING_COMMAND_QUEUED) { + timestamp = 0; // Relative start + } else if (info == CL_PROFILING_COMMAND_END || info == CL_PROFILING_COMMAND_COMPLETE) { + float ms = 0; + cuEventElapsedTime(&ms, ev->start, ev->end); + timestamp = (u64)(ms * 1e6); // Convert ms to ns + } + + if (sizeRet) *sizeRet = sizeof(u64); + if (value && size >= sizeof(u64)) memcpy(value, ×tamp, sizeof(u64)); + return CL_SUCCESS; +} + +// ---- Device info ---- + +int clGetDeviceInfo(cl_device_id dev, cl_device_info info, size_t size, void* value, size_t* sizeRet) { + if (!dev) return CL_INVALID_DEVICE; + + switch (info) { + case CL_DEVICE_NAME: { + char name[256]; + cuDeviceGetName(name, sizeof(name), dev->dev); + size_t len = strlen(name) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, name, len); + break; + } + case CL_DEVICE_VENDOR_ID: { + // Return NVIDIA vendor ID + unsigned int vid = 0x10DE; + if (sizeRet) *sizeRet = sizeof(vid); + if (value && size >= sizeof(vid)) memcpy(value, &vid, sizeof(vid)); + break; + } + case CL_DEVICE_MAX_COMPUTE_UNITS: { + int units = 0; + cuDeviceGetAttribute(&units, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, dev->dev); + unsigned int val = units; + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DEVICE_MAX_CLOCK_FREQUENCY: { + int mhz = 0; + cuDeviceGetAttribute(&mhz, CU_DEVICE_ATTRIBUTE_CLOCK_RATE, dev->dev); + unsigned int val = mhz / 1000; // kHz to MHz + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DEVICE_GLOBAL_MEM_SIZE: { + size_t mem = 0; + cuDeviceTotalMem(&mem, dev->dev); + u64 val = mem; + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DRIVER_VERSION: + case CL_DEVICE_VERSION: { + int ver = 0; + cuDriverGetVersion(&ver); + char verStr[64]; + snprintf(verStr, sizeof(verStr), "CUDA %d.%d", ver / 1000, (ver % 1000) / 10); + size_t len = strlen(verStr) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, verStr, len); + break; + } + case CL_DEVICE_ERROR_CORRECTION_SUPPORT: { + int ecc = 0; + cuDeviceGetAttribute(&ecc, CU_DEVICE_ATTRIBUTE_ECC_ENABLED, dev->dev); + cl_bool val = ecc; + if (sizeRet) *sizeRet = sizeof(val); + if (value && size >= sizeof(val)) memcpy(value, &val, sizeof(val)); + break; + } + case CL_DEVICE_BUILT_IN_KERNELS: { + const char* empty = ""; + if (sizeRet) *sizeRet = 1; + if (value && size >= 1) memcpy(value, empty, 1); + break; + } + case CL_DEVICE_BOARD_NAME_AMD: + case CL_DEVICE_PCIE_ID_AMD: + case CL_DEVICE_TOPOLOGY_AMD: { + // AMD-specific queries — return failure + return CL_INVALID_VALUE; + } + case CL_DEVICE_GLOBAL_FREE_MEMORY_AMD: { + size_t freeMem = 0, totalMem = 0; + cuMemGetInfo(&freeMem, &totalMem); + // AMD returns in KB + u64 freeKB = freeMem / 1024; + if (sizeRet) *sizeRet = sizeof(freeKB); + if (value && size >= sizeof(freeKB)) memcpy(value, &freeKB, sizeof(freeKB)); + break; + } + default: + return CL_INVALID_VALUE; + } + return CL_SUCCESS; +} + +int clGetPlatformInfo(cl_platform_id, cl_device_info info, size_t size, void* value, size_t* sizeRet) { + if (info == CL_PLATFORM_VERSION) { + const char* ver = "CUDA (via PRPLL CUDA backend)"; + size_t len = strlen(ver) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, ver, len); + return CL_SUCCESS; + } + return CL_INVALID_VALUE; +} + +int clGetCommandQueueInfo(cl_command_queue q, cl_command_queue_info info, + size_t size, void* value, size_t* sizeRet) { + if (info == CL_QUEUE_CONTEXT) { + if (sizeRet) *sizeRet = sizeof(cl_context); + if (value && size >= sizeof(cl_context)) memcpy(value, &q->context, sizeof(cl_context)); + return CL_SUCCESS; + } + return CL_INVALID_VALUE; +} + +// ---- Kernel info ---- + +int clGetKernelInfo(cl_kernel k, cl_kernel_info info, size_t size, void* value, size_t* sizeRet) { + if (!k) return CL_INVALID_KERNEL; + if (info == CL_KERNEL_NUM_ARGS) { + int n = k->numArgs; + if (sizeRet) *sizeRet = sizeof(n); + if (value && size >= sizeof(n)) memcpy(value, &n, sizeof(n)); + } else if (info == CL_KERNEL_ATTRIBUTES) { + const char* empty = ""; + if (sizeRet) *sizeRet = 1; + if (value && size >= 1) memcpy(value, empty, 1); + } + return CL_SUCCESS; +} + +int clGetKernelArgInfo(cl_kernel k, unsigned pos, cl_kernel_arg_info info, + size_t size, void* value, size_t* sizeRet) { + if (info == CL_KERNEL_ARG_NAME) { + char name[32]; + snprintf(name, sizeof(name), "arg%u", pos); + size_t len = strlen(name) + 1; + if (sizeRet) *sizeRet = len; + if (value && size >= len) memcpy(value, name, len); + } + return CL_SUCCESS; +} + +int clGetKernelWorkGroupInfo(cl_kernel k, cl_device_id dev, cl_kernel_work_group_info info, + size_t size, void* value, size_t* sizeRet) { + if (!k) return CL_INVALID_KERNEL; + ensureContextCurrent(); + if (info == CL_KERNEL_COMPILE_WORK_GROUP_SIZE) { + // Return the __launch_bounds__ value parsed from source during clCreateKernel. + // This matches OpenCL's CL_KERNEL_COMPILE_WORK_GROUP_SIZE which returns reqd_work_group_size. + // Previously we used CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK which returns the hardware max + // based on register/shared memory usage — NOT the declared group size. This caused wrong + // block sizes for every kernel (e.g., tailMul expected 64 threads but got 1024). + int wgSize = k->reqWorkGroupSize > 0 ? k->reqWorkGroupSize : 256; + size_t wgs[3] = { (size_t)wgSize, 1, 1 }; + if (sizeRet) *sizeRet = sizeof(wgs); + if (value && size >= sizeof(wgs)) memcpy(value, wgs, sizeof(wgs)); + } + return CL_SUCCESS; +} + +// ---- SVM (not used but must exist) ---- + +void* clSVMAlloc(cl_context, cl_svm_mem_flags, size_t size, unsigned) { + CUdeviceptr ptr; + ensureContextCurrent(); + cuMemAlloc(&ptr, size); + return (void*)(uintptr_t)ptr; +} + +void clSVMFree(cl_context, void* ptr) { + ensureContextCurrent(); + cuMemFree((CUdeviceptr)(uintptr_t)ptr); +} + +int clSetKernelArgSVMPointer(cl_kernel k, unsigned pos, const void* ptr) { + CUdeviceptr dp = (CUdeviceptr)(uintptr_t)ptr; + k->setArg(pos, sizeof(dp), &dp); + return CL_SUCCESS; +} + +} // extern "C" + +// C++ linkage — must be outside the extern "C" block above. + +// Set L2 cache persistence for multiple read-only buffers on the given stream. +// Computes the minimum address span covering all buffers, then sets one access policy +// window with hitRatio sized so that only the actual buffer bytes get persisting treatment, +// not the gaps between non-contiguous allocations. +void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) { + if (!q) return; + + // Find address span and total data size + CUdeviceptr minAddr = ~(CUdeviceptr)0; + CUdeviceptr maxAddr = 0; + size_t totalDataBytes = 0; + + for (auto buf : buffers) { + if (!buf || buf->size == 0) continue; + CUdeviceptr lo = buf->ptr; + CUdeviceptr hi = buf->ptr + buf->size; + if (lo < minAddr) minAddr = lo; + if (hi > maxAddr) maxAddr = hi; + totalDataBytes += buf->size; + } + + if (totalDataBytes == 0 || maxAddr <= minAddr) return; + + size_t spanBytes = (size_t)(maxAddr - minAddr); + + // Query the device's max access policy window size + int maxWindowSize = 0; + cuDeviceGetAttribute(&maxWindowSize, CU_DEVICE_ATTRIBUTE_MAX_ACCESS_POLICY_WINDOW_SIZE, 0); + if (maxWindowSize > 0 && spanBytes > (size_t)maxWindowSize) { + fprintf(stderr, "L2 persist: span %zuMB exceeds max window %dMB, clamping\n", + spanBytes / (1024*1024), maxWindowSize / (1024*1024)); + spanBytes = maxWindowSize; + } + + // hitRatio = actual data / window span. This way only the real buffer data gets + // persisting treatment, and any gaps between allocations get streaming treatment. + float hitRatio = (float)totalDataBytes / (float)spanBytes; + if (hitRatio > 1.0f) hitRatio = 1.0f; + + CUstreamAttrValue attr; + memset(&attr, 0, sizeof(attr)); + attr.accessPolicyWindow.base_ptr = (void*)(uintptr_t)minAddr; + attr.accessPolicyWindow.num_bytes = spanBytes; + attr.accessPolicyWindow.hitRatio = hitRatio; + attr.accessPolicyWindow.hitProp = CU_ACCESS_PROPERTY_PERSISTING; + attr.accessPolicyWindow.missProp = CU_ACCESS_PROPERTY_STREAMING; + + CUresult r = cuStreamSetAttribute(q->stream, CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW, &attr); + if (r != CUDA_SUCCESS) { + fprintf(stderr, "L2 persist: cuStreamSetAttribute failed (%d)\n", (int)r); + } else { + fprintf(stderr, "L2 persist: window %zuMB (%.1f%% hit ratio), %zuMB actual data, %zu buffers\n", + spanBytes / (1024*1024), hitRatio * 100.0f, totalDataBytes / (1024*1024), + buffers.size()); + } +} + diff --git a/src/cuda/cudawrap.cpp b/src/cuda/cudawrap.cpp new file mode 100644 index 00000000..905e34c7 --- /dev/null +++ b/src/cuda/cudawrap.cpp @@ -0,0 +1,505 @@ +// CUDA Driver API wrappers implementation + +#include "cudawrap.h" + +#include +#include +#include +#include +#include + +// Local log function — avoids dependency on PRPLL's log.h/File.h chain +static void cuda_log(const char* fmt, ...) { + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); +} + +// ---- Error checking ---- + +void checkCuda(CUresult err, const char* file, int line, const char* func, const char* expr) { + if (err != CUDA_SUCCESS) { + const char* errName = nullptr; + const char* errStr = nullptr; + cuGetErrorName(err, &errName); + cuGetErrorString(err, &errStr); + char buf[512]; + snprintf(buf, sizeof(buf), "CUDA error %d (%s): %s at %s:%d in %s: %s", + (int)err, errName ? errName : "?", errStr ? errStr : "?", + file, line, func, expr); + cuda_log("%s\n", buf); + throw std::runtime_error(buf); + } +} + +void checkNvrtc(nvrtcResult err, const char* file, int line, const char* func, const char* expr) { + if (err != NVRTC_SUCCESS) { + char buf[512]; + snprintf(buf, sizeof(buf), "NVRTC error %d (%s) at %s:%d in %s: %s", + (int)err, nvrtcGetErrorString(err), file, line, func, expr); + cuda_log("%s\n", buf); + throw std::runtime_error(buf); + } +} + +// ---- Device management ---- + +std::vector getAllDevices() { + CU_CHECK(cuInit(0)); + int count = 0; + CU_CHECK(cuDeviceGetCount(&count)); + std::vector devices(count); + for (int i = 0; i < count; ++i) { + CU_CHECK(cuDeviceGet(&devices[i], i)); + } + return devices; +} + +std::string getDeviceName(CUdevice dev) { + char name[256]; + CU_CHECK(cuDeviceGetName(name, sizeof(name), dev)); + return name; +} + +std::string getDriverVersion() { + int ver = 0; + CU_CHECK(cuDriverGetVersion(&ver)); + char buf[32]; + snprintf(buf, sizeof(buf), "%d.%d", ver / 1000, (ver % 1000) / 10); + return buf; +} + +float getGpuRamGB(CUdevice dev) { + size_t bytes = 0; + CU_CHECK(cuDeviceTotalMem(&bytes, dev)); + return bytes / (1024.0f * 1024.0f * 1024.0f); +} + +u64 getFreeMem(CUdevice dev) { + // Need a context to query free memory + size_t free_bytes = 0, total = 0; + CU_CHECK(cuMemGetInfo(&free_bytes, &total)); + return free_bytes; +} + +std::string getShortInfo(CUdevice dev) { + int major = 0, minor = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev); + cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev); + char buf[256]; + snprintf(buf, sizeof(buf), "%s (sm_%d%d, %.1f GB)", getDeviceName(dev).c_str(), + major, minor, getGpuRamGB(dev)); + return buf; +} + +// ---- Context ---- + +CudaContext::CudaContext(CUdevice dev) : device(dev) { +#if CUDA_VERSION >= 13000 + CUctxCreateParams params{}; + CU_CHECK(cuCtxCreate_v4(&ctx, ¶ms, CU_CTX_SCHED_BLOCKING_SYNC, dev)); +#else + CU_CHECK(cuCtxCreate(&ctx, CU_CTX_SCHED_BLOCKING_SYNC, dev)); +#endif +} + +CudaContext::~CudaContext() { + if (ctx) cuCtxDestroy(ctx); +} + +void CudaContext::makeCurrent() { + CU_CHECK(cuCtxSetCurrent(ctx)); +} + +// ---- Module ---- + +CudaModule::CudaModule(const std::string& ptx, const std::string& name) { + CUjit_option options[] = { CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES, CU_JIT_ERROR_LOG_BUFFER }; + char errorLog[4096] = {}; + void* optionValues[] = { (void*)(size_t)sizeof(errorLog), (void*)errorLog }; + + CUresult err = cuModuleLoadDataEx(&module, ptx.c_str(), 2, options, optionValues); + if (err != CUDA_SUCCESS) { + cuda_log("Module load error for '%s': %s\n", name.c_str(), errorLog); + checkCuda(err, __FILE__, __LINE__, __func__, "cuModuleLoadDataEx"); + } +} + +CudaModule::~CudaModule() { + if (module) cuModuleUnload(module); +} + +CUfunction CudaModule::getFunction(const char* name) const { + CUfunction func{}; + CU_CHECK(cuModuleGetFunction(&func, module, name)); + return func; +} + +// ---- Stream ---- + +CudaStream::CudaStream() { + CU_CHECK(cuStreamCreate(&stream, CU_STREAM_DEFAULT)); +} + +CudaStream::~CudaStream() { + if (stream) cuStreamDestroy(stream); +} + +void CudaStream::sync() { + CU_CHECK(cuStreamSynchronize(stream)); +} + +// ---- Buffer ---- + +CudaBuffer::CudaBuffer(size_t bytes) : bytes(bytes) { + if (bytes > 0) { + CU_CHECK(cuMemAlloc(&ptr, bytes)); + } +} + +CudaBuffer::~CudaBuffer() { + if (ptr) cuMemFree(ptr); +} + +void CudaBuffer::readSync(void* dst, size_t n) const { + assert(n <= bytes); + CU_CHECK(cuMemcpyDtoH(dst, ptr, n)); +} + +void CudaBuffer::writeSync(const void* src, size_t n) { + assert(n <= bytes); + CU_CHECK(cuMemcpyHtoD(ptr, src, n)); +} + +void CudaBuffer::zero() { + if (bytes > 0) { + CU_CHECK(cuMemsetD8(ptr, 0, bytes)); + } +} + +void CudaBuffer::copyFrom(const CudaBuffer& src) { + assert(bytes == src.bytes); + CU_CHECK(cuMemcpyDtoD(ptr, src.ptr, bytes)); +} + +void CudaBuffer::fillPattern(const void* pattern, size_t patternSize) { + if (patternSize == 4) { + u32 val; + memcpy(&val, pattern, 4); + CU_CHECK(cuMemsetD32(ptr, val, bytes / 4)); + } else if (patternSize == 1) { + u8 val; + memcpy(&val, pattern, 1); + CU_CHECK(cuMemsetD8(ptr, val, bytes)); + } else { + // For other sizes, use cuMemsetD32 with multiple passes or copy pattern manually + // This is rarely used + assert(false && "fillPattern: unsupported pattern size"); + } +} + +// ---- NVRTC Compilation ---- + +// Helper: skip balanced parentheses starting at '(' at position i +static size_t skipBalancedParens(const std::string& s, size_t i) { + if (i >= s.size() || s[i] != '(') return i; + int depth = 1; + i++; + while (i < s.size() && depth > 0) { + if (s[i] == '(') depth++; + else if (s[i] == ')') depth--; + i++; + } + return i; +} + +// Strip OpenCL-specific constructs that NVRTC can't handle: +// 1. __global/global pointer qualifiers (without breaking __global__) +// 2. #pragma OPENCL ... directives +// 3. __attribute__((overloadable)) and __attribute__((reqd_work_group_size(...))) +// 4. OpenCL vector cast syntax: (type2)(a, b) → make_type2(a, b) [deferred to compat header] +std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { + std::string result; + result.reserve(source.size()); + size_t i = 0; + while (i < source.size()) { + // Strip #pragma OPENCL ... lines + if (i + 14 <= source.size() && source.compare(i, 14, "#pragma OPENCL") == 0) { + bool atLineStart = (i == 0 || source[i-1] == '\n'); + if (atLineStart) { + while (i < source.size() && source[i] != '\n') i++; + result += "// [stripped pragma]"; + continue; + } + } + + // Replace base.cl's KERNEL macro with CUDA-compatible version + // OpenCL: #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void + // CUDA: #define KERNEL(x) extern "C" __global__ void __launch_bounds__(x) + if (i + 15 <= source.size() && source.compare(i, 15, "#define KERNEL(") == 0) { + bool atLineStart = (i == 0 || source[i-1] == '\n'); + if (atLineStart) { + while (i < source.size() && source[i] != '\n') i++; + result += "#ifdef CUDA_MIN_BLOCKS\n"; + result += "#define KERNEL(x) extern \"C\" __global__ void __launch_bounds__(x, CUDA_MIN_BLOCKS)\n"; + result += "#else\n"; + result += "#define KERNEL(x) extern \"C\" __global__ void __launch_bounds__(x)\n"; + result += "#endif"; + continue; + } + } + + // Strip base.cl's OpenCL typedefs that conflict with opencl_compat.cuh + // Only strip exact OpenCL patterns (using OpenCL types like 'long', 'ulong', 'uint') + // NOT our compat header's versions (which use 'long long', 'unsigned long long', etc.) + if (i + 7 <= source.size() && source.compare(i, 7, "typedef") == 0) { + bool atLineStart = (i == 0 || source[i-1] == '\n'); + if (atLineStart) { + size_t lineEnd = source.find('\n', i); + if (lineEnd == std::string::npos) lineEnd = source.size(); + std::string line = source.substr(i, lineEnd - i); + // Strip trailing whitespace/CR for comparison + while (!line.empty() && (line.back() == ' ' || line.back() == '\r')) line.pop_back(); + // Only strip the EXACT OpenCL patterns from base.cl: + if (line == "typedef int i32;" || + line == "typedef uint u32;" || + line == "typedef long i64;" || + line == "typedef ulong u64;") { + result += "// [stripped OpenCL typedef]"; + i = lineEnd; + continue; + } + } + } + + // Convert vector array access: .a[0] → .a.x, .a[1] → .a.y + // OpenCL allows vec2[i] indexing; CUDA uses .x/.y member access. + // Used in fftp.cl: union { uint2 a; u64 b; } m31_combo; #define frac_bits m31_combo.a[0] + if (i + 5 <= source.size() && source.compare(i, 5, ".a[0]") == 0) { + result += ".a.x"; + i += 5; + continue; + } + if (i + 5 <= source.size() && source.compare(i, 5, ".a[1]") == 0) { + result += ".a.y"; + i += 5; + continue; + } + + // Handle __attribute__((...)) for overloadable and reqd_work_group_size + if (i + 15 <= source.size() && source.compare(i, 15, "__attribute__((") == 0) { + size_t nameStart = i + 15; + if (nameStart + 12 <= source.size() && source.compare(nameStart, 12, "overloadable") == 0) { + // Strip __attribute__((overloadable)) — C++ has native overloading + i = skipBalancedParens(source, i + 13); + while (i < source.size() && source[i] == ' ') i++; + continue; + } + if (nameStart + 20 <= source.size() && source.compare(nameStart, 20, "reqd_work_group_size") == 0) { + // Convert __attribute__((reqd_work_group_size(N, 1, 1))) → __launch_bounds__(N) + // Extract the first number from the args + size_t argsStart = nameStart + 20; + while (argsStart < source.size() && source[argsStart] == '(') argsStart++; + size_t numEnd = argsStart; + while (numEnd < source.size() && source[numEnd] >= '0' && source[numEnd] <= '9') numEnd++; + if (numEnd > argsStart) { + std::string wgSize = source.substr(argsStart, numEnd - argsStart); + result += "__launch_bounds__(" + wgSize + ") "; + } + // Skip past the entire __attribute__((...)) + i = skipBalancedParens(source, i + 13); + while (i < source.size() && source[i] == ' ') i++; + continue; + } + } + + // Convert OpenCL vector cast syntax: (type2)(a, b) → make_type2(a, b) + // Matches: (double2), (float2), (int2), (uint2), (long2), (ulong2) + if (source[i] == '(' && i + 1 < source.size()) { + static const char* vecTypes[] = { + "double2)", "float2)", "int2)", "uint2)", "long2)", "ulong2)", "Word2)", nullptr + }; + static const char* makeNames[] = { + "make_double2", "make_float2", "make_int2", "make_uint2", "make_long2", "make_ulong2", "make_Word2" + }; + bool matched = false; + for (int vi = 0; vecTypes[vi]; vi++) { + size_t tlen = strlen(vecTypes[vi]); + if (i + 1 + tlen <= source.size() && source.compare(i + 1, tlen, vecTypes[vi]) == 0) { + // Check what follows: should be whitespace then '(' for a cast constructor + size_t after = i + 1 + tlen; + while (after < source.size() && source[after] == ' ') after++; + if (after < source.size() && source[after] == '(') { + // It's (type2) (args) — replace with make_type2(args) + result += makeNames[vi]; + i = after; // now pointing at '(' of args + matched = true; + break; + } + // Also handle (type2){args} — less common but possible + if (after < source.size() && source[after] == '{') { + result += makeNames[vi]; + result += '('; + i = after + 1; // skip '{' + // Find matching '}' and replace with ')' + int depth = 1; + while (i < source.size() && depth > 0) { + if (source[i] == '{') depth++; + else if (source[i] == '}') { depth--; if (depth == 0) { result += ')'; i++; break; } } + else result += source[i]; + i++; + } + matched = true; + break; + } + } + } + if (matched) continue; + } + + // Convert "local TYPE NAME[" to "__shared__ TYPE NAME[" for shared memory declarations + // This handles: " local T2 lds[WIDTH / 4];" → " __shared__ T2 lds[WIDTH / 4];" + // Also handles: " local T lds[IN_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)];" + // But NOT: "local T2 *lds" in function params (which becomes just "T2 *lds" via macro) + if (i + 6 <= source.size() && source.compare(i, 6, "local ") == 0) { + bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + if (!preceded) { + // Check if this "local" is followed by a type then a name then '[' + // i.e., it's a shared memory array declaration + size_t lineEnd = source.find('\n', i); + if (lineEnd == std::string::npos) lineEnd = source.size(); + std::string line = source.substr(i, lineEnd - i); + // Match: "local TYPE IDENT[" pattern — indicates array declaration + // Array declarations have '[' and end with ';'. They may also contain '(' in + // the array size expression (e.g., ternary operators). The key distinction is + // that function parameters don't have '['. + if (line.find('[') != std::string::npos) { + result += "__shared__ "; + i += 6; // skip "local " + continue; + } + // For everything else (params, casts), just skip "local " → empty + i += 6; + continue; + } + } + // Same for __local + if (i + 8 <= source.size() && source.compare(i, 8, "__local ") == 0) { + bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + if (!preceded) { + size_t lineEnd = source.find('\n', i); + if (lineEnd == std::string::npos) lineEnd = source.size(); + std::string line = source.substr(i, lineEnd - i); + if (line.find('[') != std::string::npos) { + result += "__shared__ "; + i += 8; + continue; + } + i += 8; + continue; + } + } + + // Replace "n" constraint with "r" in PTX asm (NVRTC requires true constants for "n") + // Match: "n"( → "r"( + if (i + 3 <= source.size() && source.compare(i, 3, "\"n\"") == 0) { + // Check we're inside an asm statement context (look for preceding ':') + // Simple heuristic: if the previous non-whitespace char is ':', ':' + space, or ',' + size_t back = i; + while (back > 0 && (source[back-1] == ' ' || source[back-1] == '\t')) back--; + if (back > 0 && (source[back-1] == ':' || source[back-1] == ',')) { + result += "\"r\""; + i += 3; + continue; + } + } + + // Match __global NOT followed by _ + if (i + 8 <= source.size() && source.compare(i, 8, "__global") == 0) { + if (i + 8 < source.size() && source[i + 8] == '_') { + result += source[i++]; + } else { + bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + if (preceded) { + result += source[i++]; + } else { + i += 8; + } + } + } + // Match standalone "global" (not inside a word or PTX instruction) + else if (i + 6 <= source.size() && source.compare(i, 6, "global") == 0) { + bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_' || source[i-1] == '.')); + bool followed = (i + 6 < source.size() && (isalnum(source[i + 6]) || source[i + 6] == '_')); + if (!preceded && !followed) { + i += 6; + } else { + result += source[i++]; + } + } else { + result += source[i++]; + } + } + return result; +} + +std::string NvrtcProgram::compile(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers) { + // Prepare header arrays + std::vector headerSources, headerNames; + for (auto& [hName, hSource] : headers) { + headerNames.push_back(hName.c_str()); + headerSources.push_back(hSource.c_str()); + } + + nvrtcProgram prog; + NVRTC_CHECK(nvrtcCreateProgram(&prog, source.c_str(), name.c_str(), + (int)headers.size(), + headerSources.empty() ? nullptr : headerSources.data(), + headerNames.empty() ? nullptr : headerNames.data())); + + // Convert options to char* + std::vector opts; + for (auto& o : options) opts.push_back(o.c_str()); + + nvrtcResult compileResult = nvrtcCompileProgram(prog, (int)opts.size(), opts.data()); + + // Get compilation log + size_t logSize; + nvrtcGetProgramLogSize(prog, &logSize); + if (logSize > 1) { + std::string compileLog(logSize, '\0'); + nvrtcGetProgramLog(prog, compileLog.data()); + if (compileResult != NVRTC_SUCCESS) { + cuda_log("NVRTC compile error for '%s':\n%s\n", name.c_str(), compileLog.c_str()); + } + } + + if (compileResult != NVRTC_SUCCESS) { + nvrtcDestroyProgram(&prog); + throw std::runtime_error("NVRTC compilation failed for " + name); + } + + // Get PTX + size_t ptxSize; + NVRTC_CHECK(nvrtcGetPTXSize(prog, &ptxSize)); + std::string ptx(ptxSize, '\0'); + NVRTC_CHECK(nvrtcGetPTX(prog, ptx.data())); + + nvrtcDestroyProgram(&prog); + return ptx; +} + +// ---- Kernel launcher ---- + +void CudaKernelLauncher::launch(CUstream stream, u32 gridSize, void** args, u32 sharedMem) { + CU_CHECK(cuLaunchKernel(func, + gridSize, 1, 1, // grid dimensions + blockSize, 1, 1, // block dimensions + sharedMem, // shared memory bytes + stream, // stream + args, // kernel arguments + nullptr)); // extra +} diff --git a/src/cuda/cudawrap.h b/src/cuda/cudawrap.h new file mode 100644 index 00000000..bd0e6e46 --- /dev/null +++ b/src/cuda/cudawrap.h @@ -0,0 +1,138 @@ +// CUDA Driver API wrappers, parallel to clwrap.h for OpenCL. +// Uses the CUDA Driver API (cu*) for maximum control. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// Types — when building standalone tests, define minimal types. +// When building as part of PRPLL, common.h is already included. +#if __has_include("../common.h") && !defined(CUDAWRAP_STANDALONE) +#include "../common.h" +#else +#ifndef CUDAWRAP_TYPES_DEFINED +#define CUDAWRAP_TYPES_DEFINED +typedef unsigned int u32; +typedef unsigned long long u64; +typedef unsigned char u8; +#endif +#endif + +// ---- Error checking ---- +void checkCuda(CUresult err, const char* file, int line, const char* func, const char* expr); +void checkNvrtc(nvrtcResult err, const char* file, int line, const char* func, const char* expr); + +#define CU_CHECK(expr) checkCuda((expr), __FILE__, __LINE__, __func__, #expr) +#define NVRTC_CHECK(expr) checkNvrtc((expr), __FILE__, __LINE__, __func__, #expr) + +// ---- Device management ---- +std::vector getAllDevices(); +std::string getDeviceName(CUdevice dev); +std::string getDriverVersion(); +float getGpuRamGB(CUdevice dev); +u64 getFreeMem(CUdevice dev); +std::string getShortInfo(CUdevice dev); + +// ---- Context ---- +class CudaContext { + CUcontext ctx{}; + CUdevice device{}; +public: + explicit CudaContext(CUdevice dev); + ~CudaContext(); + + CUcontext get() const { return ctx; } + CUdevice getDevice() const { return device; } + void makeCurrent(); +}; + +// ---- Module (compiled kernels) ---- +class CudaModule { + CUmodule module{}; +public: + CudaModule() = default; + explicit CudaModule(const std::string& ptx, const std::string& name = ""); + ~CudaModule(); + + CudaModule(CudaModule&& rhs) noexcept : module(rhs.module) { rhs.module = nullptr; } + CudaModule& operator=(CudaModule&& rhs) noexcept { + if (this != &rhs) { if (module) cuModuleUnload(module); module = rhs.module; rhs.module = nullptr; } + return *this; + } + + CUmodule get() const { return module; } + CUfunction getFunction(const char* name) const; +}; + +// ---- Stream (equivalent to cl_command_queue) ---- +class CudaStream { + CUstream stream{}; +public: + CudaStream(); + ~CudaStream(); + + CUstream get() const { return stream; } + void sync(); + + CudaStream(CudaStream&& rhs) noexcept : stream(rhs.stream) { rhs.stream = nullptr; } + CudaStream& operator=(CudaStream&&) = delete; +}; + +// ---- Memory buffer ---- +class CudaBuffer { + CUdeviceptr ptr{}; + size_t bytes{}; +public: + CudaBuffer() = default; + explicit CudaBuffer(size_t bytes); + ~CudaBuffer(); + + CudaBuffer(CudaBuffer&& rhs) noexcept : ptr(rhs.ptr), bytes(rhs.bytes) { rhs.ptr = 0; rhs.bytes = 0; } + CudaBuffer& operator=(CudaBuffer&& rhs) noexcept { + if (this != &rhs) { if (ptr) cuMemFree(ptr); ptr = rhs.ptr; bytes = rhs.bytes; rhs.ptr = 0; rhs.bytes = 0; } + return *this; + } + + CUdeviceptr get() const { return ptr; } + size_t size() const { return bytes; } + + void readSync(void* dst, size_t n) const; + void writeSync(const void* src, size_t n); + void zero(); + void copyFrom(const CudaBuffer& src); + void fillPattern(const void* pattern, size_t patternSize); +}; + +// ---- NVRTC Compilation ---- +struct NvrtcProgram { + // Preprocess OpenCL source for CUDA: strip __global/global pointer qualifiers + // without breaking __global__ (CUDA kernel qualifier). + static std::string preprocessOpenCL(const std::string& source); + + static std::string compile(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers = {}); +}; + +// ---- Kernel launcher ---- +class CudaKernelLauncher { + CUfunction func{}; + std::string name; + u32 blockSize{}; + +public: + CudaKernelLauncher() = default; + CudaKernelLauncher(CUfunction f, const std::string& name, u32 blockSize) + : func(f), name(name), blockSize(blockSize) {} + + void launch(CUstream stream, u32 gridSize, void** args, u32 sharedMem = 0); + + CUfunction get() const { return func; } + const std::string& getName() const { return name; } +}; diff --git a/src/cuda/opencl_compat.cuh b/src/cuda/opencl_compat.cuh new file mode 100644 index 00000000..3316c26f --- /dev/null +++ b/src/cuda/opencl_compat.cuh @@ -0,0 +1,365 @@ +// OpenCL → CUDA compatibility header for PRPLL +// Allows .cl kernel files to compile under NVRTC with minimal changes. +// Used together with NvrtcProgram::preprocessOpenCL() which strips: +// - __global/global pointer qualifiers (can't #define without breaking __global__) +// - #pragma OPENCL ... directives +// - __attribute__((overloadable)) and __attribute__((reqd_work_group_size(...))) + +#pragma once + +// ---- Qualifiers ---- +// __kernel / kernel → extern "C" __global__ (CUDA kernel launch qualifier) +// extern "C" is needed so cuModuleGetFunction() can find kernels by unmangled name +#define __kernel extern "C" __global__ +#define kernel extern "C" __global__ + +// __local / local — OpenCL address space qualifier for shared memory. +// In CUDA, __shared__ can only be used on variable declarations, NOT on function parameters. +// The preprocessor handles this: it strips "local" from function parameter lists +// and adds __shared__ for variable declarations matching "local TYPE NAME[". +#define __local +#define local + +// __constant / constant → const (CUDA __constant__ is file-scope only, can't be used +// for kernel params). The compiler auto-uses __ldg() for const pointers on sm_35+. +#define __constant const +#define constant const + +// restrict → __restrict__ (different keyword in CUDA) +#define restrict __restrict__ + +// ---- Work-item functions ---- +#define get_local_id(d) ((unsigned int)threadIdx.x) +#define get_group_id(d) ((unsigned int)blockIdx.x) +#define get_local_size(d) ((unsigned int)blockDim.x) +#define get_global_id(d) ((unsigned int)(blockIdx.x * blockDim.x + threadIdx.x)) +#define get_num_groups(d) ((unsigned int)gridDim.x) +#define get_global_size(d) ((unsigned int)(gridDim.x * blockDim.x)) +#define get_enqueued_local_size(d) get_local_size(d) + +// ---- Barriers ---- +#define CLK_LOCAL_MEM_FENCE 0 +#define CLK_GLOBAL_MEM_FENCE 0 +#define barrier(flags) __syncthreads() + +// ---- Memory fences ---- +// OpenCL write_mem_fence / read_mem_fence → CUDA __threadfence() +#define write_mem_fence(flags) __threadfence() +#define read_mem_fence(flags) __threadfence() +#define mem_fence(flags) __threadfence() + +// ---- Overloadable ---- +// CUDA C++ supports function overloading natively +#define OVERLOAD + +// ---- OpenCL extension macros ---- +#define cl_khr_fp64 1 +#define cl_khr_subgroups 1 + +// ---- Kernel macro ---- +// PRPLL uses KERNEL(WG_SIZE) void kernelName(...) +// base.cl defines: #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void +// Our preprocessor replaces base.cl's KERNEL macro with a CUDA version. +// This fallback is only used if base.cl hasn't been included yet. +// KERNEL macro fallback — usually overridden by base.cl KERNEL macro replacement in cudawrap.cpp +#define KERNEL(x) extern "C" __global__ void __launch_bounds__(x) + +// ---- Pointer macros ---- +#define P(x) x* __restrict__ +#define CP(x) const x* __restrict__ + +// ---- OpenCL type aliases ---- +// CRITICAL: On Linux x86_64, CUDA's built-in vector types (long2, ulong2) use +// 'long' / 'unsigned long' for their members. These are DISTINCT C++ types from +// 'long long' / 'unsigned long long' even though both are 64-bit. +// We MUST use 'unsigned long' for ulong so that Z61 (typedef'd from ulong) matches +// ulong2 member types. Otherwise overloaded functions like add(Z31,Z31) vs add(Z61,Z61) +// become ambiguous when called with ulong2 member values (which are 'unsigned long'). +typedef unsigned long ulong; +typedef long slong; // OpenCL's signed 'long' (64-bit) + +// Standard PRPLL type aliases +typedef unsigned int uint; + +// These match OpenCL's types exactly. The preprocessor strips base.cl's +// re-definitions of i32/u32/i64/u64 to avoid redeclaration errors. +// Must use 'long' / 'unsigned long' to match CUDA vector type members. +typedef unsigned int u32; +typedef int i32; +typedef unsigned long u64; +typedef long i64; + +// ---- Math constants ---- +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif +#ifndef M_SQRT1_2 +#define M_SQRT1_2 0.70710678118654752440 +#endif + +// ---- Vector arithmetic operators ---- +// OpenCL supports +, -, *, / on vector types natively. CUDA does not. +// double2 +__device__ __forceinline__ double2 operator+(double2 a, double2 b) { return make_double2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ double2 operator-(double2 a, double2 b) { return make_double2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ double2 operator*(double2 a, double2 b) { return make_double2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ double2 operator-(double2 a) { return make_double2(-a.x, -a.y); } +__device__ __forceinline__ double2 operator*(double s, double2 a) { return make_double2(s*a.x, s*a.y); } +__device__ __forceinline__ double2 operator*(double2 a, double s) { return make_double2(a.x*s, a.y*s); } +__device__ __forceinline__ double2& operator+=(double2& a, double2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ double2& operator-=(double2& a, double2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// float2 +__device__ __forceinline__ float2 operator+(float2 a, float2 b) { return make_float2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ float2 operator-(float2 a, float2 b) { return make_float2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ float2 operator*(float2 a, float2 b) { return make_float2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ float2 operator-(float2 a) { return make_float2(-a.x, -a.y); } +__device__ __forceinline__ float2 operator*(float s, float2 a) { return make_float2(s*a.x, s*a.y); } +__device__ __forceinline__ float2 operator*(float2 a, float s) { return make_float2(a.x*s, a.y*s); } +__device__ __forceinline__ float2& operator+=(float2& a, float2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ float2& operator-=(float2& a, float2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// int2 +__device__ __forceinline__ int2 operator+(int2 a, int2 b) { return make_int2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ int2 operator-(int2 a, int2 b) { return make_int2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ int2 operator*(int2 a, int2 b) { return make_int2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ int2 operator-(int2 a) { return make_int2(-a.x, -a.y); } +__device__ __forceinline__ int2& operator+=(int2& a, int2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ int2& operator-=(int2& a, int2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// uint2 +__device__ __forceinline__ uint2 operator+(uint2 a, uint2 b) { return make_uint2(a.x+b.x, a.y+b.y); } +__device__ __forceinline__ uint2 operator-(uint2 a, uint2 b) { return make_uint2(a.x-b.x, a.y-b.y); } +__device__ __forceinline__ uint2 operator*(uint2 a, uint2 b) { return make_uint2(a.x*b.x, a.y*b.y); } +__device__ __forceinline__ uint2& operator+=(uint2& a, uint2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ uint2& operator-=(uint2& a, uint2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// long2 +__device__ __forceinline__ long2 operator+(long2 a, long2 b) { return {a.x+b.x, a.y+b.y}; } +__device__ __forceinline__ long2 operator-(long2 a, long2 b) { return {a.x-b.x, a.y-b.y}; } +__device__ __forceinline__ long2 operator*(long2 a, long2 b) { return {a.x*b.x, a.y*b.y}; } +__device__ __forceinline__ long2 operator-(long2 a) { return {-a.x, -a.y}; } +__device__ __forceinline__ long2& operator+=(long2& a, long2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ long2& operator-=(long2& a, long2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// ulong2 +__device__ __forceinline__ ulong2 operator+(ulong2 a, ulong2 b) { return {a.x+b.x, a.y+b.y}; } +__device__ __forceinline__ ulong2 operator-(ulong2 a, ulong2 b) { return {a.x-b.x, a.y-b.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 a, ulong2 b) { return {a.x*b.x, a.y*b.y}; } +__device__ __forceinline__ ulong2& operator+=(ulong2& a, ulong2 b) { a.x+=b.x; a.y+=b.y; return a; } +__device__ __forceinline__ ulong2& operator-=(ulong2& a, ulong2 b) { a.x-=b.x; a.y-=b.y; return a; } + +// Scalar * vector operators for types not built-in to NVRTC +// (NVRTC already provides double2*double, int2*int, uint2*uint, etc.) +// These cover cross-type scalar*vector that OpenCL supports natively. +__device__ __forceinline__ long2 operator*(long long s, long2 v) { return {s*v.x, s*v.y}; } +__device__ __forceinline__ long2 operator*(long2 v, long long s) { return {v.x*s, v.y*s}; } +__device__ __forceinline__ ulong2 operator*(unsigned long long s, ulong2 v) { return {s*v.x, s*v.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 v, unsigned long long s) { return {v.x*s, v.y*s}; } +// int * ulong2 (common in NTT code: int literal * GF61) +__device__ __forceinline__ ulong2 operator*(int s, ulong2 v) { return {(unsigned long long)s*v.x, (unsigned long long)s*v.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 v, int s) { return {v.x*(unsigned long long)s, v.y*(unsigned long long)s}; } + +// ---- Vector constructors (U2) ---- +// OpenCL (type2)(a,b) cast syntax is converted to make_type2(a,b) by the preprocessor. +// base.cl defines U2() as overloaded functions — they'll work after preprocessing. +// NVRTC provides make_double2, make_float2, make_int2, make_uint2, make_long2, make_ulong2 built-in. + +// ---- Type reinterpretation (as_*) ---- +// Scalar ↔ vector bitwise reinterpretations (OpenCL as_type functions) + +// as_uint2: split 64-bit value into two 32-bit halves +__device__ __forceinline__ uint2 as_uint2(double v) { + unsigned long long bits = __double_as_longlong(v); + return make_uint2((unsigned int)(bits), (unsigned int)(bits >> 32)); +} +__device__ __forceinline__ uint2 as_uint2(unsigned long long v) { + return make_uint2((unsigned int)(v), (unsigned int)(v >> 32)); +} +__device__ __forceinline__ uint2 as_uint2(unsigned long v) { + return make_uint2((unsigned int)(v), (unsigned int)((unsigned long long)v >> 32)); +} +__device__ __forceinline__ uint2 as_uint2(long long v) { + return make_uint2((unsigned int)((unsigned long long)v), (unsigned int)((unsigned long long)v >> 32)); +} +__device__ __forceinline__ uint2 as_uint2(long v) { + return make_uint2((unsigned int)((unsigned long)v), (unsigned int)((unsigned long long)v >> 32)); +} + +// as_int2: split 64-bit value into two signed 32-bit halves +__device__ __forceinline__ int2 as_int2(double v) { + unsigned long long bits = __double_as_longlong(v); + return make_int2((int)(unsigned int)(bits), (int)(unsigned int)(bits >> 32)); +} +__device__ __forceinline__ int2 as_int2(long long v) { + return make_int2((int)(unsigned int)((unsigned long long)v), (int)(unsigned int)((unsigned long long)v >> 32)); +} +__device__ __forceinline__ int2 as_int2(long v) { + return make_int2((int)(unsigned int)((unsigned long)v), (int)(unsigned int)((unsigned long long)v >> 32)); +} + +// as_double: reinterpret bits as double +__device__ __forceinline__ double as_double(int2 v) { + unsigned long long bits = ((unsigned long long)(unsigned int)v.y << 32) | (unsigned int)v.x; + return __longlong_as_double(bits); +} +__device__ __forceinline__ double as_double(uint2 v) { + unsigned long long bits = ((unsigned long long)v.y << 32) | v.x; + return __longlong_as_double(bits); +} +__device__ __forceinline__ double as_double(unsigned long long v) { + return __longlong_as_double(v); +} +__device__ __forceinline__ double as_double(unsigned long v) { + return __longlong_as_double((unsigned long long)v); +} +__device__ __forceinline__ double as_double(long long v) { return __longlong_as_double(v); } +__device__ __forceinline__ double as_double(long v) { return __longlong_as_double((long long)v); } + +// as_ulong: reinterpret as unsigned 64-bit (returns 'unsigned long' to match ulong typedef) +__device__ __forceinline__ unsigned long as_ulong(uint2 v) { + return (unsigned long)(((unsigned long long)v.y << 32) | v.x); +} +__device__ __forceinline__ unsigned long as_ulong(int2 v) { + return (unsigned long)(((unsigned long long)(unsigned int)v.y << 32) | (unsigned int)v.x); +} +__device__ __forceinline__ unsigned long as_ulong(double v) { + return (unsigned long)__double_as_longlong(v); +} + +// as_long: reinterpret as signed 64-bit (returns 'long' to match slong/i64) +__device__ __forceinline__ long as_long(int2 v) { + return (long)(((unsigned long long)(unsigned int)v.y << 32) | (unsigned int)v.x); +} +__device__ __forceinline__ long as_long(uint2 v) { + return (long)(((unsigned long long)v.y << 32) | v.x); +} +__device__ __forceinline__ long as_long(double v) { return (long)__double_as_longlong(v); } + +// as_float / as_int / as_uint: 32-bit reinterprets +__device__ __forceinline__ float as_float(int v) { return __int_as_float(v); } +__device__ __forceinline__ float as_float(unsigned int v) { return __int_as_float((int)v); } +__device__ __forceinline__ int as_int(float v) { return __float_as_int(v); } +__device__ __forceinline__ unsigned int as_uint(float v) { return (unsigned int)__float_as_int(v); } + +// 16-byte reinterprets: int4 ↔ double2 ↔ ulong2 +__device__ __forceinline__ int4 as_int4(double2 v) { + union { double2 d; int4 i; } u; + u.d = v; + return u.i; +} +__device__ __forceinline__ int4 as_int4(ulong2 v) { + union { ulong2 ul; int4 i; } u; + u.ul = v; + return u.i; +} +__device__ __forceinline__ double2 as_double2(int4 v) { + union { int4 i; double2 d; } u; + u.i = v; + return u.d; +} +__device__ __forceinline__ ulong2 as_ulong2(int4 v) { + union { int4 i; ulong2 ul; } u; + u.i = v; + return u.ul; +} +__device__ __forceinline__ double2 as_double2(ulong2 v) { + union { ulong2 ul; double2 d; } u; + u.ul = v; + return u.d; +} +__device__ __forceinline__ ulong2 as_ulong2(double2 v) { + union { double2 d; ulong2 ul; } u; + u.d = v; + return u.ul; +} + +// ---- Math builtins ---- +// fma for vector types (OpenCL supports element-wise fma on vector types) +__device__ __forceinline__ double2 fma(double2 a, double2 b, double2 c) { + return make_double2(fma(a.x, b.x, c.x), fma(a.y, b.y, c.y)); +} +__device__ __forceinline__ float2 fma(float2 a, float2 b, float2 c) { + return make_float2(fmaf(a.x, b.x, c.x), fmaf(a.y, b.y, c.y)); +} +// Mixed scalar-vector fma: fma(scalar, vec2, vec2) — broadcasts scalar +__device__ __forceinline__ double2 fma(double a, double2 b, double2 c) { + return make_double2(fma(a, b.x, c.x), fma(a, b.y, c.y)); +} +__device__ __forceinline__ float2 fma(float a, float2 b, float2 c) { + return make_float2(fmaf(a, b.x, c.x), fmaf(a, b.y, c.y)); +} + +// mul_hi: upper half of multiplication +__device__ __forceinline__ unsigned int mul_hi(unsigned int a, unsigned int b) { + return __umulhi(a, b); +} +// Overloads for both 'unsigned long long' and 'unsigned long' (distinct types on Linux x86_64) +__device__ __forceinline__ unsigned long long mul_hi(unsigned long long a, unsigned long long b) { + return __umul64hi(a, b); +} +__device__ __forceinline__ unsigned long mul_hi(unsigned long a, unsigned long b) { + return (unsigned long)__umul64hi((unsigned long long)a, (unsigned long long)b); +} +__device__ __forceinline__ unsigned int mad_hi(unsigned int a, unsigned int b, unsigned int c) { + return __umulhi(a, b) + c; +} + +// ---- Atomic operations ---- +#define atomic_max(p, v) atomicMax((unsigned int*)(p), (unsigned int)(v)) +#define atomic_add(p, v) atomicAdd(p, v) + +// OpenCL 2.0 C11-style atomics — optimized for CUDA carry stairway pattern. +// The carryFused kernel uses: producer writes data, threadfence, bar, atomic_store(flag, 1) +// then consumer does: atomic_load(flag) in spin loop, bar, threadfence, read data. +// We minimize redundant fences while maintaining correctness. +__device__ __forceinline__ void atomic_store_uint(volatile unsigned int* p, unsigned int v) { + // Volatile store only — no fence needed here. The caller always does + // write_mem_fence(CLK_GLOBAL_MEM_FENCE) [= __threadfence()] before calling + // atomic_store(), which already orders all prior writes before this store. + // Adding a second __threadfence() here was redundant but costly (~100-400 cycles). + *p = v; +} +__device__ __forceinline__ unsigned int atomic_load_uint(volatile unsigned int* p) { + // Acquire load: volatile ensures we re-read from memory, not from register. + // No fence needed here — the caller does read_mem_fence AFTER confirming the flag. + return *p; +} +#define atomic_store(p, v) atomic_store_uint((volatile unsigned int*)(p), (unsigned int)(v)) +#define atomic_load_explicit(p, order, scope) atomic_load_uint((volatile unsigned int*)(p)) +#define memory_order_relaxed 0 +#define memory_order_acquire 0 +#define memory_order_release 0 +#define memory_scope_device 0 +typedef volatile unsigned int atomic_uint; + +// ---- Inline assembly ---- +// OpenCL uses __asm(); NVRTC uses asm() +#define __asm asm + +// ---- sub_group / warp functions ---- +#define sub_group_broadcast(v, lane) __shfl_sync(0xFFFFFFFF, (v), (lane)) + +// ---- Mark as CUDA compilation ---- +#define CUDA_BACKEND 1 + +// ---- Word2 constructor (typedef for long2 or int2) ---- +// base.cl defines Word2 as long2 (WordSize==8) or int2 (WordSize==4). +// The preprocessor converts (Word2)(a, b) → make_Word2(a, b). +// Must key on WordSize, not CARRY64, because FFT3261 has WordSize=8 without CARRY64. +#if WordSize == 8 +__device__ __forceinline__ long2 make_Word2(long a, long b) { return make_long2(a, b); } +#else +__device__ __forceinline__ int2 make_Word2(int a, int b) { return make_int2(a, b); } +#endif + +// ---- Force NVIDIAGPU and HAS_PTX ---- +#ifndef NVIDIAGPU +#define NVIDIAGPU 1 +#endif +#ifndef HAS_PTX +#define HAS_PTX 1200 +#endif +#ifndef AMDGPU +#define AMDGPU 0 +#endif diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h new file mode 100644 index 00000000..5a34c15e --- /dev/null +++ b/src/cuda/tinycuda.h @@ -0,0 +1,317 @@ +// CUDA type shim — replaces tinycl.h for the native CUDA backend. +// Maps OpenCL types and constants to CUDA Driver API equivalents. +// Used together with clwrap_cuda.cpp which implements cl* functions via cu*. + +#pragma once + +#include "../common.h" +#include +#include +#include +#include +#include +#include +#include + +// ---- Opaque handle wrappers ---- +// OpenCL uses opaque pointer types (struct _cl_foo*). +// CUDA uses different representations (int, pointer, u64). +// We wrap CUDA handles in structs so they're pointer-like opaque types. + +// cl_device_id wraps CUdevice (int) +struct _cl_device_id { CUdevice dev; }; +typedef _cl_device_id* cl_device_id; + +// cl_context wraps CUcontext +struct _cl_context { CUcontext ctx; CUdevice dev; }; +typedef _cl_context* cl_context; + +// cl_command_queue wraps CUstream +struct _cl_command_queue { + CUstream stream; + cl_context context; + bool profiling; +}; +typedef _cl_command_queue* cl_command_queue; + +// cl_mem wraps CUdeviceptr + size +struct _cl_mem { + CUdeviceptr ptr; + size_t size; +}; +typedef _cl_mem* cl_mem; + +// cl_program: dual-purpose — stores either source string or compiled PTX/module +struct _cl_program { + std::string source; // OpenCL source (before NVRTC compilation) + std::string preprocessedSource; // CUDA source after preprocessOpenCL (for parsing __launch_bounds__) + std::string ptx; // Compiled PTX (after NVRTC compilation) + CUmodule module; // Loaded module (after cuModuleLoadData) + bool compiled; + bool moduleLoaded; + + _cl_program() : module{}, compiled{false}, moduleLoaded{false} {} +}; +typedef _cl_program* cl_program; + +// cl_kernel wraps CUfunction + accumulated arguments +struct _cl_kernel { + CUfunction func; + std::string name; + CUmodule parentModule; // Keep reference so module isn't unloaded + int numArgs; + int reqWorkGroupSize; // From __launch_bounds__(N) in source, matches OpenCL reqd_work_group_size + + // Argument accumulator for setArg/launch pattern + static constexpr int MAX_ARGS = 32; + static constexpr int MAX_ARG_BYTES = 512; + char argData[MAX_ARG_BYTES]; + size_t argSizes[MAX_ARGS]; + size_t argOffsets[MAX_ARGS]; + + _cl_kernel() : func{}, parentModule{}, numArgs{0}, reqWorkGroupSize{0} { + memset(argData, 0, sizeof(argData)); + memset(argSizes, 0, sizeof(argSizes)); + memset(argOffsets, 0, sizeof(argOffsets)); + } + + void setArg(int pos, size_t size, const void* value) { + if (pos >= MAX_ARGS) return; + if (pos >= numArgs) numArgs = pos + 1; + + // Fixed 8-byte slots per arg position. CUDA kernel args are pointers (8 bytes) + // or small scalars (4 bytes). Using fixed slots avoids data corruption when + // args are set out of order (e.g., setFixedArgs(2,3) then operator()(0,1)). + size_t offset = pos * 8; + argOffsets[pos] = offset; + argSizes[pos] = size; + if (offset + size <= MAX_ARG_BYTES && value) { + memcpy(argData + offset, value, size); + } + } + + // Build void* args[] array for cuLaunchKernel + void buildArgPointers(void** ptrs) const { + for (int i = 0; i < numArgs; i++) { + ptrs[i] = const_cast(argData + argOffsets[i]); + } + } +}; +typedef _cl_kernel* cl_kernel; + +// cl_event wraps CUevent pair (start + end for profiling) +struct _cl_event { + CUevent start; + CUevent end; + bool hasTimings; + u32 commandType; + + _cl_event() : start{}, end{}, hasTimings{false}, commandType{0} {} + ~_cl_event() { + if (start) cuEventDestroy(start); + if (end) cuEventDestroy(end); + } +}; +typedef _cl_event* cl_event; + +// Unused types — just need to exist for compilation +typedef struct _cl_platform_id* cl_platform_id; +typedef struct _cl_sampler* cl_sampler; + +typedef unsigned cl_bool; +typedef unsigned cl_program_build_info; +typedef unsigned cl_program_info; +typedef unsigned cl_device_info; +typedef unsigned cl_kernel_info; +typedef unsigned cl_kernel_arg_info; +typedef unsigned cl_kernel_work_group_info; +typedef unsigned cl_profiling_info; +typedef unsigned cl_event_info; +typedef unsigned cl_command_queue_info; + +typedef u64 cl_mem_flags; +typedef u64 cl_svm_mem_flags; +typedef u64 cl_device_type; +typedef u64 cl_queue_properties; + +using cl_queue = cl_command_queue; + +// ---- Constants ---- +#define CL_SUCCESS 0 +#define CL_DEVICE_TYPE_GPU (1 << 2) +#define CL_DEVICE_TYPE_ALL 0xFFFFFFFF + +#define CL_DEVICE_VENDOR_ID 0x1001 +#define CL_DEVICE_MAX_COMPUTE_UNITS 0x1002 +#define CL_DEVICE_MAX_CLOCK_FREQUENCY 0x100C +#define CL_DEVICE_GLOBAL_MEM_SIZE 0x101F +#define CL_DEVICE_ERROR_CORRECTION_SUPPORT 0x1024 +#define CL_DEVICE_NAME 0x102B +#define CL_DRIVER_VERSION 0x102D +#define CL_DEVICE_VERSION 0x102F +#define CL_DEVICE_BUILT_IN_KERNELS 0x103F +#define CL_PLATFORM_VERSION 0x0901 + +#define CL_PROGRAM_BINARY_SIZES 0x1165 +#define CL_PROGRAM_BINARIES 0x1166 +#define CL_PROGRAM_BUILD_LOG 0x1183 + +#define CL_MEM_READ_WRITE (1 << 0) +#define CL_MEM_WRITE_ONLY (1 << 1) +#define CL_MEM_READ_ONLY (1 << 2) +#define CL_MEM_USE_HOST_PTR (1 << 3) +#define CL_MEM_ALLOC_HOST_PTR (1 << 4) +#define CL_MEM_COPY_HOST_PTR (1 << 5) +#define CL_MEM_HOST_WRITE_ONLY (1 << 7) +#define CL_MEM_HOST_READ_ONLY (1 << 8) +#define CL_MEM_HOST_NO_ACCESS (1 << 9) +#define CL_MEM_SVM_FINE_GRAIN_BUFFER (1 << 10) +#define CL_MEM_SVM_ATOMICS (1 << 11) + +#define CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE (1 << 0) +#define CL_QUEUE_PROFILING_ENABLE (1 << 1) +#define CL_QUEUE_ON_DEVICE (1 << 2) +#define CL_QUEUE_ON_DEVICE_DEFAULT (1 << 3) + +#define CL_QUEUE_CONTEXT 0x1090 +#define CL_QUEUE_DEVICE 0x1091 +#define CL_QUEUE_REFERENCE_COUNT 0x1092 +#define CL_QUEUE_PROPERTIES 0x1093 + +#define CL_PROFILING_COMMAND_QUEUED 0x1280 +#define CL_PROFILING_COMMAND_SUBMIT 0x1281 +#define CL_PROFILING_COMMAND_START 0x1282 +#define CL_PROFILING_COMMAND_END 0x1283 +#define CL_PROFILING_COMMAND_COMPLETE 0x1284 + +#define CL_EVENT_COMMAND_QUEUE 0x11D0 +#define CL_EVENT_COMMAND_TYPE 0x11D1 +#define CL_EVENT_REFERENCE_COUNT 0x11D2 +#define CL_EVENT_COMMAND_EXECUTION_STATUS 0x11D3 +#define CL_EVENT_CONTEXT 0x11D4 + +#define CL_COMMAND_NDRANGE_KERNEL 0x11F0 +#define CL_COMMAND_READ_BUFFER 0x11F3 +#define CL_COMMAND_WRITE_BUFFER 0x11F4 +#define CL_COMMAND_COPY_BUFFER 0x11F5 +#define CL_COMMAND_FILL_BUFFER 0x1207 +#define CL_COMMAND_MARKER 0x11FE + +#define CL_COMPLETE 0x0 +#define CL_RUNNING 0x1 +#define CL_SUBMITTED 0x2 +#define CL_QUEUED 0x3 + +#define CL_KERNEL_NUM_ARGS 0x1191 +#define CL_KERNEL_ARG_NAME 0x119A +#define CL_KERNEL_ATTRIBUTES 0x1195 +#define CL_KERNEL_COMPILE_WORK_GROUP_SIZE 0x11B1 + +// AMD-specific (unused but must exist for compilation) +#define CL_DEVICE_PCIE_ID_AMD 0x4034 +#define CL_DEVICE_TOPOLOGY_AMD 0x4037 +#define CL_DEVICE_BOARD_NAME_AMD 0x4038 +#define CL_DEVICE_GLOBAL_FREE_MEMORY_AMD 0x4039 + +typedef union { + struct { u32 type; u32 data[5]; } raw; + struct { u32 type; char unused[17]; char bus; char device; char function; } pcie; +} cl_device_topology_amd; + +// Error codes +#define CL_DEVICE_NOT_FOUND -1 +#define CL_DEVICE_NOT_AVAILABLE -2 +#define CL_COMPILER_NOT_AVAILABLE -3 +#define CL_MEM_OBJECT_ALLOCATION_FAILURE -4 +#define CL_OUT_OF_RESOURCES -5 +#define CL_OUT_OF_HOST_MEMORY -6 +#define CL_PROFILING_INFO_NOT_AVAILABLE -7 +#define CL_BUILD_PROGRAM_FAILURE -11 +#define CL_COMPILE_PROGRAM_FAILURE -15 +#define CL_LINK_PROGRAM_FAILURE -17 +#define CL_INVALID_VALUE -30 +#define CL_INVALID_DEVICE -33 +#define CL_INVALID_CONTEXT -34 +#define CL_INVALID_MEM_OBJECT -38 +#define CL_INVALID_BINARY -42 +#define CL_INVALID_BUILD_OPTIONS -43 +#define CL_INVALID_PROGRAM -44 +#define CL_INVALID_KERNEL_NAME -46 +#define CL_INVALID_KERNEL -48 +#define CL_INVALID_ARG_INDEX -49 +#define CL_INVALID_ARG_VALUE -50 +#define CL_INVALID_ARG_SIZE -51 +#define CL_INVALID_WORK_GROUP_SIZE -54 +#define CL_INVALID_GLOBAL_WORK_SIZE -63 + +// ---- OpenCL API function declarations ---- +// These are implemented in clwrap_cuda.cpp using CUDA Driver API. +// They match the signatures from tinycl.h so clwrap.h compiles unchanged. + +extern "C" { + +unsigned clGetPlatformIDs(unsigned, cl_platform_id*, unsigned*); +int clGetDeviceIDs(cl_platform_id, cl_device_type, unsigned, cl_device_id*, unsigned*); +cl_context clCreateContext(const intptr_t*, unsigned, const cl_device_id*, + void (*)(const char*, const void*, size_t, void*), void*, int*); +int clReleaseContext(cl_context); +int clReleaseProgram(cl_program); +int clReleaseCommandQueue(cl_command_queue); +int clEnqueueNDRangeKernel(cl_command_queue, cl_kernel, unsigned, const size_t*, + const size_t*, const size_t*, unsigned, const cl_event*, cl_event*); + +cl_program clCreateProgramWithSource(cl_context, unsigned, const char**, const size_t*, int*); +cl_program clCreateProgramWithBinary(cl_context, unsigned, const cl_device_id*, const size_t*, + const unsigned char**, int*, int*); + +int clBuildProgram(cl_program, unsigned, const cl_device_id*, const char*, + void (*)(cl_program, void*), void*); +int clCompileProgram(cl_program, unsigned, const cl_device_id*, const char*, + unsigned numHeaders, const cl_program* headers, const char* const* headerNames, + void (*)(cl_program, void*), void*); +cl_program clLinkProgram(cl_context, unsigned, const cl_device_id*, const char*, + unsigned nProgs, const cl_program* progs, + void (*)(cl_program, void*), void*, int* err); + +int clGetProgramBuildInfo(cl_program, cl_device_id, cl_program_build_info, size_t, void*, size_t*); +int clGetProgramInfo(cl_program, cl_program_info, size_t, void*, size_t*); +int clGetDeviceInfo(cl_device_id, cl_device_info, size_t, void*, size_t*); +int clGetPlatformInfo(cl_platform_id, cl_device_info, size_t, void*, size_t*); +int clGetCommandQueueInfo(cl_command_queue, cl_command_queue_info, size_t, void*, size_t*); + +cl_kernel clCreateKernel(cl_program, const char*, int*); +int clReleaseKernel(cl_kernel); +cl_mem clCreateBuffer(cl_context, cl_mem_flags, size_t, void*, int*); +int clReleaseMemObject(cl_mem); +cl_command_queue clCreateCommandQueueWithProperties(cl_context, cl_device_id, + const cl_queue_properties*, int*); + +int clEnqueueMarkerWithWaitList(cl_command_queue, unsigned, const cl_event*, cl_event*); +int clEnqueueReadBuffer(cl_command_queue, cl_mem, cl_bool, size_t, size_t, void*, + unsigned, const cl_event*, cl_event*); +int clEnqueueWriteBuffer(cl_command_queue, cl_mem, cl_bool, size_t, size_t, const void*, + unsigned, const cl_event*, cl_event*); +int clEnqueueCopyBuffer(cl_command_queue, cl_mem, cl_mem, size_t, size_t, size_t, + unsigned, const cl_event*, cl_event*); +int clEnqueueFillBuffer(cl_command_queue, cl_mem, const void*, size_t, size_t, size_t, + unsigned, const cl_event*, cl_event*); + +int clFlush(cl_command_queue); +int clFinish(cl_command_queue); +int clSetKernelArg(cl_kernel, unsigned, size_t, const void*); + +int clReleaseEvent(cl_event); +int clWaitForEvents(unsigned, const cl_event*); + +int clGetKernelInfo(cl_kernel, cl_kernel_info, size_t, void*, size_t*); +int clGetKernelArgInfo(cl_kernel, unsigned, cl_kernel_arg_info, size_t, void*, size_t*); +int clGetKernelWorkGroupInfo(cl_kernel, cl_device_id, cl_kernel_work_group_info, size_t, void*, size_t*); + +int clGetEventInfo(cl_event, cl_event_info, size_t, void*, size_t*); +int clGetEventProfilingInfo(cl_event, cl_profiling_info, size_t, void*, size_t*); + +void* clSVMAlloc(cl_context, cl_svm_mem_flags, size_t, unsigned); +void clSVMFree(cl_context, void*); +int clSetKernelArgSVMPointer(cl_kernel, unsigned, const void*); + +} diff --git a/src/tune.cpp b/src/tune.cpp index 21db6f40..fb8aca16 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -133,36 +133,36 @@ float Tune::maxBpw(FFTConfig fft) { // Fine tune our estimate for Z=34 float z1 = zForBpw(bpw1, fft, 1); printf ("Guess bpw for %s is %.2f first Z34 is %.2f\n", fft.spec().c_str(), bpw1, z1); - while (z1 < 31.0 || z1 > 37.0) { + while (z1 < 31.0f || z1 > 37.0f) { float prev_bpw1 = bpw1; float prev_z1 = z1; - bpw1 = bpw1 + (z1 - 34) * bpw_step; + bpw1 = bpw1 + (z1 - 34.0f) * bpw_step; z1 = zForBpw(bpw1, fft, 1); printf ("Reguess bpw for %s is %.2f first Z34 is %.2f\n", fft.spec().c_str(), bpw1, z1); bpw_step = - (bpw1 - prev_bpw1) / (z1 - prev_z1); - if (bpw_step < 0.005) bpw_step = 0.005; - if (bpw_step > 0.025) bpw_step = 0.025; + if (bpw_step < 0.005f) bpw_step = 0.005f; + if (bpw_step > 0.025f) bpw_step = 0.025f; } // Get more samples for this bpw -- average in the sample we already have z1 = (z1 + (sample_size - 1) * zForBpw(bpw1, fft, sample_size - 1)) / sample_size; // Pick a bpw somewhere near Z=22 then fine tune the guess - float bpw2 = bpw1 + (z1 - 22) * bpw_step; + float bpw2 = bpw1 + (z1 - 22.0f) * bpw_step; float z2 = zForBpw(bpw2, fft, 1); printf ("Guess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bpw2, z2); - while (z2 < 20.0 || z2 > 25.0) { + while (z2 < 20.0f || z2 > 25.0f) { float prev_bpw2 = bpw2; float prev_z2 = z2; // bool error_recovery = (z2 <= 0.0); // if (error_recovery) bpw2 -= bpw_step; else - bpw2 = bpw2 + (z2 - 21) * bpw_step; + bpw2 = bpw2 + (z2 - 21.0f) * bpw_step; z2 = zForBpw(bpw2, fft, 1); printf ("Reguess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bpw2, z2); // if (error_recovery) { if (z2 >= 20.0) break; else continue; } bpw_step = - (bpw2 - prev_bpw2) / (z2 - prev_z2); - if (bpw_step < 0.005) bpw_step = 0.005; - if (bpw_step > 0.025) bpw_step = 0.025; + if (bpw_step < 0.005f) bpw_step = 0.005f; + if (bpw_step > 0.025f) bpw_step = 0.025f; } // Get more samples for this bpw -- average in the sample we already have @@ -174,7 +174,7 @@ printf ("Reguess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bp float Tune::zForBpw(float bpw, FFTConfig fft, u32 count) { u64 exponent = (count == 1) ? primes.prevPrime(fft.size() * bpw) : primes.nextPrime(fft.size() * bpw); - float total_z = 0.0; + float total_z = 0.0f; for (u32 i = 0; i < count; i++, exponent = primes.nextPrime (exponent + 1)) { auto [ok, res, roeSq, roeMul] = Gpu::make(q, exponent, shared, fft, {}, false)->measureROE(true); float z = roeSq.z(); @@ -196,7 +196,7 @@ void Tune::ztune() { u32 variant = 202; u32 sample_size = 5; FFTConfig fft{shape, variant, CARRY_AUTO}; - for (float bpw = 18.18; bpw < 18.305; bpw += 0.02) { + for (float bpw = 18.18f; bpw < 18.305f; bpw += 0.02f) { float z = zForBpw(bpw, fft, sample_size); log ("Avg zForBpw %s %.2f %.2f\n", fft.spec().c_str(), bpw, z); } @@ -248,7 +248,7 @@ void Tune::carryTune() { vector zv; double m = 0; const float mid = fft.shape.carry32BPW(); - for (float bpw : {mid - 0.05, mid + 0.05}) { + for (float bpw : {mid - 0.05f, mid + 0.05f}) { u64 exponent = primes.nearestPrime(fft.size() * bpw); auto [ok, carry] = Gpu::make(q, exponent, shared, fft, {}, false)->measureCarry(); m = carry.max; @@ -340,7 +340,7 @@ void configsUpdate(double current_cost, double best_cost, double threshold, cons void Tune::tune() { Args *args = shared.args; vector shapes = FFTShape::multiSpec(args->fftSpec); - + // There are some options and variants that are different based on GPU manufacturer bool AMDGPU = isAmdGpu(q->context->deviceId()); bool NVIDIAGPU = isNvidiaGpu(q->context->deviceId()); @@ -349,8 +349,8 @@ void Tune::tune() { bool time_FFTs = 0; bool time_NTTs = 0; bool time_FP32 = 1; - bool time_inplace_only = 0; - int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) + bool time_inplace_only = NVIDIAGPU ? 1 : 0; // Default is nVidia is better off with INPLACE=1, AMD GPUs need to time extra options used when INPLACE=0 + int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) u64 min_exponent = 75000000; u64 max_exponent = 350000000; if (!args->fftSpec.empty()) { min_exponent = 0; max_exponent = 1000000000000ull; } @@ -581,28 +581,212 @@ void Tune::tune() { args->flags["INPLACE"] = to_string(best_inplace); } - // Find best NONTEMPORAL setting + // Find best LOADS/STORES settings + if (1) { + u32 loads = args->value("LOADS", 0); + u32 stores = args->value("STORES", 0); + + // Find best FFT data LOADS setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_fft_load = 0; + double best_cost = -1.0; + for (u32 fft_load : {0, 1, 2, 3, 4}) { + if (!NVIDIAGPU && fft_load >= 2) continue; + args->flags["LOADS"] = to_string(loads / 10 * 10 + fft_load); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using FFT load=%u is %6.1f\n", fft.spec().c_str(), fft_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_load = fft_load; } + } + log("Best FFT load is %u. Default is 0.\n", best_fft_load); + loads = loads / 10 * 10 + best_fft_load; + args->flags["LOADS"] = to_string(loads); + } + + // Find best FFT data STORES setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_fft_store = 0; + double best_cost = -1.0; + for (u32 fft_store : {0, 1, 2, 3, 4}) { + if (!NVIDIAGPU && fft_store >= 2) continue; + args->flags["STORES"] = to_string(stores / 10 * 10 + fft_store); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using FFT store=%u is %6.1f\n", fft.spec().c_str(), fft_store, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_store = fft_store; } + } + log("Best FFT store is %u. Default is 0.\n", best_fft_store); + stores = stores / 10 * 10 + best_fft_store; + args->flags["STORES"] = to_string(stores); + } + + // Find best carryShuttle LOADS/STORES settings + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_cs_load = 0, best_cs_store = 0; + double best_cost = -1.0; + for (u32 cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load and L2 store + if (!NVIDIAGPU && cs == 2) continue; + u32 cs_load = cs == 0 ? 0 : cs == 1 ? 1 : 4; + u32 cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; + args->flags["LOADS"] = to_string(loads / 100 * 100 + cs_load * 10 + loads % 10); + args->flags["STORES"] = to_string(stores / 100 * 100 + cs_store * 10 + stores % 10); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using carry shuttle load=%u, store=%u is %6.1f\n", fft.spec().c_str(), cs_load, cs_store, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_cs_load = cs_load; best_cs_store = cs_store; } + } + log("Best carry shuttle load/store is %u/%u. Default is 0/0.\n", best_cs_load, best_cs_store); + loads = loads / 100 * 100 + best_cs_load * 10 + loads % 10; + stores = stores / 100 * 100 + best_cs_store * 10 + stores % 10; + args->flags["LOADS" ] = to_string(loads); + args->flags["STORES"] = to_string(stores); + } + + // Find best TRIG frequently used data LOADS setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_trig_load = 0; + double best_cost = -1.0; + for (u32 trig_load : {0, 5}) { + if (!NVIDIAGPU && trig_load >= 2) continue; + args->flags["LOADS"] = to_string(loads / 1000 * 1000 + trig_load * 100 + loads % 100); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using Trig frequently used load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } + } + log("Best Trig frquently used load is %u. Default is 0.\n", best_trig_load); + loads = loads / 1000 * 1000 + best_trig_load * 100 + loads % 100; + args->flags["LOADS" ] = to_string(loads); + } + + // Find best TRIG several uses data LOADS setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_trig_load = 0; + double best_cost = -1.0; + for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { + if (!NVIDIAGPU && trig_load >= 2) continue; + args->flags["LOADS"] = to_string(loads / 10000 * 10000 + trig_load * 1000 + loads % 1000); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using Trig several uses load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } + } + log("Best Trig several uses load is %u. Default is 0.\n", best_trig_load); + loads = loads / 10000 * 10000 + best_trig_load * 1000 + loads % 1000; + args->flags["LOADS" ] = to_string(loads); + } + + // Find best TRIG used once data LOADS setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_trig_load = 0; + double best_cost = -1.0; + for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { + if (!NVIDIAGPU && trig_load >= 2) continue; + args->flags["LOADS"] = to_string(trig_load * 10000 + loads % 10000); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using Trig used once load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } + } + log("Best Trig used once load is %u. Default is 0.\n", best_trig_load); + loads = best_trig_load * 10000 + loads % 10000; + args->flags["LOADS" ] = to_string(loads); + } + + // Write accumulated LOADS/STORES settings + configsUpdate(1.000, 0.000, 0.000, "LOADS", loads, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["LOADS"] = to_string(loads); + configsUpdate(1.000, 0.000, 0.000, "STORES", stores, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["STORES"] = to_string(stores); + } + + // Find best CUDA compiler options +#if CUDA_BACKEND + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_cfblks = 0; + u32 current_cfblks = args->value("CFBLKS", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 cfblks : {0, 1, 2}) { + args->flags["CFBLKS"] = to_string(cfblks); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using CFBLKS=%u is %6.1f\n", fft.spec().c_str(), cfblks, cost); + if (cfblks == current_cfblks) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_cfblks = cfblks; } + } + log("Best CFBLKS is %u. Default CFBLKS is 0.\n", best_cfblks); + configsUpdate(current_cost, best_cost, 0.000, "CFBLKS", best_cfblks, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["CFBLKS"] = to_string(best_cfblks); + } + + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_miblks = 0; + u32 current_miblks = args->value("MIBLKS", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 miblks : {0, 1, 2}) { + args->flags["MIBLKS"] = to_string(miblks); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MIBLKS=%u is %6.1f\n", fft.spec().c_str(), miblks, cost); + if (miblks == current_miblks) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_miblks = miblks; } + } + log("Best MIBLKS is %u. Default MIBLKS is 0.\n", best_miblks); + configsUpdate(current_cost, best_cost, 0.000, "MIBLKS", best_miblks, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MIBLKS"] = to_string(best_miblks); + } + + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_moblks = 0; + u32 current_moblks = args->value("MOBLKS", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 moblks : {0, 1, 2}) { + args->flags["MOBLKS"] = to_string(moblks); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MOBLKS=%u is %6.1f\n", fft.spec().c_str(), moblks, cost); + if (moblks == current_moblks) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_moblks = moblks; } + } + log("Best MOBLKS is %u. Default MOBLKS is 0.\n", best_moblks); + configsUpdate(current_cost, best_cost, 0.000, "MOBLKS", best_moblks, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MOBLKS"] = to_string(best_moblks); + } + if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_nontemporal = 0; - u32 current_nontemporal = args->value("NONTEMPORAL", 0); + u32 best_tsblks = 0; + u32 current_tsblks = args->value("TSBLKS", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 nontemporal : {0, 1, 2}) { - args->flags["NONTEMPORAL"] = to_string(nontemporal); + for (u32 tsblks : {0, 1, 2}) { + args->flags["TSBLKS"] = to_string(tsblks); double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using NONTEMPORAL=%u is %6.1f\n", fft.spec().c_str(), nontemporal, cost); - if (nontemporal == current_nontemporal) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_nontemporal = nontemporal; } + log("Time for %12s using TSBLKS=%u is %6.1f\n", fft.spec().c_str(), tsblks, cost); + if (tsblks == current_tsblks) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tsblks = tsblks; } } - log("Best NONTEMPORAL is %u. Default NONTEMPORAL is 0.\n", best_nontemporal); - configsUpdate(current_cost, best_cost, 0.000, "NONTEMPORAL", best_nontemporal, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["NONTEMPORAL"] = to_string(best_nontemporal); + log("Best TSBLKS is %u. Default TSBLKS is 0.\n", best_tsblks); + configsUpdate(current_cost, best_cost, 0.000, "TSBLKS", best_tsblks, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["TSBLKS"] = to_string(best_tsblks); } +#endif // Find best FAST_BARRIER setting - if (1 /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) + if (1 /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_fast_barrier = 0; @@ -912,7 +1096,7 @@ void Tune::tune() { } // Find best WMUL setting - if (1) { + if (1 && defaultShape->width != 4096) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_wmul = 0; @@ -931,46 +1115,6 @@ void Tune::tune() { args->flags["WMUL"] = to_string(best_wmul); } - // Find best ENABLE_L2STORE setting - if (NVIDIAGPU) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_enable_l2store = 0; - u32 current_enable_l2store = args->value("ENABLE_L2STORE", 2); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 enable_l2store : {0, 1}) { - args->flags["ENABLE_L2STORE"] = to_string(enable_l2store); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using ENABLE_L2STORE=%u is %6.1f\n", fft.spec().c_str(), enable_l2store, cost); - if (enable_l2store == current_enable_l2store) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_enable_l2store = enable_l2store; } - } - log("Best ENABLE_L2STORE is %u. Default ENABLE_L2STORE is 1.\n", best_enable_l2store); - configsUpdate(current_cost, best_cost, 0.003, "ENABLE_L2STORE", best_enable_l2store, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["ENABLE_L2STORE"] = to_string(best_enable_l2store); - } - - // Find best ENABLE_LULOAD setting - if (NVIDIAGPU) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_enable_luload = 0; - u32 current_enable_luload = args->value("ENABLE_LULOAD", 2); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 enable_luload : {0, 1}) { - args->flags["ENABLE_LULOAD"] = to_string(enable_luload); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using ENABLE_LULOAD=%u is %6.1f\n", fft.spec().c_str(), enable_luload, cost); - if (enable_luload == current_enable_luload) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_enable_luload = enable_luload; } - } - log("Best ENABLE_LULOAD is %u. Default ENABLE_LULOAD is 1.\n", best_enable_luload); - configsUpdate(current_cost, best_cost, 0.003, "ENABLE_LULOAD", best_enable_luload, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["ENABLE_LULOAD"] = to_string(best_enable_luload); - } - // Find best BIGLIT setting if (0 && time_FFTs) { // Deprecated FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; @@ -1070,14 +1214,14 @@ skip_1K_256 = 0; if (variant_W(variant) == 0) { if (!AMDGPU) continue; if (shape.width > 1024) continue; - if (args->value("NO_ASM", 0)) continue; + if (args->value("NO_ASM", 0)) continue; } // Only AMD GPUs support variant zero (BCAST) and only if height <= 1024. if (variant_H(variant) == 0) { if (!AMDGPU) continue; if (shape.height > 1024) continue; - if (args->value("NO_ASM", 0)) continue; + if (args->value("NO_ASM", 0)) continue; } // Reject shapes that won't be used to test exponents in the user's desired range @@ -1165,7 +1309,7 @@ skip_1K_256 = 0; double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); bool isUseful = TuneEntry{cost, fft}.update(results); log("%c %6.1f %12s %9" PRIu64 "\n", isUseful ? '*' : ' ', cost, fft.spec().c_str(), fft.maxExp()); - if (isUseful) TuneEntry::writeTuneFile(results); + if (isUseful) TuneEntry::writeTuneFile(results); } } } From 0fe2ee25dcb51df44069933f4aad677d48c8008c Mon Sep 17 00:00:00 2001 From: george Date: Mon, 23 Mar 2026 19:24:44 +0000 Subject: [PATCH 015/214] More changes (many are lint-like fixes) due to potential MSVC support. --- genbundle.sh | 10 +- src/FFTConfig.cpp | 12 +- src/FFTConfig.h | 8 +- src/File.cpp | 2 +- src/File.h | 37 ++-- src/TrigBufCache.cpp | 2 +- src/U128.cpp | 285 ++++++++++++++++++++++++++++++ src/U128.h | 390 +++++++++++++++++++++++++++++++++++++++++ src/common.h | 5 + src/fftbpw.h | 402 +++++++++++++++++++++---------------------- src/main.cpp | 6 + src/state.cpp | 2 +- 12 files changed, 930 insertions(+), 231 deletions(-) create mode 100644 src/U128.cpp create mode 100644 src/U128.h diff --git a/genbundle.sh b/genbundle.sh index 9d176062..d13521f8 100755 --- a/genbundle.sh +++ b/genbundle.sh @@ -19,10 +19,14 @@ do names=${names}\"${x}\", echo "// $xx" - #echo const char ${x}_cl[] = R\"cltag\( - echo 'R"cltag(' - cat "$xx" + + # MSVC cannot handle string constants longer than 16KB. Thus, output one raw line at a time and concatenate them using the C++ preprocessor + echo ' R"cltag(' + while IFS= read -r line; do + echo ')cltag" R"cltag('"$line" + done < $xx echo ')cltag",' + echo done echo '};' diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index c21607e1..5441392d 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -34,7 +34,7 @@ u32 parseInt(const string& s) { assert(!s.empty()); char c = s.back(); u32 multiple = c == 'k' || c == 'K' ? 1024 : c == 'm' || c == 'M' ? 1024 * 1024 : 1; - return strtod(s.c_str(), nullptr) * multiple; + return u32(strtod(s.c_str(), nullptr) * multiple); } } // namespace @@ -157,7 +157,7 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : while (h < 256) { h *= 2; m /= 2; } if (m == 1) m = 2; bpw = FFTShape{w, m, h}.bpw; - for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) bpw[j] -= 0.05; // Assume this fft spec is worse than measured fft specs + for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) bpw[j] -= 0.05f; // Assume this fft spec is worse than measured fft specs if (this->isFavoredShape()) { // Don't output this warning message for non-favored shapes (we expect the BPW info to be missing) printf("BPW info for %s not found, defaults={", s.c_str()); for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) printf("%s%.2f", j ? ", " : "", (double) bpw[j]); @@ -177,9 +177,9 @@ float FFTShape::carry32BPW() const { //GW: I have no idea why this is needed. Without it, -tune fails on FFT sizes from 256K to 1M // Perhaps it has something to do with RNDVALdoubleToLong in carryutil -if (18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())) > 19.0) return 19.0; +if (18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())) > 19.0) return 19.0f; - return 18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())); + return float(18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size()))); } bool FFTShape::needsLargeCarry(u64 E) const { @@ -227,7 +227,7 @@ FFTConfig::FFTConfig(const string& spec) { } } -FFTConfig::FFTConfig(FFTShape shape, u32 variant, u32 carry) : +FFTConfig::FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry) : shape{shape}, variant{variant}, carry{carry} @@ -265,7 +265,7 @@ float FFTConfig::maxBpw() const { else { float b1 = shape.bpw[variant_M(variant) * 3 + variant_W(variant)]; float b2 = shape.bpw[variant_M(variant) * 3 + variant_H(variant)]; - b = (b1 + b2) / 2.0; + b = (b1 + b2) / 2.0f; } // Only some FFTs support both 32 and 64 bit carries. return (carry == CARRY_32 && (shape.fft_type == FFT64 || shape.fft_type == FFT3231)) ? std::min(shape.carry32BPW(), b) : b; diff --git a/src/FFTConfig.h b/src/FFTConfig.h index a9ad052c..1b139ea6 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -27,8 +27,6 @@ class FFTShape { public: static std::vector allShapes(u32 from=0, u32 to = -1); - static tuple getChainLengths(u32 fftSize, u64 exponent, u32 middle); - static vector multiSpec(const string& spec); enum FFT_TYPES fft_type; @@ -87,14 +85,14 @@ struct FFTConfig { FFTShape shape{}; u32 variant; - u32 carry; + enum CARRY_KIND carry; explicit FFTConfig(const string& spec); - FFTConfig(FFTShape shape, u32 variant, u32 carry); + FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry); std::string spec() const; u64 size() const { return shape.size(); } - u64 maxExp() const { return maxBpw() * shape.size(); } + u64 maxExp() const { return u64(maxBpw() * shape.size()); } float minBpw() const { return shape.minBpw(); } float maxBpw() const; diff --git a/src/File.cpp b/src/File.cpp index 3aaad2e0..2d54bda2 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -6,7 +6,7 @@ using namespace std; -File::File(const std::filesystem::__cxx11::path& path, const string& mode, bool throwOnError) +File::File(const std::filesystem::path& path, const string& mode, bool throwOnError) : readOnly{mode == "rb"}, name{path.string()} { assert(readOnly || throwOnError); diff --git a/src/File.h b/src/File.h index 1e1766e4..f3b9b177 100644 --- a/src/File.h +++ b/src/File.h @@ -8,7 +8,9 @@ #include #include #include +#ifndef _MSC_VER // unistd.h does not exist for MSVC #include +#endif #include #include #include @@ -28,6 +30,15 @@ #define HAS_SETLINEBUF 0 #endif +//! Macros for __attribute__ compiler/crossplatform support +#ifndef _MSC_VER +#define FORMAT_PRINTF(fmt_idx, arg_idx) __attribute__((format(printf, fmt_idx, arg_idx))) +#define FORMAT_SCANF(fmt_idx, arg_idx) __attribute__((format(scanf, fmt_idx, arg_idx))) +#else +#define FORMAT_PRINTF(fmt_idx, arg_idx) +#define FORMAT_SCANF(fmt_idx, arg_idx) +#endif + namespace fs = std::filesystem; struct CRCError { @@ -48,7 +59,7 @@ class File { File(const fs::path &path, const string& mode, bool throwOnError); - bool readNoThrow(void* data, u32 nBytes) const { return fread(data, nBytes, 1, get()); } + bool readNoThrow(void* data, u32 nBytes) const { return fread(data, nBytes, 1, this->get()); } void read(void* data, u32 nBytes) const { if (!readNoThrow(data, nBytes)) { throw ReadError{name}; } @@ -128,18 +139,18 @@ class File { void write(const T& x) const { write(&x, sizeof(T)); } void write(const void* data, u32 nBytes) const { - if (!fwrite(data, nBytes, 1, get())) { throw WriteError{name}; } + if (!fwrite(data, nBytes, 1, this->get())) { throw WriteError{name}; } } void seek(long offset, int whence = SEEK_SET) { - int ret = fseek(get(), offset, whence); + int ret = fseek(this->get(), offset, whence); if (ret) { throw ReadError{name}; } // throw(std::ios_base::failure(("fseek: "s + to_string(ret)).c_str())); } - void flush() { fflush(get()); } + void flush() { fflush(this->get()); } - int printf(const char *fmt, ...) const __attribute__((format(printf, 2, 3))) { + int printf(const char *fmt, ...) const FORMAT_PRINTF(2, 3) { va_list va; va_start(va, fmt); int ret = vfprintf(f, fmt, va); @@ -152,7 +163,7 @@ class File { return ret; } - int scanf(const char *fmt, ...) __attribute__((format(scanf, 2, 3))) { + int scanf(const char *fmt, ...) FORMAT_SCANF(2, 3) { va_list va; va_start(va, fmt); int ret = vfscanf(f, fmt, va); @@ -162,13 +173,13 @@ class File { void write(const string& s) { write(string_view(s)); } void write(const char* s) { write(string_view(s)); } - void write(string_view s) { write(s.data(), s.size()); } + void write(string_view s) { write(s.data(), u32(s.size())); } operator bool() const { return f != nullptr; } FILE* get() const { return f; } long ftell() const { - long pos = ::ftell(get()); + long pos = ::ftell(this->get()); assert(pos >= 0); return pos; } @@ -191,7 +202,7 @@ class File { std::string readLine() { char buf[1024]; buf[0] = 0; - bool ok = fgets(buf, sizeof(buf), get()); + bool ok = fgets(buf, sizeof(buf), this->get()); if (!ok) { return ""; } // EOF or error string line = buf; if (line.empty() || line.back() != '\n') { @@ -244,11 +255,11 @@ class File { read(data.data(), nBytes); return data; } - - u32 readUpTo(void* data, u32 nUpToBytes) { return fread(data, 1, nUpToBytes, get()); } - + + u32 readUpTo(void* data, u32 nUpToBytes) { return u32(fread(data, 1, nUpToBytes, this->get())); } + string readAll() { - size_t sz = size(); + u32 sz = u32(size()); return {read(sz).data(), sz}; } }; diff --git a/src/TrigBufCache.cpp b/src/TrigBufCache.cpp index 03c8e1e6..0edb1f29 100644 --- a/src/TrigBufCache.cpp +++ b/src/TrigBufCache.cpp @@ -593,7 +593,7 @@ class Z61 static uint64_t _mul(const uint64_t a, const uint64_t b) { - const __uint128_t t = a * __uint128_t(b); + const u128 t = a * u128(b); const uint64_t lo = uint64_t(t), hi = uint64_t(t >> 64); const uint64_t lo61 = lo & _p, hi61 = (lo >> 61) | (hi << 3); return _add(lo61, hi61); diff --git a/src/U128.cpp b/src/U128.cpp new file mode 100644 index 00000000..6b2b858d --- /dev/null +++ b/src/U128.cpp @@ -0,0 +1,285 @@ +/******************************************************************* +* +* Author: Kareem Omar +* kareem.h.omar@gmail.com +* https://github.com/komrad36 +* +* Last updated Feb 15, 2021 +*******************************************************************/ + +#include +#include +#include +#include + +#include "U128.h" + +using I8 = int8_t; +using I16 = int16_t; +using I32 = int32_t; +using I64 = int64_t; + +using U8 = uint8_t; +using U16 = uint16_t; +using U32 = uint32_t; +using U64 = uint64_t; + +static inline bool FitsHardwareDivL(U64 nHi, U64 nLo, U64 d) +{ + return !(nHi | (d >> 32)) && nLo < (d << 32); +} + +static inline U64 HardwareDivL(U64 n, U64 d, U64& rem) +{ + U32 rLo; + const U32 qLo = _udiv64(n, U32(d), &rLo); + rem = rLo; + return qLo; +} + +static inline U64 HardwareDivQ(U64 nHi, U64 nLo, U64 d, U64& rem) +{ + nLo = _udiv128(nHi, nLo, d, &nHi); + rem = nHi; + return nLo; +} + +static inline bool IsPow2(U64 hi, U64 lo) +{ + const U64 T = hi | lo; + return !((hi & lo) | (T & (T - 1))); +} + +static inline U64 CountTrailingZeros(U64 hi, U64 lo) +{ + const U64 nLo = _tzcnt_u64(lo); + const U64 nHi = 64ULL + _tzcnt_u64(hi); + return lo ? nLo : nHi; +} + +static inline U64 CountLeadingZeros(U64 hi, U64 lo) +{ + const U64 nLo = 64ULL + _lzcnt_u64(lo); + const U64 nHi = _lzcnt_u64(hi); + return hi ? nHi : nLo; +} + +static inline U128 MaskBitsBelow(U64 hi, U64 lo, U64 n) +{ + return U128(_bzhi_u64(hi, U32(n < 64 ? 0 : n - 64)), _bzhi_u64(lo, U32(n))); +} + +U128 DivMod(U128 N, U128 D, U128& rem) +{ + if (D > N) + { + rem = N; + return 0; + } + + U64 nHi = N.m_hi; + U64 nLo = N.m_lo; + U64 dHi = D.m_hi; + U64 dLo = D.m_lo; + + if (IsPow2(dHi, dLo)) + { + const U64 n = CountTrailingZeros(dHi, dLo); + rem = MaskBitsBelow(nHi, nLo, n); + return N >> n; + } + + if (!dHi) + { + if (nHi < dLo) + { + U64 remLo; + U64 Q; + if (FitsHardwareDivL(nHi, nLo, dLo)) + Q = HardwareDivL(nLo, dLo, remLo); + else + Q = HardwareDivQ(nHi, nLo, dLo, remLo); + rem = remLo; + return Q; + } + + U64 remLo; + const U64 qHi = HardwareDivQ(0, nHi, dLo, remLo); + const U64 qLo = HardwareDivQ(remLo, nLo, dLo, remLo); + rem = remLo; + return U128(qHi, qLo); + } + + U64 n = _lzcnt_u64(dHi) - _lzcnt_u64(nHi); + + dHi = __shiftleft128(dLo, dHi, U8(n)); + dLo <<= n; + + U64 Q = 0; + ++n; + + do + { + U64 tLo, tHi; + unsigned char carry = _subborrow_u64(_subborrow_u64(0, nLo, dLo, &tLo), nHi, dHi, &tHi); + nLo = !carry ? tLo : nLo; + nHi = !carry ? tHi : nHi; + Q = (Q << 1) + !carry; + dLo = __shiftright128(dLo, dHi, 1); + dHi >>= 1; + } while (--n); + + rem = U128(nHi, nLo); + return Q; +} + +U128::U128(float x) +{ + const U32 bits = U32(_mm_cvtsi128_si32(_mm_castps_si128(_mm_set_ss(x)))); + const U32 s = bits >> 31; + + // technically UB but let's be nice + if (s) + { + m_hi = m_lo = 0ULL; + return; + } + + const U32 e = (bits >> 23) - 127; + const U32 m = (bits & ((1U << 23) - 1U)) | (1U << 23); + + // again, technically UB but let's be nice + if (e >= 128) + { + m_hi = m_lo = ~0ULL; + return; + } + + if (e >= 23) + *this = U128(m) << (e - 23); + else + *this = m >> (23 - e); +} + +U128::U128(double x) +{ + const U64 bits = U64(_mm_cvtsi128_si64(_mm_castpd_si128(_mm_set_sd(x)))); + const U64 s = bits >> 63; + + // technically UB but let's be nice + if (s) + { + m_hi = m_lo = 0ULL; + return; + } + + const U64 e = (bits >> 52) - 1023; + const U64 m = (bits & ((1ULL << 52) - 1ULL)) | (1ULL << 52); + + // again, technically UB but let's be nice + if (e >= 128) + { + m_hi = m_lo = ~0ULL; + return; + } + + if (e >= 52) + *this = U128(m) << (e - 52); + else + *this = m >> (52 - e); +} + +U128::operator float() const +{ + if (!*this) + return 0.0f; + + const U32 numBits = 128U - U32(CountLeadingZeros(m_hi, m_lo)); + + U32 bits; + + if (numBits <= 24) + { + const U32 m = (U32(m_lo) << (24 - numBits)) & ~(1U << 23); + const U32 e = numBits + 126; + bits = (e << 23) | m; + } + else + { + const U32 s = numBits - 24; + const U32 m = U32(*this >> s) & ~(1U << 23); + const U32 G = U32(*this >> (s - 1)); + const U32 R = U32(bool(MaskBitsBelow(m_hi, m_lo, s < 2 ? 0 : s - 2))); + const U32 e = numBits + 126; + bits = ((e << 23) | m) + (G & (R | m) & 1U); + } + + return _mm_cvtss_f32(_mm_castsi128_ps(_mm_cvtsi32_si128((I32)bits))); +} + +U128::operator double() const +{ + if (!*this) + return 0.0; + + const U64 numBits = 128ULL - CountLeadingZeros(m_hi, m_lo); + + U64 bits; + + if (numBits <= 53) + { + const U64 m = (m_lo << (53 - numBits)) & ~(1ULL << 52); + const U64 e = numBits + 1022; + bits = (e << 52) | m; + } + else + { + const U64 s = numBits - 53; + const U64 m = U64(*this >> s) & ~(1ULL << 52); + const U64 G = U64(*this >> (s - 1)); + const U64 R = U64(bool(MaskBitsBelow(m_hi, m_lo, s < 2 ? 0 : s - 2))); + const U64 e = numBits + 1022; + bits = ((e << 52) | m) + (G & (R | m) & 1ULL); + } + + return _mm_cvtsd_f64(_mm_castsi128_pd(_mm_cvtsi64_si128((I64)bits))); +} + +void U128::ToString(char* buf, U64 base/* = 10*/) const +{ + U64 i = 0; + if (base >= 2 && base <= 36) + { + U128 n = *this; + U128 r, b = base; + do + { + n = DivMod(n, b, r); + const char c(r); + buf[i++] = c + (c >= 10 ? '7' : '0'); + } while (n); + + for (U64 j = 0; j < (i >> 1); ++j) + { + const char t = buf[j]; + buf[j] = buf[i - j - 1]; + buf[i - j - 1] = t; + } + } + buf[i] = '\0'; +} + +std::ostream& operator<<(std::ostream& os, const U128& x) +{ + char buf[40]; + x.ToString(buf); + os << buf; + return os; +} + +const char* NatVisStr_DebugOnly(const U128& x) +{ + static char buf[40]; + x.ToString(buf); + return buf; +} diff --git a/src/U128.h b/src/U128.h new file mode 100644 index 00000000..2379a8f1 --- /dev/null +++ b/src/U128.h @@ -0,0 +1,390 @@ +/******************************************************************* +* +* Author: Kareem Omar +* kareem.h.omar@gmail.com +* https://github.com/komrad36 +* +* Last updated Feb 15, 2021 +*******************************************************************/ + +#pragma once + +#include +#include +#include + +using I8 = int8_t; +using I16 = int16_t; +using I32 = int32_t; +using I64 = int64_t; + +using U8 = uint8_t; +using U16 = uint16_t; +using U32 = uint32_t; +using U64 = uint64_t; + +#define MAKE_BINARY_OP_HELPERS(op) \ +friend auto operator op(const U128& x, U8 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, U16 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, U32 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, U64 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I8 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I16 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I32 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, I64 y) { return operator op(x, (U128)y); } \ +friend auto operator op(const U128& x, char y) { return operator op(x, (U128)y); } \ +friend auto operator op(U8 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(U16 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(U32 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(U64 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I8 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I16 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I32 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(I64 x, const U128& y) { return operator op((U128)x, y); } \ +friend auto operator op(char x, const U128& y) { return operator op((U128)x, y); } + +#define MAKE_BINARY_OP_HELPERS_FLOAT(op) \ +friend auto operator op(const U128& x, float y) { return (float)x op y; } \ +friend auto operator op(const U128& x, double y) { return (double)x op y; } \ +friend auto operator op(float x, const U128& y) { return x op (float)y; } \ +friend auto operator op(double x, const U128& y) { return x op (double)y; } + +#define MAKE_BINARY_OP_HELPERS_U64(op) \ +friend U128 operator op(const U128& x, U8 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, U16 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, U32 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I8 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I16 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I32 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, I64 n) { return operator op(x, (U64)n); } \ +friend U128 operator op(const U128& x, const U128& n) { return operator op(x, (U64)n); } + +class U128 +{ +public: + friend U128 DivMod(U128 n, U128 d, U128& rem); + + U128() = default; + U128(U8 x) : m_lo(x), m_hi(0) {} + U128(U16 x) : m_lo(x), m_hi(0) {} + U128(U32 x) : m_lo(x), m_hi(0) {} + U128(U64 x) : m_lo(x), m_hi(0) {} + U128(I8 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(I16 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(I32 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(I64 x) : m_lo(I64(x)), m_hi(I64(x) >> 63) {} + U128(U64 hi, U64 lo) : m_lo(lo), m_hi(hi) {} + + // inexact values truncate, as per the Standard [conv.fpint] + // passing values unrepresentable in the destination format is undefined behavior, + // as per the Standard, but this implementation saturates + U128(float x); + + // inexact values truncate, as per the Standard [conv.fpint] + // passing values unrepresentable in the destination format is undefined behavior, + // as per the Standard, but this implementation saturates + U128(double x); + + U128& operator+=(const U128& x) + { + static_cast(_addcarry_u64(_addcarry_u64(0, m_lo, x.m_lo, &m_lo), m_hi, x.m_hi, &m_hi)); + return *this; + } + + friend U128 operator+(const U128& x, const U128& y) + { + U128 ret; + static_cast(_addcarry_u64(_addcarry_u64(0, x.m_lo, y.m_lo, &ret.m_lo), x.m_hi, y.m_hi, &ret.m_hi)); + return ret; + } + + MAKE_BINARY_OP_HELPERS(+); + MAKE_BINARY_OP_HELPERS_FLOAT(+); + + U128& operator-=(const U128& x) + { + static_cast(_subborrow_u64(_subborrow_u64(0, m_lo, x.m_lo, &m_lo), m_hi, x.m_hi, &m_hi)); + return *this; + } + + friend U128 operator-(const U128& x, const U128& y) + { + U128 ret; + static_cast(_subborrow_u64(_subborrow_u64(0, x.m_lo, y.m_lo, &ret.m_lo), x.m_hi, y.m_hi, &ret.m_hi)); + return ret; + } + + MAKE_BINARY_OP_HELPERS(-); + MAKE_BINARY_OP_HELPERS_FLOAT(-); + + U128& operator*=(const U128& x) + { + // ab * cd + // == + // (2^64*a + b) * (2^64*c + d) + // if a*c == e, a*d == f, b*c == g, b*d == h + // |ee|ee| | | + // | |fg|fg| | + // | | |hh|hh| + + U64 hHi; + const U64 hLo = _umul128(m_lo, x.m_lo, &hHi); + m_hi = hHi + m_hi * x.m_lo + m_lo * x.m_hi; + m_lo = hLo; + return *this; + } + + friend U128 operator*(const U128& x, const U128& y) + { + U128 ret; + U64 hHi; + ret.m_lo = _umul128(x.m_lo, y.m_lo, &hHi); + ret.m_hi = hHi + y.m_hi * x.m_lo + y.m_lo * x.m_hi; + return ret; + } + + MAKE_BINARY_OP_HELPERS(*); + MAKE_BINARY_OP_HELPERS_FLOAT(*); + + U128& operator/=(const U128& x) + { + U128 rem; + *this = DivMod(*this, x, rem); + return *this; + } + + friend U128 operator/(const U128& x, const U128& y) + { + U128 rem; + return DivMod(x, y, rem); + } + + MAKE_BINARY_OP_HELPERS(/); + MAKE_BINARY_OP_HELPERS_FLOAT(/); + + U128& operator%=(const U128& x) + { + static_cast(DivMod(*this, x, *this)); + return *this; + } + + friend U128 operator%(const U128& x, const U128& y) + { + U128 ret; + static_cast(DivMod(x, y, ret)); + return ret; + } + + MAKE_BINARY_OP_HELPERS(%); + + U128& operator&=(const U128& x) + { + m_hi &= x.m_hi; + m_lo &= x.m_lo; + return *this; + } + + friend U128 operator&(const U128& x, const U128& y) + { + return U128(x.m_hi & y.m_hi, x.m_lo & y.m_lo); + } + + MAKE_BINARY_OP_HELPERS(&); + + U128& operator|=(const U128& x) + { + m_hi |= x.m_hi; + m_lo |= x.m_lo; + return *this; + } + + friend U128 operator|(const U128& x, const U128& y) + { + return U128(x.m_hi | y.m_hi, x.m_lo | y.m_lo); + } + + MAKE_BINARY_OP_HELPERS(|); + + U128& operator^=(const U128& x) + { + m_hi ^= x.m_hi; + m_lo ^= x.m_lo; + return *this; + } + + friend U128 operator^(const U128& x, const U128& y) + { + return U128(x.m_hi ^ y.m_hi, x.m_lo ^ y.m_lo); + } + + MAKE_BINARY_OP_HELPERS(^); + + U128& operator>>=(U64 n) + { + const U64 lo = __shiftright128(m_lo, m_hi, (U8)n); + const U64 hi = m_hi >> (n & 63ULL); + + m_lo = n & 64 ? hi : lo; + m_hi = n & 64 ? 0 : hi; + + return *this; + } + + friend U128 operator>>(const U128& x, U64 n) + { + U128 ret; + + const U64 lo = __shiftright128(x.m_lo, x.m_hi, (U8)n); + const U64 hi = x.m_hi >> (n & 63ULL); + + ret.m_lo = n & 64 ? hi : lo; + ret.m_hi = n & 64 ? 0 : hi; + + return ret; + } + + MAKE_BINARY_OP_HELPERS_U64(>>); + + U128& operator<<=(U64 n) + { + const U64 hi = __shiftleft128(m_lo, m_hi, (U8)n); + const U64 lo = m_lo << (n & 63ULL); + + m_hi = n & 64 ? lo : hi; + m_lo = n & 64 ? 0 : lo; + + return *this; + } + + friend U128 operator<<(const U128& x, U64 n) + { + U128 ret; + + const U64 hi = __shiftleft128(x.m_lo, x.m_hi, (U8)n); + const U64 lo = x.m_lo << (n & 63ULL); + + ret.m_hi = n & 64 ? lo : hi; + ret.m_lo = n & 64 ? 0 : lo; + + return ret; + } + + MAKE_BINARY_OP_HELPERS_U64(<<); + + friend U128 operator~(const U128& x) + { + return U128(~x.m_hi, ~x.m_lo); + } + + friend U128 operator+(const U128& x) + { + return x; + } + + friend U128 operator-(const U128& x) + { + U128 ret; + static_cast(_subborrow_u64(_subborrow_u64(0, 0, x.m_lo, &ret.m_lo), 0, x.m_hi, &ret.m_hi)); + return ret; + } + + U128& operator++() + { + operator+=(1); + return *this; + } + + U128 operator++(int) + { + const U128 x = *this; + operator++(); + return x; + } + + U128& operator--() + { + operator-=(1); + return *this; + } + + U128 operator--(int) + { + const U128 x = *this; + operator--(); + return x; + } + + friend bool operator<(const U128& x, const U128& y) + { + U64 unusedLo, unusedHi; + return _subborrow_u64(_subborrow_u64(0, x.m_lo, y.m_lo, &unusedLo), x.m_hi, y.m_hi, &unusedHi); + } + MAKE_BINARY_OP_HELPERS(<); + MAKE_BINARY_OP_HELPERS_FLOAT(<); + + friend bool operator>(const U128& x, const U128& y) { return y < x; } + MAKE_BINARY_OP_HELPERS(>); + MAKE_BINARY_OP_HELPERS_FLOAT(>); + + friend bool operator<=(const U128& x, const U128& y) { return !(x > y); } + MAKE_BINARY_OP_HELPERS(<=); + MAKE_BINARY_OP_HELPERS_FLOAT(<=); + + friend bool operator>=(const U128& x, const U128& y) { return !(x < y); } + MAKE_BINARY_OP_HELPERS(>=); + MAKE_BINARY_OP_HELPERS_FLOAT(>=); + + friend bool operator==(const U128& x, const U128& y) + { + return !((x.m_hi ^ y.m_hi) | (x.m_lo ^ y.m_lo)); + } + MAKE_BINARY_OP_HELPERS(==); + MAKE_BINARY_OP_HELPERS_FLOAT(==); + + friend bool operator!=(const U128& x, const U128& y) { return !(x == y); } + MAKE_BINARY_OP_HELPERS(!=); + MAKE_BINARY_OP_HELPERS_FLOAT(!=); + + explicit operator bool() const { return m_hi | m_lo; } + + operator U8 () const { return (U8) m_lo; } + operator U16() const { return (U16)m_lo; } + operator U32() const { return (U32)m_lo; } + operator U64() const { return (U64)m_lo; } + + operator I8 () const { return (I8) m_lo; } + operator I16() const { return (I16)m_lo; } + operator I32() const { return (I32)m_lo; } + operator I64() const { return (I64)m_lo; } + + operator char() const { return (char)m_lo; } + + // rounding method is implementation-defined as per the Standard [conv.fpint] + // this implementation performs IEEE 754-compliant "round half to even" rounding to nearest, + // regardless of the current FPU rounding mode, which matches the behavior of clang and GCC + operator float() const; + + // rounding method is implementation-defined as per the Standard [conv.fpint] + // this implementation performs IEEE 754-compliant "round half to even" rounding to nearest, + // regardless of the current FPU rounding mode, which matches the behavior of clang and GCC + operator double() const; + + // caller is responsible for ensuring that buf has space for the U128 AND the null terminator + // that follows, in the given output base. + // Common bases and worst-case size requirements: + // Base 2: 129 bytes (128 + null terminator) + // Base 8: 44 bytes ( 43 + null terminator) + // Base 10: 40 bytes ( 39 + null terminator) + // Base 16: 33 bytes ( 32 + null terminator) + void ToString(char* buf, U64 base = 10) const; + +private: + U64 m_lo; + U64 m_hi; +}; + +#undef MAKE_BINARY_OP_HELPERS +#undef MAKE_BINARY_OP_HELPERS_FLOAT +#undef MAKE_BINARY_OP_HELPERS_U64 + +std::ostream& operator<<(std::ostream& os, const U128& x); diff --git a/src/common.h b/src/common.h index 1b549950..2a2c0bce 100644 --- a/src/common.h +++ b/src/common.h @@ -11,8 +11,13 @@ using i32 = int32_t; using u32 = uint32_t; using i64 = int64_t; using u64 = uint64_t; +#ifndef _MSC_VER // MSVC does not suppport 128-bit values using i128 = __int128; using u128 = unsigned __int128; +#else +#include "U128.h" +using u128 = U128; +#endif // using f128 = __float128; static_assert(sizeof(u8) == 1, "size u8"); diff --git a/src/fftbpw.h b/src/fftbpw.h index f3af7da7..fdd490fc 100644 --- a/src/fftbpw.h +++ b/src/fftbpw.h @@ -1,208 +1,208 @@ // FFT64 - Computed by targeting Z=28 -{ "256:2:256", {19.204, 19.547, 19.636, 19.204, 19.547, 19.636}}, -{ "256:3:256", {19.106, 19.386, 19.369, 19.106, 19.399, 19.361}}, -{ "256:4:256", {18.928, 19.236, 19.322, 18.954, 19.272, 19.367}}, -{ "256:5:256", {19.093, 19.094, 19.242, 19.147, 19.142, 19.314}}, -{ "256:6:256", {18.805, 19.065, 19.005, 18.854, 19.134, 19.101}}, -{ "256:7:256", {18.688, 18.995, 18.971, 18.763, 19.068, 19.071}}, -{ "256:8:256", {18.600, 18.909, 18.964, 18.679, 19.018, 19.120}}, -{ "512:4:256", {18.729, 18.913, 19.062, 18.770, 18.963, 19.120}}, -{ "256:9:256", {18.634, 18.871, 18.802, 18.688, 18.981, 18.961}}, -{"256:10:256", {18.748, 18.770, 18.906, 18.895, 18.896, 19.051}}, -{ "512:5:256", {18.751, 18.766, 18.891, 18.812, 18.832, 18.966}}, -{"256:11:256", {18.523, 18.782, 18.791, 18.594, 18.910, 18.946}}, -{"256:12:256", {18.558, 18.749, 18.669, 18.612, 18.880, 18.842}}, -{ "512:6:256", {18.641, 18.748, 18.838, 18.686, 18.809, 18.949}}, -{"256:13:256", {18.423, 18.693, 18.794, 18.497, 18.820, 18.938}}, -{"256:14:256", {18.450, 18.671, 18.639, 18.519, 18.808, 18.785}}, -{ "512:7:256", {18.547, 18.671, 18.782, 18.629, 18.738, 18.895}}, -{"256:15:256", {18.666, 18.652, 18.628, 18.798, 18.784, 18.791}}, -{"256:16:256", {18.277, 18.568, 18.649, 18.425, 18.741, 18.849}}, -{ "512:8:256", {18.455, 18.592, 18.682, 18.508, 18.673, 18.835}}, -{ "512:4:512", {18.565, 18.599, 18.642, 18.594, 18.629, 18.716}}, -{ "512:9:256", {18.491, 18.579, 18.648, 18.546, 18.661, 18.792}}, -{ "1K:5:256", {18.553, 18.547, 18.713, 18.622, 18.601, 18.762}}, -{"512:10:256", {18.461, 18.479, 18.570, 18.561, 18.572, 18.697}}, -{ "512:5:512", {18.450, 18.473, 18.513, 18.499, 18.513, 18.583}}, -{"512:11:256", {18.335, 18.481, 18.586, 18.444, 18.579, 18.744}}, -{ "1K:6:256", {18.294, 18.528, 18.476, 18.335, 18.596, 18.562}}, -{"512:12:256", {18.381, 18.462, 18.537, 18.448, 18.560, 18.678}}, -{ "512:6:512", {18.440, 18.462, 18.585, 18.479, 18.504, 18.644}}, -{"512:13:256", {18.247, 18.401, 18.482, 18.348, 18.497, 18.645}}, -{ "1K:7:256", {18.171, 18.455, 18.442, 18.238, 18.531, 18.521}}, -{"512:14:256", {18.266, 18.376, 18.467, 18.374, 18.490, 18.637}}, -{ "512:7:512", {18.372, 18.372, 18.462, 18.428, 18.432, 18.567}}, -{"512:15:256", {18.351, 18.355, 18.460, 18.453, 18.465, 18.619}}, -{ "1K:8:256", {18.093, 18.359, 18.435, 18.156, 18.461, 18.570}}, -{"512:16:256", {18.108, 18.260, 18.298, 18.223, 18.420, 18.570}}, -{ "512:8:512", {18.256, 18.280, 18.314, 18.319, 18.369, 18.444}}, -{ "1K:9:256", {18.123, 18.328, 18.245, 18.171, 18.443, 18.426}}, -{ "512:9:512", {18.243, 18.262, 18.343, 18.315, 18.342, 18.493}}, -{ "1K:10:256", {18.233, 18.228, 18.370, 18.357, 18.343, 18.516}}, -{ "1K:5:512", {18.237, 18.232, 18.385, 18.281, 18.272, 18.479}}, -{"512:10:512", {18.142, 18.160, 18.174, 18.226, 18.243, 18.314}}, -{ "1K:11:256", {18.001, 18.234, 18.243, 18.084, 18.362, 18.411}}, -{"512:11:512", {18.156, 18.170, 18.196, 18.242, 18.251, 18.356}}, -{ "1K:12:256", {18.052, 18.225, 18.151, 18.090, 18.332, 18.291}}, -{ "1K:6:512", {18.125, 18.227, 18.293, 18.160, 18.283, 18.401}}, -{"512:12:512", {18.141, 18.163, 18.245, 18.217, 18.237, 18.409}}, -{ "1K:13:256", {17.903, 18.155, 18.249, 17.980, 18.261, 18.380}}, -{"512:13:512", {18.097, 18.104, 18.135, 18.168, 18.171, 18.239}}, -{ "1K:14:256", {17.929, 18.143, 18.103, 18.005, 18.268, 18.244}}, -{ "1K:7:512", {18.043, 18.150, 18.237, 18.100, 18.208, 18.347}}, -{"512:14:512", {18.094, 18.091, 18.154, 18.171, 18.173, 18.311}}, -{ "1K:15:256", {18.131, 18.127, 18.115, 18.260, 18.236, 18.238}}, -{"512:15:512", {18.049, 18.061, 18.112, 18.130, 18.136, 18.236}}, -{ "1K:16:256", {17.752, 18.019, 18.128, 17.892, 18.183, 18.326}}, -{ "1K:8:512", {17.930, 18.069, 18.190, 17.982, 18.137, 18.333}}, -{"512:16:512", {17.971, 17.997, 17.976, 18.072, 18.100, 18.170}}, -{ "1K:9:512", {17.944, 18.049, 18.124, 18.012, 18.122, 18.247}}, -{ "1K:10:512", {17.940, 17.939, 18.044, 18.044, 18.043, 18.195}}, -{ "1K:5:1K", {18.033, 18.016, 18.180, 18.101, 18.072, 18.237}}, -{ "1K:11:512", {17.829, 17.956, 18.078, 17.922, 18.044, 18.219}}, -{ "1K:12:512", {17.829, 17.944, 18.002, 17.912, 18.034, 18.139}}, -{ "1K:6:1K", {17.741, 18.003, 17.931, 17.794, 18.065, 18.051}}, -{ "1K:13:512", {17.744, 17.869, 18.010, 17.822, 17.959, 18.133}}, -{ "1K:14:512", {17.748, 17.854, 17.933, 17.848, 17.968, 18.100}}, -{ "1K:7:1K", {17.647, 17.914, 17.865, 17.714, 17.995, 17.994}}, -{ "1K:15:512", {17.828, 17.824, 17.943, 17.926, 17.934, 18.100}}, -{ "1K:16:512", {17.613, 17.744, 17.813, 17.711, 17.875, 18.073}}, -{ "1K:8:1K", {17.572, 17.810, 17.914, 17.628, 17.923, 18.046}}, -{ "1K:9:1K", {17.600, 17.807, 17.734, 17.636, 17.908, 17.892}}, -{ "1K:10:1K", {17.709, 17.681, 17.841, 17.823, 17.795, 17.981}}, -{ "1K:11:1K", {17.458, 17.682, 17.712, 17.560, 17.811, 17.891}}, -{ "1K:12:1K", {17.474, 17.698, 17.635, 17.560, 17.807, 17.752}}, -{ "1K:13:1K", {17.388, 17.619, 17.706, 17.465, 17.716, 17.853}}, -{ "1K:14:1K", {17.417, 17.627, 17.582, 17.491, 17.734, 17.715}}, -{ "1K:15:1K", {17.612, 17.592, 17.582, 17.720, 17.697, 17.722}}, -{ "1K:16:1K", {17.220, 17.471, 17.615, 17.369, 17.641, 17.778}}, -{ "4K:9:512", {17.457, 17.468, 17.550, 17.519, 17.531, 17.648}}, -{ "4K:10:512", {17.336, 17.336, 17.364, 17.416, 17.433, 17.507}}, -{ "4K:11:512", {17.351, 17.356, 17.393, 17.437, 17.440, 17.548}}, -{ "4K:12:512", {17.351, 17.362, 17.447, 17.420, 17.437, 17.576}}, -{ "4K:13:512", {17.278, 17.271, 17.324, 17.349, 17.351, 17.441}}, -{ "4K:14:512", {17.267, 17.270, 17.342, 17.359, 17.359, 17.503}}, -{ "4K:15:512", {17.238, 17.239, 17.295, 17.305, 17.315, 17.432}}, -{ "4K:16:512", {17.149, 17.163, 17.143, 17.251, 17.271, 17.352}}, -{ "4K:9:1K", {17.130, 17.225, 17.346, 17.189, 17.291, 17.485}}, -{ "4K:10:1K", {17.110, 17.111, 17.209, 17.199, 17.188, 17.351}}, -{ "4K:11:1K", {16.993, 17.108, 17.214, 17.084, 17.196, 17.405}}, -{ "4K:12:1K", {17.006, 17.123, 17.219, 17.101, 17.201, 17.370}}, -{ "4K:13:1K", {16.932, 17.045, 17.154, 17.002, 17.116, 17.298}}, -{ "4K:14:1K", {16.942, 17.055, 17.160, 17.027, 17.127, 17.306}}, -{ "4K:15:1K", {17.021, 17.007, 17.137, 17.104, 17.087, 17.282}}, -{ "4K:16:1K", {16.744, 16.887, 16.966, 16.921, 17.048, 17.208}}, +{ "256:2:256", {19.204f, 19.547f, 19.636f, 19.204f, 19.547f, 19.636f}}, +{ "256:3:256", {19.106f, 19.386f, 19.369f, 19.106f, 19.399f, 19.361f}}, +{ "256:4:256", {18.928f, 19.236f, 19.322f, 18.954f, 19.272f, 19.367f}}, +{ "256:5:256", {19.093f, 19.094f, 19.242f, 19.147f, 19.142f, 19.314f}}, +{ "256:6:256", {18.805f, 19.065f, 19.005f, 18.854f, 19.134f, 19.101f}}, +{ "256:7:256", {18.688f, 18.995f, 18.971f, 18.763f, 19.068f, 19.071f}}, +{ "256:8:256", {18.600f, 18.909f, 18.964f, 18.679f, 19.018f, 19.120f}}, +{ "512:4:256", {18.729f, 18.913f, 19.062f, 18.770f, 18.963f, 19.120f}}, +{ "256:9:256", {18.634f, 18.871f, 18.802f, 18.688f, 18.981f, 18.961f}}, +{"256:10:256", {18.748f, 18.770f, 18.906f, 18.895f, 18.896f, 19.051f}}, +{ "512:5:256", {18.751f, 18.766f, 18.891f, 18.812f, 18.832f, 18.966f}}, +{"256:11:256", {18.523f, 18.782f, 18.791f, 18.594f, 18.910f, 18.946f}}, +{"256:12:256", {18.558f, 18.749f, 18.669f, 18.612f, 18.880f, 18.842f}}, +{ "512:6:256", {18.641f, 18.748f, 18.838f, 18.686f, 18.809f, 18.949f}}, +{"256:13:256", {18.423f, 18.693f, 18.794f, 18.497f, 18.820f, 18.938f}}, +{"256:14:256", {18.450f, 18.671f, 18.639f, 18.519f, 18.808f, 18.785f}}, +{ "512:7:256", {18.547f, 18.671f, 18.782f, 18.629f, 18.738f, 18.895f}}, +{"256:15:256", {18.666f, 18.652f, 18.628f, 18.798f, 18.784f, 18.791f}}, +{"256:16:256", {18.277f, 18.568f, 18.649f, 18.425f, 18.741f, 18.849f}}, +{ "512:8:256", {18.455f, 18.592f, 18.682f, 18.508f, 18.673f, 18.835f}}, +{ "512:4:512", {18.565f, 18.599f, 18.642f, 18.594f, 18.629f, 18.716f}}, +{ "512:9:256", {18.491f, 18.579f, 18.648f, 18.546f, 18.661f, 18.792f}}, +{ "1K:5:256", {18.553f, 18.547f, 18.713f, 18.622f, 18.601f, 18.762f}}, +{"512:10:256", {18.461f, 18.479f, 18.570f, 18.561f, 18.572f, 18.697f}}, +{ "512:5:512", {18.450f, 18.473f, 18.513f, 18.499f, 18.513f, 18.583f}}, +{"512:11:256", {18.335f, 18.481f, 18.586f, 18.444f, 18.579f, 18.744f}}, +{ "1K:6:256", {18.294f, 18.528f, 18.476f, 18.335f, 18.596f, 18.562f}}, +{"512:12:256", {18.381f, 18.462f, 18.537f, 18.448f, 18.560f, 18.678f}}, +{ "512:6:512", {18.440f, 18.462f, 18.585f, 18.479f, 18.504f, 18.644f}}, +{"512:13:256", {18.247f, 18.401f, 18.482f, 18.348f, 18.497f, 18.645f}}, +{ "1K:7:256", {18.171f, 18.455f, 18.442f, 18.238f, 18.531f, 18.521f}}, +{"512:14:256", {18.266f, 18.376f, 18.467f, 18.374f, 18.490f, 18.637f}}, +{ "512:7:512", {18.372f, 18.372f, 18.462f, 18.428f, 18.432f, 18.567f}}, +{"512:15:256", {18.351f, 18.355f, 18.460f, 18.453f, 18.465f, 18.619f}}, +{ "1K:8:256", {18.093f, 18.359f, 18.435f, 18.156f, 18.461f, 18.570f}}, +{"512:16:256", {18.108f, 18.260f, 18.298f, 18.223f, 18.420f, 18.570f}}, +{ "512:8:512", {18.256f, 18.280f, 18.314f, 18.319f, 18.369f, 18.444f}}, +{ "1K:9:256", {18.123f, 18.328f, 18.245f, 18.171f, 18.443f, 18.426f}}, +{ "512:9:512", {18.243f, 18.262f, 18.343f, 18.315f, 18.342f, 18.493f}}, +{ "1K:10:256", {18.233f, 18.228f, 18.370f, 18.357f, 18.343f, 18.516f}}, +{ "1K:5:512", {18.237f, 18.232f, 18.385f, 18.281f, 18.272f, 18.479f}}, +{"512:10:512", {18.142f, 18.160f, 18.174f, 18.226f, 18.243f, 18.314f}}, +{ "1K:11:256", {18.001f, 18.234f, 18.243f, 18.084f, 18.362f, 18.411f}}, +{"512:11:512", {18.156f, 18.170f, 18.196f, 18.242f, 18.251f, 18.356f}}, +{ "1K:12:256", {18.052f, 18.225f, 18.151f, 18.090f, 18.332f, 18.291f}}, +{ "1K:6:512", {18.125f, 18.227f, 18.293f, 18.160f, 18.283f, 18.401f}}, +{"512:12:512", {18.141f, 18.163f, 18.245f, 18.217f, 18.237f, 18.409f}}, +{ "1K:13:256", {17.903f, 18.155f, 18.249f, 17.980f, 18.261f, 18.380f}}, +{"512:13:512", {18.097f, 18.104f, 18.135f, 18.168f, 18.171f, 18.239f}}, +{ "1K:14:256", {17.929f, 18.143f, 18.103f, 18.005f, 18.268f, 18.244f}}, +{ "1K:7:512", {18.043f, 18.150f, 18.237f, 18.100f, 18.208f, 18.347f}}, +{"512:14:512", {18.094f, 18.091f, 18.154f, 18.171f, 18.173f, 18.311f}}, +{ "1K:15:256", {18.131f, 18.127f, 18.115f, 18.260f, 18.236f, 18.238f}}, +{"512:15:512", {18.049f, 18.061f, 18.112f, 18.130f, 18.136f, 18.236f}}, +{ "1K:16:256", {17.752f, 18.019f, 18.128f, 17.892f, 18.183f, 18.326f}}, +{ "1K:8:512", {17.930f, 18.069f, 18.190f, 17.982f, 18.137f, 18.333f}}, +{"512:16:512", {17.971f, 17.997f, 17.976f, 18.072f, 18.100f, 18.170f}}, +{ "1K:9:512", {17.944f, 18.049f, 18.124f, 18.012f, 18.122f, 18.247f}}, +{ "1K:10:512", {17.940f, 17.939f, 18.044f, 18.044f, 18.043f, 18.195f}}, +{ "1K:5:1K", {18.033f, 18.016f, 18.180f, 18.101f, 18.072f, 18.237f}}, +{ "1K:11:512", {17.829f, 17.956f, 18.078f, 17.922f, 18.044f, 18.219f}}, +{ "1K:12:512", {17.829f, 17.944f, 18.002f, 17.912f, 18.034f, 18.139f}}, +{ "1K:6:1K", {17.741f, 18.003f, 17.931f, 17.794f, 18.065f, 18.051f}}, +{ "1K:13:512", {17.744f, 17.869f, 18.010f, 17.822f, 17.959f, 18.133f}}, +{ "1K:14:512", {17.748f, 17.854f, 17.933f, 17.848f, 17.968f, 18.100f}}, +{ "1K:7:1K", {17.647f, 17.914f, 17.865f, 17.714f, 17.995f, 17.994f}}, +{ "1K:15:512", {17.828f, 17.824f, 17.943f, 17.926f, 17.934f, 18.100f}}, +{ "1K:16:512", {17.613f, 17.744f, 17.813f, 17.711f, 17.875f, 18.073f}}, +{ "1K:8:1K", {17.572f, 17.810f, 17.914f, 17.628f, 17.923f, 18.046f}}, +{ "1K:9:1K", {17.600f, 17.807f, 17.734f, 17.636f, 17.908f, 17.892f}}, +{ "1K:10:1K", {17.709f, 17.681f, 17.841f, 17.823f, 17.795f, 17.981f}}, +{ "1K:11:1K", {17.458f, 17.682f, 17.712f, 17.560f, 17.811f, 17.891f}}, +{ "1K:12:1K", {17.474f, 17.698f, 17.635f, 17.560f, 17.807f, 17.752f}}, +{ "1K:13:1K", {17.388f, 17.619f, 17.706f, 17.465f, 17.716f, 17.853f}}, +{ "1K:14:1K", {17.417f, 17.627f, 17.582f, 17.491f, 17.734f, 17.715f}}, +{ "1K:15:1K", {17.612f, 17.592f, 17.582f, 17.720f, 17.697f, 17.722f}}, +{ "1K:16:1K", {17.220f, 17.471f, 17.615f, 17.369f, 17.641f, 17.778f}}, +{ "4K:9:512", {17.457f, 17.468f, 17.550f, 17.519f, 17.531f, 17.648f}}, +{ "4K:10:512", {17.336f, 17.336f, 17.364f, 17.416f, 17.433f, 17.507f}}, +{ "4K:11:512", {17.351f, 17.356f, 17.393f, 17.437f, 17.440f, 17.548f}}, +{ "4K:12:512", {17.351f, 17.362f, 17.447f, 17.420f, 17.437f, 17.576f}}, +{ "4K:13:512", {17.278f, 17.271f, 17.324f, 17.349f, 17.351f, 17.441f}}, +{ "4K:14:512", {17.267f, 17.270f, 17.342f, 17.359f, 17.359f, 17.503f}}, +{ "4K:15:512", {17.238f, 17.239f, 17.295f, 17.305f, 17.315f, 17.432f}}, +{ "4K:16:512", {17.149f, 17.163f, 17.143f, 17.251f, 17.271f, 17.352f}}, +{ "4K:9:1K", {17.130f, 17.225f, 17.346f, 17.189f, 17.291f, 17.485f}}, +{ "4K:10:1K", {17.110f, 17.111f, 17.209f, 17.199f, 17.188f, 17.351f}}, +{ "4K:11:1K", {16.993f, 17.108f, 17.214f, 17.084f, 17.196f, 17.405f}}, +{ "4K:12:1K", {17.006f, 17.123f, 17.219f, 17.101f, 17.201f, 17.370f}}, +{ "4K:13:1K", {16.932f, 17.045f, 17.154f, 17.002f, 17.116f, 17.298f}}, +{ "4K:14:1K", {16.942f, 17.055f, 17.160f, 17.027f, 17.127f, 17.306f}}, +{ "4K:15:1K", {17.021f, 17.007f, 17.137f, 17.104f, 17.087f, 17.282f}}, +{ "4K:16:1K", {16.744f, 16.887f, 16.966f, 16.921f, 17.048f, 17.208f}}, // FFT3161 - Computed by targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "1:256:2:256", {40.54, 40.54, 40.54, 40.54, 40.54, 40.54}}, -{ "1:256:4:256", {40.19, 40.19, 40.19, 40.19, 40.19, 40.19}}, -{ "1:256:8:256", {39.98, 39.98, 39.98, 39.98, 39.98, 39.98}}, -{ "1:512:4:256", {39.98, 39.98, 39.98, 39.98, 39.98, 39.98}}, -{"1:256:16:256", {39.67, 39.67, 39.67, 39.67, 39.67, 39.67}}, -{ "1:512:8:256", {39.67, 39.67, 39.67, 39.67, 39.67, 39.67}}, -{ "1:512:4:512", {39.67, 39.67, 39.67, 39.67, 39.67, 39.67}}, -{ "1:1K:8:256", {39.46, 39.46, 39.46, 39.46, 39.46, 39.46}}, -{"1:512:16:256", {39.46, 39.46, 39.46, 39.46, 39.46, 39.46}}, -{ "1:512:8:512", {39.46, 39.46, 39.46, 39.46, 39.46, 39.46}}, -{ "1:1K:16:256", {39.15, 39.15, 39.15, 39.15, 39.15, 39.15}}, -{ "1:1K:8:512", {39.15, 39.15, 39.15, 39.15, 39.15, 39.15}}, -{"1:512:16:512", {39.15, 39.15, 39.15, 39.15, 39.15, 39.15}}, -{ "1:1K:16:512", {38.97, 38.97, 38.97, 38.97, 38.97, 38.97}}, -{ "1:1K:8:1K", {38.97, 38.97, 38.97, 38.97, 38.97, 38.97}}, -{ "1:1K:16:1K", {38.62, 38.62, 38.62, 38.62, 38.62, 38.62}}, -{ "1:4K:16:512", {38.37, 38.37, 38.37, 38.37, 38.37, 38.37}}, -{ "1:4K:16:1K", {38.12, 38.12, 38.12, 38.12, 38.12, 38.12}}, // Estimated +{ "1:256:2:256", {40.54f, 40.54f, 40.54f, 40.54f, 40.54f, 40.54f}}, +{ "1:256:4:256", {40.19f, 40.19f, 40.19f, 40.19f, 40.19f, 40.19f}}, +{ "1:256:8:256", {39.98f, 39.98f, 39.98f, 39.98f, 39.98f, 39.98f}}, +{ "1:512:4:256", {39.98f, 39.98f, 39.98f, 39.98f, 39.98f, 39.98f}}, +{"1:256:16:256", {39.67f, 39.67f, 39.67f, 39.67f, 39.67f, 39.67f}}, +{ "1:512:8:256", {39.67f, 39.67f, 39.67f, 39.67f, 39.67f, 39.67f}}, +{ "1:512:4:512", {39.67f, 39.67f, 39.67f, 39.67f, 39.67f, 39.67f}}, +{ "1:1K:8:256", {39.46f, 39.46f, 39.46f, 39.46f, 39.46f, 39.46f}}, +{"1:512:16:256", {39.46f, 39.46f, 39.46f, 39.46f, 39.46f, 39.46f}}, +{ "1:512:8:512", {39.46f, 39.46f, 39.46f, 39.46f, 39.46f, 39.46f}}, +{ "1:1K:16:256", {39.15f, 39.15f, 39.15f, 39.15f, 39.15f, 39.15f}}, +{ "1:1K:8:512", {39.15f, 39.15f, 39.15f, 39.15f, 39.15f, 39.15f}}, +{"1:512:16:512", {39.15f, 39.15f, 39.15f, 39.15f, 39.15f, 39.15f}}, +{ "1:1K:16:512", {38.97f, 38.97f, 38.97f, 38.97f, 38.97f, 38.97f}}, +{ "1:1K:8:1K", {38.97f, 38.97f, 38.97f, 38.97f, 38.97f, 38.97f}}, +{ "1:1K:16:1K", {38.62f, 38.62f, 38.62f, 38.62f, 38.62f, 38.62f}}, +{ "1:4K:16:512", {38.37f, 38.37f, 38.37f, 38.37f, 38.37f, 38.37f}}, +{ "1:4K:16:1K", {38.12f, 38.12f, 38.12f, 38.12f, 38.12f, 38.12f}}, // Estimated // FFT3261 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "2:256:2:256", {34.57, 34.57, 34.57, 34.57, 34.57, 34.57}}, -{ "2:256:4:256", {34.24, 34.24, 34.24, 34.24, 34.24, 34.24}}, -{ "2:256:8:256", {34.06, 34.06, 34.06, 34.06, 34.06, 34.06}}, -{ "2:512:4:256", {34.06, 34.06, 34.06, 34.06, 34.06, 34.06}}, -{"2:256:16:256", {32.07, 32.07, 32.07, 32.07, 32.07, 32.07}}, -{ "2:512:8:256", {32.07, 32.07, 32.07, 32.07, 32.07, 32.07}}, -{ "2:512:4:512", {32.07, 32.07, 32.07, 32.07, 32.07, 32.07}}, -{ "2:1K:8:256", {31.81, 31.81, 31.81, 31.81, 31.81, 31.81}}, -{"2:512:16:256", {31.81, 31.81, 31.81, 31.81, 31.81, 31.81}}, -{ "2:512:8:512", {31.81, 31.81, 31.81, 31.81, 31.81, 31.81}}, -{ "2:1K:16:256", {31.50, 31.50, 31.50, 31.50, 31.50, 31.50}}, -{ "2:1K:8:512", {31.50, 31.50, 31.50, 31.50, 31.50, 31.50}}, -{"2:512:16:512", {31.50, 31.50, 31.50, 31.50, 31.50, 31.50}}, -{ "2:1K:16:512", {28.68, 28.68, 28.68, 28.68, 28.68, 28.68}}, // Very strange. 481421001 has a maxROE of 0.180, 481422001 has a maxROE of 0.5 -{ "2:1K:8:1K", {28.68, 28.68, 28.68, 28.68, 28.68, 28.68}}, -{ "2:1K:16:1K", {25.37, 25.37, 25.37, 25.37, 25.37, 25.37}}, // Also strange. 851422001 ROEmax=0.273, ROEavg=0.003 -{ "2:4K:16:512", {23.27, 23.27, 23.27, 23.27, 23.27, 23.27}}, -{ "2:4K:16:1K", {21.15, 21.15, 21.15, 21.15, 21.15, 21.15}}, // Estimated +{ "2:256:2:256", {34.57f, 34.57f, 34.57f, 34.57f, 34.57f, 34.57f}}, +{ "2:256:4:256", {34.24f, 34.24f, 34.24f, 34.24f, 34.24f, 34.24f}}, +{ "2:256:8:256", {34.06f, 34.06f, 34.06f, 34.06f, 34.06f, 34.06f}}, +{ "2:512:4:256", {34.06f, 34.06f, 34.06f, 34.06f, 34.06f, 34.06f}}, +{"2:256:16:256", {32.07f, 32.07f, 32.07f, 32.07f, 32.07f, 32.07f}}, +{ "2:512:8:256", {32.07f, 32.07f, 32.07f, 32.07f, 32.07f, 32.07f}}, +{ "2:512:4:512", {32.07f, 32.07f, 32.07f, 32.07f, 32.07f, 32.07f}}, +{ "2:1K:8:256", {31.81f, 31.81f, 31.81f, 31.81f, 31.81f, 31.81f}}, +{"2:512:16:256", {31.81f, 31.81f, 31.81f, 31.81f, 31.81f, 31.81f}}, +{ "2:512:8:512", {31.81f, 31.81f, 31.81f, 31.81f, 31.81f, 31.81f}}, +{ "2:1K:16:256", {31.50f, 31.50f, 31.50f, 31.50f, 31.50f, 31.50f}}, +{ "2:1K:8:512", {31.50f, 31.50f, 31.50f, 31.50f, 31.50f, 31.50f}}, +{"2:512:16:512", {31.50f, 31.50f, 31.50f, 31.50f, 31.50f, 31.50f}}, +{ "2:1K:16:512", {28.68f, 28.68f, 28.68f, 28.68f, 28.68f, 28.68f}}, // Very strange. 481421001 has a maxROE of 0.180, 481422001 has a maxROE of 0.5 +{ "2:1K:8:1K", {28.68f, 28.68f, 28.68f, 28.68f, 28.68f, 28.68f}}, +{ "2:1K:16:1K", {25.37f, 25.37f, 25.37f, 25.37f, 25.37f, 25.37f}}, // Also strange. 851422001 ROEmax=0.273, ROEavg=0.003 +{ "2:4K:16:512", {23.27f, 23.27f, 23.27f, 23.27f, 23.27f, 23.27f}}, +{ "2:4K:16:1K", {21.15f, 21.15f, 21.15f, 21.15f, 21.15f, 21.15f}}, // Estimated // FFT61 - Computed by targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "3:256:2:256", {25.02, 25.02, 25.02, 25.02, 25.02, 25.02}}, -{ "3:256:4:256", {24.72, 24.10, 24.10, 24.10, 24.10, 24.10}}, -{ "3:256:8:256", {24.46, 24.46, 24.46, 24.46, 24.46, 24.46}}, -{ "3:512:4:256", {24.46, 24.46, 24.46, 24.46, 24.46, 24.46}}, -{"3:256:16:256", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, -{ "3:512:8:256", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, -{ "3:512:4:512", {24.15, 24.15, 24.15, 24.15, 24.15, 24.15}}, -{ "3:1K:8:256", {23.84, 23.84, 23.84, 23.84, 23.84, 23.84}}, // LL of 100028317 failed (ROEmax=0.294, ROEavg=0.247). Lowering bpw from 23.94 to 23.84. -{"3:512:16:256", {23.84, 23.84, 23.84, 23.84, 23.84, 23.84}}, -{ "3:512:8:512", {23.84, 23.84, 23.84, 23.84, 23.84, 23.84}}, -{ "3:1K:16:256", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, -{ "3:1K:8:512", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, -{"3:512:16:512", {23.65, 23.65, 23.65, 23.65, 23.65, 23.65}}, -{ "3:1K:16:512", {23.42, 23.42, 23.42, 23.42, 23.42, 23.42}}, -{ "3:1K:8:1K", {23.42, 23.42, 23.42, 23.42, 23.42, 23.42}}, -{ "3:1K:16:1K", {23.13, 23.13, 23.13, 23.13, 23.13, 23.13}}, -{ "3:4K:16:512", {22.92, 22.92, 22.92, 22.92, 22.92, 22.92}}, -{ "3:4K:16:1K", {22.72, 22.72, 22.72, 22.72, 22.72, 22.72}}, // Estimated +{ "3:256:2:256", {25.02f, 25.02f, 25.02f, 25.02f, 25.02f, 25.02f}}, +{ "3:256:4:256", {24.72f, 24.10f, 24.10f, 24.10f, 24.10f, 24.10f}}, +{ "3:256:8:256", {24.46f, 24.46f, 24.46f, 24.46f, 24.46f, 24.46f}}, +{ "3:512:4:256", {24.46f, 24.46f, 24.46f, 24.46f, 24.46f, 24.46f}}, +{"3:256:16:256", {24.15f, 24.15f, 24.15f, 24.15f, 24.15f, 24.15f}}, +{ "3:512:8:256", {24.15f, 24.15f, 24.15f, 24.15f, 24.15f, 24.15f}}, +{ "3:512:4:512", {24.15f, 24.15f, 24.15f, 24.15f, 24.15f, 24.15f}}, +{ "3:1K:8:256", {23.84f, 23.84f, 23.84f, 23.84f, 23.84f, 23.84f}}, // LL of 100028317 failed (ROEmax=0.294, ROEavg=0.247). Lowering bpw from 23.94 to 23.84. +{"3:512:16:256", {23.84f, 23.84f, 23.84f, 23.84f, 23.84f, 23.84f}}, +{ "3:512:8:512", {23.84f, 23.84f, 23.84f, 23.84f, 23.84f, 23.84f}}, +{ "3:1K:16:256", {23.65f, 23.65f, 23.65f, 23.65f, 23.65f, 23.65f}}, +{ "3:1K:8:512", {23.65f, 23.65f, 23.65f, 23.65f, 23.65f, 23.65f}}, +{"3:512:16:512", {23.65f, 23.65f, 23.65f, 23.65f, 23.65f, 23.65f}}, +{ "3:1K:16:512", {23.42f, 23.42f, 23.42f, 23.42f, 23.42f, 23.42f}}, +{ "3:1K:8:1K", {23.42f, 23.42f, 23.42f, 23.42f, 23.42f, 23.42f}}, +{ "3:1K:16:1K", {23.13f, 23.13f, 23.13f, 23.13f, 23.13f, 23.13f}}, +{ "3:4K:16:512", {22.92f, 22.92f, 22.92f, 22.92f, 22.92f, 22.92f}}, +{ "3:4K:16:1K", {22.72f, 22.72f, 22.72f, 22.72f, 22.72f, 22.72f}}, // Estimated // FFT323161 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "4:256:2:256", {50.05, 50.05, 50.05, 50.05, 50.05, 50.05}}, -{ "4:256:4:256", {49.76, 49.76, 49.76, 49.76, 49.76, 49.76}}, -{ "4:256:8:256", {49.59, 49.59, 49.59, 49.59, 49.59, 49.59}}, -{ "4:512:4:256", {49.59, 49.59, 49.59, 49.59, 49.59, 49.59}}, -{"4:256:16:256", {47.59, 47.59, 47.59, 47.59, 47.59, 47.59}}, -{ "4:512:8:256", {47.59, 47.59, 47.59, 47.59, 47.59, 47.59}}, -{ "4:512:4:512", {47.59, 47.59, 47.59, 47.59, 47.59, 47.59}}, -{ "4:1K:8:256", {47.33, 47.33, 47.33, 47.33, 47.33, 47.33}}, -{"4:512:16:256", {47.33, 47.33, 47.33, 47.33, 47.33, 47.33}}, -{ "4:512:8:512", {47.33, 47.33, 47.33, 47.33, 47.33, 47.33}}, -{ "4:1K:16:256", {47.00, 47.00, 47.00, 47.00, 47.00, 47.00}}, -{ "4:1K:8:512", {47.00, 47.00, 47.00, 47.00, 47.00, 47.00}}, -{"4:512:16:512", {47.00, 47.00, 47.00, 47.00, 47.00, 47.00}}, -{ "4:1K:16:512", {44.52, 44.52, 44.52, 44.52, 44.52, 44.52}}, -{ "4:1K:8:1K", {44.52, 44.52, 44.52, 44.52, 44.52, 44.52}}, -{ "4:1K:16:1K", {41.72, 41.72, 41.72, 41.72, 41.72, 41.72}}, // Strange 41.72 has tiny error, 41.75 is 0.5 -{ "4:4K:16:512", {39.50, 39.50, 39.50, 39.50, 39.50, 39.50}}, // Estimated -{ "4:4K:16:1K", {37.50, 37.50, 37.50, 37.50, 37.50, 37.50}}, // Estimated +{ "4:256:2:256", {50.05f, 50.05f, 50.05f, 50.05f, 50.05f, 50.05f}}, +{ "4:256:4:256", {49.76f, 49.76f, 49.76f, 49.76f, 49.76f, 49.76f}}, +{ "4:256:8:256", {49.59f, 49.59f, 49.59f, 49.59f, 49.59f, 49.59f}}, +{ "4:512:4:256", {49.59f, 49.59f, 49.59f, 49.59f, 49.59f, 49.59f}}, +{"4:256:16:256", {47.59f, 47.59f, 47.59f, 47.59f, 47.59f, 47.59f}}, +{ "4:512:8:256", {47.59f, 47.59f, 47.59f, 47.59f, 47.59f, 47.59f}}, +{ "4:512:4:512", {47.59f, 47.59f, 47.59f, 47.59f, 47.59f, 47.59f}}, +{ "4:1K:8:256", {47.33f, 47.33f, 47.33f, 47.33f, 47.33f, 47.33f}}, +{"4:512:16:256", {47.33f, 47.33f, 47.33f, 47.33f, 47.33f, 47.33f}}, +{ "4:512:8:512", {47.33f, 47.33f, 47.33f, 47.33f, 47.33f, 47.33f}}, +{ "4:1K:16:256", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, +{ "4:1K:8:512", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, +{"4:512:16:512", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, +{ "4:1K:16:512", {44.52f, 44.52f, 44.52f, 44.52f, 44.52f, 44.52f}}, +{ "4:1K:8:1K", {44.52f, 44.52f, 44.52f, 44.52f, 44.52f, 44.52f}}, +{ "4:1K:16:1K", {41.72f, 41.72f, 41.72f, 41.72f, 41.72f, 41.72f}}, // Strange 41.72 has tiny error, 41.75 is 0.5 +{ "4:4K:16:512", {39.50f, 39.50f, 39.50f, 39.50f, 39.50f, 39.50f}}, // Estimated +{ "4:4K:16:1K", {37.50f, 37.50f, 37.50f, 37.50f, 37.50f, 37.50f}}, // Estimated // FFT3231 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "50:256:2:256", {19.57, 19.57, 19.57, 19.57, 19.57, 19.57}}, -{ "50:256:4:256", {19.23, 19.23, 19.23, 19.23, 19.23, 19.23}}, -{ "50:256:8:256", {19.07, 19.07, 19.07, 19.07, 19.07, 19.07}}, -{ "50:512:4:256", {19.07, 19.07, 19.07, 19.07, 19.07, 19.07}}, -{"50:256:16:256", {17.07, 17.07, 17.07, 17.07, 17.07, 17.07}}, -{ "50:512:8:256", {17.07, 17.07, 17.07, 17.07, 17.07, 17.07}}, -{ "50:512:4:512", {17.07, 17.07, 17.07, 17.07, 17.07, 17.07}}, -{ "50:1K:8:256", {16.78, 16.78, 16.78, 16.78, 16.78, 16.78}}, -{"50:512:16:256", {16.78, 16.78, 16.78, 16.78, 16.78, 16.78}}, -{ "50:512:8:512", {16.78, 16.78, 16.78, 16.78, 16.78, 16.78}}, -{ "50:1K:16:256", {16.52, 16.52, 16.52, 16.52, 16.52, 16.52}}, -{ "50:1K:8:512", {16.52, 16.52, 16.52, 16.52, 16.52, 16.52}}, -{"50:512:16:512", {16.52, 16.52, 16.52, 16.52, 16.52, 16.52}}, -{ "50:1K:16:512", {14.01, 14.01, 14.01, 14.01, 14.01, 14.01}}, -{ "50:1K:8:1K", {14.01, 14.01, 14.01, 14.01, 14.01, 14.01}}, -{ "50:1K:16:1K", {11.21, 11.21, 11.21, 11.21, 11.21, 11.21}}, // Estimated -{ "50:4K:16:512", {9.15, 9.15, 9.15, 9.15, 9.15, 9.15}}, // Estimated -{ "50:4K:16:1K", {7.05, 7.05, 7.05, 7.05, 7.05, 7.05}}, // Estimated +{ "50:256:2:256", {19.57f, 19.57f, 19.57f, 19.57f, 19.57f, 19.57f}}, +{ "50:256:4:256", {19.23f, 19.23f, 19.23f, 19.23f, 19.23f, 19.23f}}, +{ "50:256:8:256", {19.07f, 19.07f, 19.07f, 19.07f, 19.07f, 19.07f}}, +{ "50:512:4:256", {19.07f, 19.07f, 19.07f, 19.07f, 19.07f, 19.07f}}, +{"50:256:16:256", {17.07f, 17.07f, 17.07f, 17.07f, 17.07f, 17.07f}}, +{ "50:512:8:256", {17.07f, 17.07f, 17.07f, 17.07f, 17.07f, 17.07f}}, +{ "50:512:4:512", {17.07f, 17.07f, 17.07f, 17.07f, 17.07f, 17.07f}}, +{ "50:1K:8:256", {16.78f, 16.78f, 16.78f, 16.78f, 16.78f, 16.78f}}, +{"50:512:16:256", {16.78f, 16.78f, 16.78f, 16.78f, 16.78f, 16.78f}}, +{ "50:512:8:512", {16.78f, 16.78f, 16.78f, 16.78f, 16.78f, 16.78f}}, +{ "50:1K:16:256", {16.52f, 16.52f, 16.52f, 16.52f, 16.52f, 16.52f}}, +{ "50:1K:8:512", {16.52f, 16.52f, 16.52f, 16.52f, 16.52f, 16.52f}}, +{"50:512:16:512", {16.52f, 16.52f, 16.52f, 16.52f, 16.52f, 16.52f}}, +{ "50:1K:16:512", {14.01f, 14.01f, 14.01f, 14.01f, 14.01f, 14.01f}}, +{ "50:1K:8:1K", {14.01f, 14.01f, 14.01f, 14.01f, 14.01f, 14.01f}}, +{ "50:1K:16:1K", {11.21f, 11.21f, 11.21f, 11.21f, 11.21f, 11.21f}}, // Estimated +{ "50:4K:16:512", {9.15f, 9.15f, 9.15f, 9.15f, 9.15f, 9.15f}}, // Estimated +{ "50:4K:16:1K", {7.05f, 7.05f, 7.05f, 7.05f, 7.05f, 7.05f}}, // Estimated // FFT6431 - Computed with variant 202 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "51:256:2:256", {35.27, 35.27, 35.27, 35.27, 35.27, 35.27}}, -{ "51:256:4:256", {34.98, 34.98, 34.98, 34.98, 34.98, 34.98}}, -{ "51:256:8:256", {34.61, 34.61, 34.61, 34.61, 34.61, 34.61}}, -{ "51:512:4:256", {34.61, 34.61, 34.61, 34.61, 34.61, 34.61}}, -{"51:256:16:256", {34.33, 34.33, 34.33, 34.33, 34.33, 34.33}}, -{ "51:512:8:256", {34.33, 34.33, 34.33, 34.33, 34.33, 34.33}}, -{ "51:512:4:512", {34.33, 34.33, 34.33, 34.33, 34.33, 34.33}}, -{ "51:1K:8:256", {33.97, 33.97, 33.97, 33.97, 33.97, 33.97}}, -{"51:512:16:256", {33.97, 33.97, 33.97, 33.97, 33.97, 33.97}}, -{ "51:512:8:512", {33.97, 33.97, 33.97, 33.97, 33.97, 33.97}}, -{ "51:1K:16:256", {33.64, 33.64, 33.64, 33.64, 33.64, 33.64}}, -{ "51:1K:8:512", {33.64, 33.64, 33.64, 33.64, 33.64, 33.64}}, -{"51:512:16:512", {33.64, 33.64, 33.64, 33.64, 33.64, 33.64}}, -{ "51:1K:16:512", {33.45, 33.45, 33.45, 33.45, 33.45, 33.45}}, -{ "51:1K:8:1K", {33.45, 33.45, 33.45, 33.45, 33.45, 33.45}}, -{ "51:1K:16:1K", {33.23, 33.23, 33.23, 33.23, 33.23, 33.23}}, -{ "51:4K:16:512", {32.75, 32.75, 32.75, 32.75, 32.75, 32.75}}, -{ "51:4K:16:1K", {32.25, 32.25, 32.25, 32.25, 32.25, 32.25}}, // Estimated +{ "51:256:2:256", {35.27f, 35.27f, 35.27f, 35.27f, 35.27f, 35.27f}}, +{ "51:256:4:256", {34.98f, 34.98f, 34.98f, 34.98f, 34.98f, 34.98f}}, +{ "51:256:8:256", {34.61f, 34.61f, 34.61f, 34.61f, 34.61f, 34.61f}}, +{ "51:512:4:256", {34.61f, 34.61f, 34.61f, 34.61f, 34.61f, 34.61f}}, +{"51:256:16:256", {34.33f, 34.33f, 34.33f, 34.33f, 34.33f, 34.33f}}, +{ "51:512:8:256", {34.33f, 34.33f, 34.33f, 34.33f, 34.33f, 34.33f}}, +{ "51:512:4:512", {34.33f, 34.33f, 34.33f, 34.33f, 34.33f, 34.33f}}, +{ "51:1K:8:256", {33.97f, 33.97f, 33.97f, 33.97f, 33.97f, 33.97f}}, +{"51:512:16:256", {33.97f, 33.97f, 33.97f, 33.97f, 33.97f, 33.97f}}, +{ "51:512:8:512", {33.97f, 33.97f, 33.97f, 33.97f, 33.97f, 33.97f}}, +{ "51:1K:16:256", {33.64f, 33.64f, 33.64f, 33.64f, 33.64f, 33.64f}}, +{ "51:1K:8:512", {33.64f, 33.64f, 33.64f, 33.64f, 33.64f, 33.64f}}, +{"51:512:16:512", {33.64f, 33.64f, 33.64f, 33.64f, 33.64f, 33.64f}}, +{ "51:1K:16:512", {33.45f, 33.45f, 33.45f, 33.45f, 33.45f, 33.45f}}, +{ "51:1K:8:1K", {33.45f, 33.45f, 33.45f, 33.45f, 33.45f, 33.45f}}, +{ "51:1K:16:1K", {33.23f, 33.23f, 33.23f, 33.23f, 33.23f, 33.23f}}, +{ "51:4K:16:512", {32.75f, 32.75f, 32.75f, 32.75f, 32.75f, 32.75f}}, +{ "51:4K:16:1K", {32.25f, 32.25f, 32.25f, 32.25f, 32.25f, 32.25f}}, // Estimated diff --git a/src/main.cpp b/src/main.cpp index b62f1b32..259b7f9b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -45,11 +45,17 @@ extern int putenv(char *); #endif int main(int argc, char **argv) { +//!MSVC version support +#ifdef _MSC_VER + _set_printf_count_output(1); // I'm not sure what this does (it's from CrazeTheDragon) +#endif #if defined(__MSYS__) // I was unable to get putenv to link in MSYS2 #elif defined(__MINGW32__) || defined(__MINGW64__) putenv("ROC_SIGNAL_POOL_SIZE=32"); +#elif defined(_WIN32) + _putenv_s("ROC_SIGNAL_POOL_SIZE", "32"); // For MSVC #else // Required to work around a ROCm bug when using multiple queues setenv("ROC_SIGNAL_POOL_SIZE", "32", 0); diff --git a/src/state.cpp b/src/state.cpp index 77ebff8f..942aeaab 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -107,6 +107,6 @@ vector expandBits(const vector &compactBits, u32 N, u64 E) { assert(it == itEnd); assert(bucket.size == 32 - E % 32); assert(bucket.bits == 0 || bucket.bits == 1); - data[0] += bucket.bits; // carry wrap-around. + data[0] += u32(bucket.bits); // carry wrap-around. return out; } From 1b72223f155d4f7f0c9c1472de23e71f58883c95 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 25 Mar 2026 03:05:28 +0000 Subject: [PATCH 016/214] Don't tune LOADS and STORES values if NO_ASM set --- src/tune.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/tune.cpp b/src/tune.cpp index fb8aca16..978f485c 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -344,6 +344,7 @@ void Tune::tune() { // There are some options and variants that are different based on GPU manufacturer bool AMDGPU = isAmdGpu(q->context->deviceId()); bool NVIDIAGPU = isNvidiaGpu(q->context->deviceId()); + int NO_ASM = args->value("NO_ASM", 0); bool tune_config = 1; bool time_FFTs = 0; @@ -593,7 +594,7 @@ void Tune::tune() { u32 best_fft_load = 0; double best_cost = -1.0; for (u32 fft_load : {0, 1, 2, 3, 4}) { - if (!NVIDIAGPU && fft_load >= 2) continue; + if (fft_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10 * 10 + fft_load); double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FFT load=%u is %6.1f\n", fft.spec().c_str(), fft_load, cost); @@ -611,7 +612,7 @@ void Tune::tune() { u32 best_fft_store = 0; double best_cost = -1.0; for (u32 fft_store : {0, 1, 2, 3, 4}) { - if (!NVIDIAGPU && fft_store >= 2) continue; + if (fft_store >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["STORES"] = to_string(stores / 10 * 10 + fft_store); double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FFT store=%u is %6.1f\n", fft.spec().c_str(), fft_store, cost); @@ -628,8 +629,8 @@ void Tune::tune() { u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_cs_load = 0, best_cs_store = 0; double best_cost = -1.0; - for (u32 cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load and L2 store - if (!NVIDIAGPU && cs == 2) continue; + for (u32 cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load with L2 store + if (cs >= 2 && (!NVIDIAGPU || NO_ASM)) continue; u32 cs_load = cs == 0 ? 0 : cs == 1 ? 1 : 4; u32 cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; args->flags["LOADS"] = to_string(loads / 100 * 100 + cs_load * 10 + loads % 10); @@ -652,7 +653,7 @@ void Tune::tune() { u32 best_trig_load = 0; double best_cost = -1.0; for (u32 trig_load : {0, 5}) { - if (!NVIDIAGPU && trig_load >= 2) continue; + if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 1000 * 1000 + trig_load * 100 + loads % 100); double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig frequently used load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); @@ -670,7 +671,7 @@ void Tune::tune() { u32 best_trig_load = 0; double best_cost = -1.0; for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { - if (!NVIDIAGPU && trig_load >= 2) continue; + if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10000 * 10000 + trig_load * 1000 + loads % 1000); double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig several uses load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); @@ -688,7 +689,7 @@ void Tune::tune() { u32 best_trig_load = 0; double best_cost = -1.0; for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { - if (!NVIDIAGPU && trig_load >= 2) continue; + if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(trig_load * 10000 + loads % 10000); double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig used once load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); From 4f27ed839c580b6ca7b1ae590a636fa37f94fab8 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 25 Mar 2026 03:23:03 +0000 Subject: [PATCH 017/214] Finish MSVC port. Can't use "long" it is 32-bits in MSVC and 64-bits in OpenCL and Linux gcc. --- src/File.h | 5 +- src/Gpu.cpp | 8 +-- src/cl/base.cl | 2 +- src/cuda/opencl_compat.cuh | 127 +++++++++++++++---------------------- 4 files changed, 61 insertions(+), 81 deletions(-) diff --git a/src/File.h b/src/File.h index f3b9b177..52b84d7c 100644 --- a/src/File.h +++ b/src/File.h @@ -67,7 +67,10 @@ class File { void datasync() { fflush(f); -#if defined(_WIN32) || defined(__WIN32__) +#if defined(_MSC_VER) +// We'd really like to use FlushFileBuffers(h), but we do not have easy access to the Windows file handle. +// We might could get that by getting the pathname and opening the file with native Windows routines. +#elif defined(_WIN32) || defined(__WIN32__) _commit(fileno(f)); #elif defined(__APPLE__) fcntl(fileno(f), F_FULLFSYNC, 0); diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 5d30c848..f79b7536 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -145,10 +145,10 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { string toLiteral(i32 value) { return to_string(value); } string toLiteral(u32 value) { return to_string(value) + 'u'; } -[[maybe_unused]] string toLiteral(long value) { return to_string(value) + "l"; } -[[maybe_unused]] string toLiteral(unsigned long value) { return to_string(value) + "ul"; } -[[maybe_unused]] string toLiteral(long long value) { return to_string(value) + "l"; } // Yes, this looks wrong. The Mingw64 C compiler uses -[[maybe_unused]] string toLiteral(unsigned long long value) { return to_string(value) + "ul"; } // long long for 64-bits, while openCL uses long for 64 bits. +[[maybe_unused]] string toLiteral(long value) { return to_string(value) + "ll"; } // Yes, this looks wrong. MSVC and CUDA requires we always use long long. +[[maybe_unused]] string toLiteral(unsigned long value) { return to_string(value) + "ull"; } // See discussion in opencl_compat.cuh. +[[maybe_unused]] string toLiteral(long long value) { return to_string(value) + "ll"; } +[[maybe_unused]] string toLiteral(unsigned long long value) { return to_string(value) + "ull"; } template string toLiteral(F value) { diff --git a/src/cl/base.cl b/src/cl/base.cl index 75ccabf8..f58ff388 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -248,7 +248,7 @@ error - unsupported integer WordSize double2 OVERLOAD U2(double a, double b) { return (double2) (a, b); } float2 OVERLOAD U2(float a, float b) { return (float2) (a, b); } int2 OVERLOAD U2(int a, int b) { return (int2) (a, b); } -long2 OVERLOAD U2(long a, long b) { return (long2) (a, b); } +long2 OVERLOAD U2(i64 a, i64 b) { return (long2) (a, b); } uint2 OVERLOAD U2(uint a, uint b) { return (uint2) (a, b); } ulong2 OVERLOAD U2(ulong a, ulong b) { return (ulong2) (a, b); } diff --git a/src/cuda/opencl_compat.cuh b/src/cuda/opencl_compat.cuh index 3316c26f..99c15378 100644 --- a/src/cuda/opencl_compat.cuh +++ b/src/cuda/opencl_compat.cuh @@ -69,25 +69,29 @@ #define CP(x) const x* __restrict__ // ---- OpenCL type aliases ---- -// CRITICAL: On Linux x86_64, CUDA's built-in vector types (long2, ulong2) use -// 'long' / 'unsigned long' for their members. These are DISTINCT C++ types from -// 'long long' / 'unsigned long long' even though both are 64-bit. -// We MUST use 'unsigned long' for ulong so that Z61 (typedef'd from ulong) matches -// ulong2 member types. Otherwise overloaded functions like add(Z31,Z31) vs add(Z61,Z61) -// become ambiguous when called with ulong2 member values (which are 'unsigned long'). -typedef unsigned long ulong; -typedef long slong; // OpenCL's signed 'long' (64-bit) // Standard PRPLL type aliases typedef unsigned int uint; -// These match OpenCL's types exactly. The preprocessor strips base.cl's -// re-definitions of i32/u32/i64/u64 to avoid redeclaration errors. -// Must use 'long' / 'unsigned long' to match CUDA vector type members. -typedef unsigned int u32; +// These match OpenCL's types exactly. The preprocessor strips base.cl's re-definitions of i32/u32 to avoid redeclaration errors. typedef int i32; -typedef unsigned long u64; -typedef long i64; +typedef unsigned int u32; + +// OpenCL defines long, ulong, long2, ulong2, etc. as 64-bits. CUDA defines them as 32-bits (MSVC) or 64-bits (Linux). +// CUDA defines longlong, ulonglong, longlong2, ulonglong2, etc. as 64-bits. Map OpenCL types to CUDA types. + +//#define long long long // Obviously, we can't uncomment this #define. Instead, we must make sure the opencl code never uses this data type. Use i64 instead. +#define long2 longlong2 +#define ulong unsigned long long +#define ulong2 ulonglong2 +#define make_long2 make_longlong2 +#define make_ulong2 make_ulonglong2 + +// These must match the 64-bit data types defined above. The preprocessor strips base.cl's re-definitions of i64/u64 to avoid redeclaration errors. +// Must use 'long long' / 'unsigned long long' to match CUDA vector type members so that Z61 (typedef'd from ulong) matches ulong2 member types. +// Otherwise overloaded functions like add(Z31,Z31) vs add(Z61,Z61) become ambiguous when called with ulong2 member values (which are 'unsigned long'). +typedef long long i64; +typedef unsigned long long u64; // ---- Math constants ---- #ifndef M_PI @@ -152,13 +156,13 @@ __device__ __forceinline__ ulong2& operator-=(ulong2& a, ulong2 b) { a.x-=b.x; a // Scalar * vector operators for types not built-in to NVRTC // (NVRTC already provides double2*double, int2*int, uint2*uint, etc.) // These cover cross-type scalar*vector that OpenCL supports natively. -__device__ __forceinline__ long2 operator*(long long s, long2 v) { return {s*v.x, s*v.y}; } -__device__ __forceinline__ long2 operator*(long2 v, long long s) { return {v.x*s, v.y*s}; } -__device__ __forceinline__ ulong2 operator*(unsigned long long s, ulong2 v) { return {s*v.x, s*v.y}; } -__device__ __forceinline__ ulong2 operator*(ulong2 v, unsigned long long s) { return {v.x*s, v.y*s}; } +__device__ __forceinline__ long2 operator*(i64 s, long2 v) { return {s*v.x, s*v.y}; } +__device__ __forceinline__ long2 operator*(long2 v, i64 s) { return {v.x*s, v.y*s}; } +__device__ __forceinline__ ulong2 operator*(ulong s, ulong2 v) { return {s*v.x, s*v.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 v, ulong s) { return {v.x*s, v.y*s}; } // int * ulong2 (common in NTT code: int literal * GF61) -__device__ __forceinline__ ulong2 operator*(int s, ulong2 v) { return {(unsigned long long)s*v.x, (unsigned long long)s*v.y}; } -__device__ __forceinline__ ulong2 operator*(ulong2 v, int s) { return {v.x*(unsigned long long)s, v.y*(unsigned long long)s}; } +__device__ __forceinline__ ulong2 operator*(int s, ulong2 v) { return {(ulong)s*v.x, (ulong)s*v.y}; } +__device__ __forceinline__ ulong2 operator*(ulong2 v, int s) { return {v.x*(ulong)s, v.y*(ulong)s}; } // ---- Vector constructors (U2) ---- // OpenCL (type2)(a,b) cast syntax is converted to make_type2(a,b) by the preprocessor. @@ -170,77 +174,54 @@ __device__ __forceinline__ ulong2 operator*(ulong2 v, int s) { return {v.x*(unsi // as_uint2: split 64-bit value into two 32-bit halves __device__ __forceinline__ uint2 as_uint2(double v) { - unsigned long long bits = __double_as_longlong(v); - return make_uint2((unsigned int)(bits), (unsigned int)(bits >> 32)); -} -__device__ __forceinline__ uint2 as_uint2(unsigned long long v) { - return make_uint2((unsigned int)(v), (unsigned int)(v >> 32)); + ulong bits = __double_as_longlong(v); + return make_uint2((uint)(bits), (uint)(bits >> 32)); } -__device__ __forceinline__ uint2 as_uint2(unsigned long v) { - return make_uint2((unsigned int)(v), (unsigned int)((unsigned long long)v >> 32)); +__device__ __forceinline__ uint2 as_uint2(ulong v) { + return make_uint2((uint)(v), (uint)(v >> 32)); } -__device__ __forceinline__ uint2 as_uint2(long long v) { - return make_uint2((unsigned int)((unsigned long long)v), (unsigned int)((unsigned long long)v >> 32)); -} -__device__ __forceinline__ uint2 as_uint2(long v) { - return make_uint2((unsigned int)((unsigned long)v), (unsigned int)((unsigned long long)v >> 32)); +__device__ __forceinline__ uint2 as_uint2(i64 v) { + return make_uint2((uint)((ulong)v), (uint)((ulong)v >> 32)); } // as_int2: split 64-bit value into two signed 32-bit halves __device__ __forceinline__ int2 as_int2(double v) { - unsigned long long bits = __double_as_longlong(v); - return make_int2((int)(unsigned int)(bits), (int)(unsigned int)(bits >> 32)); -} -__device__ __forceinline__ int2 as_int2(long long v) { - return make_int2((int)(unsigned int)((unsigned long long)v), (int)(unsigned int)((unsigned long long)v >> 32)); + ulong bits = __double_as_longlong(v); + return make_int2((int)(uint)(bits), (int)(uint)(bits >> 32)); } -__device__ __forceinline__ int2 as_int2(long v) { - return make_int2((int)(unsigned int)((unsigned long)v), (int)(unsigned int)((unsigned long long)v >> 32)); +__device__ __forceinline__ int2 as_int2(i64 v) { + return make_int2((int)(uint)((ulong)v), (int)(uint)((ulong)v >> 32)); } // as_double: reinterpret bits as double __device__ __forceinline__ double as_double(int2 v) { - unsigned long long bits = ((unsigned long long)(unsigned int)v.y << 32) | (unsigned int)v.x; + ulong bits = ((ulong)(uint)v.y << 32) | (uint)v.x; return __longlong_as_double(bits); } __device__ __forceinline__ double as_double(uint2 v) { - unsigned long long bits = ((unsigned long long)v.y << 32) | v.x; + ulong bits = ((ulong)v.y << 32) | v.x; return __longlong_as_double(bits); } -__device__ __forceinline__ double as_double(unsigned long long v) { +__device__ __forceinline__ double as_double(ulong v) { return __longlong_as_double(v); } -__device__ __forceinline__ double as_double(unsigned long v) { - return __longlong_as_double((unsigned long long)v); -} -__device__ __forceinline__ double as_double(long long v) { return __longlong_as_double(v); } -__device__ __forceinline__ double as_double(long v) { return __longlong_as_double((long long)v); } +__device__ __forceinline__ double as_double(i64 v) { return __longlong_as_double(v); } -// as_ulong: reinterpret as unsigned 64-bit (returns 'unsigned long' to match ulong typedef) -__device__ __forceinline__ unsigned long as_ulong(uint2 v) { - return (unsigned long)(((unsigned long long)v.y << 32) | v.x); -} -__device__ __forceinline__ unsigned long as_ulong(int2 v) { - return (unsigned long)(((unsigned long long)(unsigned int)v.y << 32) | (unsigned int)v.x); -} -__device__ __forceinline__ unsigned long as_ulong(double v) { - return (unsigned long)__double_as_longlong(v); -} +// as_ulong: reinterpret as unsigned 64-bit +__device__ __forceinline__ ulong as_ulong(uint2 v) { return (((ulong)v.y << 32) | v.x); } +__device__ __forceinline__ ulong as_ulong(int2 v) { return (((ulong)(uint)v.y << 32) | (uint)v.x); } +__device__ __forceinline__ ulong as_ulong(double v) { return (ulong)__double_as_longlong(v); } -// as_long: reinterpret as signed 64-bit (returns 'long' to match slong/i64) -__device__ __forceinline__ long as_long(int2 v) { - return (long)(((unsigned long long)(unsigned int)v.y << 32) | (unsigned int)v.x); -} -__device__ __forceinline__ long as_long(uint2 v) { - return (long)(((unsigned long long)v.y << 32) | v.x); -} -__device__ __forceinline__ long as_long(double v) { return (long)__double_as_longlong(v); } +// as_long: reinterpret as signed 64-bit +__device__ __forceinline__ i64 as_long(int2 v) { return (i64)(((ulong)(uint)v.y << 32) | (uint)v.x); } +__device__ __forceinline__ i64 as_long(uint2 v) { return (i64)(((ulong)v.y << 32) | v.x); } +__device__ __forceinline__ i64 as_long(double v) { return (i64)__double_as_longlong(v); } // as_float / as_int / as_uint: 32-bit reinterprets __device__ __forceinline__ float as_float(int v) { return __int_as_float(v); } -__device__ __forceinline__ float as_float(unsigned int v) { return __int_as_float((int)v); } +__device__ __forceinline__ float as_float(uint v) { return __int_as_float((int)v); } __device__ __forceinline__ int as_int(float v) { return __float_as_int(v); } -__device__ __forceinline__ unsigned int as_uint(float v) { return (unsigned int)__float_as_int(v); } +__device__ __forceinline__ uint as_uint(float v) { return (uint)__float_as_int(v); } // 16-byte reinterprets: int4 ↔ double2 ↔ ulong2 __device__ __forceinline__ int4 as_int4(double2 v) { @@ -291,17 +272,13 @@ __device__ __forceinline__ float2 fma(float a, float2 b, float2 c) { } // mul_hi: upper half of multiplication -__device__ __forceinline__ unsigned int mul_hi(unsigned int a, unsigned int b) { +__device__ __forceinline__ uint mul_hi(uint a, uint b) { return __umulhi(a, b); } -// Overloads for both 'unsigned long long' and 'unsigned long' (distinct types on Linux x86_64) -__device__ __forceinline__ unsigned long long mul_hi(unsigned long long a, unsigned long long b) { +__device__ __forceinline__ ulong mul_hi(ulong a, ulong b) { return __umul64hi(a, b); } -__device__ __forceinline__ unsigned long mul_hi(unsigned long a, unsigned long b) { - return (unsigned long)__umul64hi((unsigned long long)a, (unsigned long long)b); -} -__device__ __forceinline__ unsigned int mad_hi(unsigned int a, unsigned int b, unsigned int c) { +__device__ __forceinline__ uint mad_hi(uint a, uint b, uint c) { return __umulhi(a, b) + c; } @@ -348,7 +325,7 @@ typedef volatile unsigned int atomic_uint; // The preprocessor converts (Word2)(a, b) → make_Word2(a, b). // Must key on WordSize, not CARRY64, because FFT3261 has WordSize=8 without CARRY64. #if WordSize == 8 -__device__ __forceinline__ long2 make_Word2(long a, long b) { return make_long2(a, b); } +__device__ __forceinline__ long2 make_Word2(i64 a, i64 b) { return make_long2(a, b); } #else __device__ __forceinline__ int2 make_Word2(int a, int b) { return make_int2(a, b); } #endif From 4044b89b7e042b94177a8bdbecbbf7284b3910ca Mon Sep 17 00:00:00 2001 From: george Date: Wed, 25 Mar 2026 03:59:17 +0000 Subject: [PATCH 018/214] One more long/long long issue. Handle the TRIGTGF61 constant. --- src/Gpu.cpp | 4 ++-- src/cl/base.cl | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index f79b7536..d69b78ba 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -145,8 +145,8 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { string toLiteral(i32 value) { return to_string(value); } string toLiteral(u32 value) { return to_string(value) + 'u'; } -[[maybe_unused]] string toLiteral(long value) { return to_string(value) + "ll"; } // Yes, this looks wrong. MSVC and CUDA requires we always use long long. -[[maybe_unused]] string toLiteral(unsigned long value) { return to_string(value) + "ull"; } // See discussion in opencl_compat.cuh. +[[maybe_unused]] string toLiteral(long value) { return to_string(value) + "ll"; } +[[maybe_unused]] string toLiteral(unsigned long value) { return to_string(value) + "ull"; } [[maybe_unused]] string toLiteral(long long value) { return to_string(value) + "ll"; } [[maybe_unused]] string toLiteral(unsigned long long value) { return to_string(value) + "ull"; } diff --git a/src/cl/base.cl b/src/cl/base.cl index f58ff388..0c41f9fe 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -250,7 +250,8 @@ float2 OVERLOAD U2(float a, float b) { return (float2) (a, b); } int2 OVERLOAD U2(int a, int b) { return (int2) (a, b); } long2 OVERLOAD U2(i64 a, i64 b) { return (long2) (a, b); } uint2 OVERLOAD U2(uint a, uint b) { return (uint2) (a, b); } -ulong2 OVERLOAD U2(ulong a, ulong b) { return (ulong2) (a, b); } +ulong2 OVERLOAD U2(unsigned long a, unsigned long b) { return (ulong2) ((ulong)a, (ulong)b); } // Two versions dealing with longs to handle TAILTGF61 constant +ulong2 OVERLOAD U2(unsigned long long a, unsigned long long b) { return (ulong2) ((ulong)a, (ulong)b); } // Other handy macros #define RE(a) (a.x) From aeb0b571c8c1128aef6e5041c3f0176d9bda2a04 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 25 Mar 2026 05:52:51 +0000 Subject: [PATCH 019/214] I wrongly thought MIDDLE_IN/OUT_LDS_TRANSPOSE settings were not used when INPLACE=1. Tune will now always time these settings. --- src/tune.cpp | 82 ++++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/tune.cpp b/src/tune.cpp index 978f485c..41d445ce 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -445,7 +445,7 @@ void Tune::tune() { u32 variant = (defaultShape == &defaultFFTShape) ? 101 : 202; //GW: if fft spec on the command line specifies a variant then we should use that variant (I get some interesting results with 000 vs 101 vs 201 vs 202 likely due to rocm optimizer) - // IN_WG/SIZEX, OUT_WG/SIZEX, PAD, MIDDLE_IN/OUT_LDS_TRANSPOSE apply only if INPLACE=0 + // IN_WG/SIZEX, OUT_WG/SIZEX, PAD apply only if INPLACE=0 u32 current_inplace = args->value("INPLACE", 0); args->flags["INPLACE"] = to_string(0); @@ -518,46 +518,6 @@ void Tune::tune() { args->flags["PAD"] = to_string(best_pad); } - // Find best MIDDLE_IN_LDS_TRANSPOSE setting - if (!time_inplace_only) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_middle_in_lds_transpose = 0; - u32 current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 middle_in_lds_transpose : {0, 1}) { - args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(middle_in_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using MIDDLE_IN_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_in_lds_transpose, cost); - if (middle_in_lds_transpose == current_middle_in_lds_transpose) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_in_lds_transpose = middle_in_lds_transpose; } - } - log("Best MIDDLE_IN_LDS_TRANSPOSE is %u. Default MIDDLE_IN_LDS_TRANSPOSE is 1.\n", best_middle_in_lds_transpose); - configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_IN_LDS_TRANSPOSE", best_middle_in_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(best_middle_in_lds_transpose); - } - - // Find best MIDDLE_OUT_LDS_TRANSPOSE setting - if (!time_inplace_only) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_middle_out_lds_transpose = 0; - u32 current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 middle_out_lds_transpose : {0, 1}) { - args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(middle_out_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using MIDDLE_OUT_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_out_lds_transpose, cost); - if (middle_out_lds_transpose == current_middle_out_lds_transpose) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_out_lds_transpose = middle_out_lds_transpose; } - } - log("Best MIDDLE_OUT_LDS_TRANSPOSE is %u. Default MIDDLE_OUT_LDS_TRANSPOSE is 1.\n", best_middle_out_lds_transpose); - configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_OUT_LDS_TRANSPOSE", best_middle_out_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(best_middle_out_lds_transpose); - } - // If only timing INPLACE=1 options, then set INPLACE if (time_inplace_only) { args->flags["INPLACE"] = to_string(1); @@ -1016,6 +976,46 @@ void Tune::tune() { args->flags["MODM31"] = to_string(best_modm31); } + // Find best MIDDLE_IN_LDS_TRANSPOSE setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_middle_in_lds_transpose = 0; + u32 current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 middle_in_lds_transpose : {0, 1}) { + args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(middle_in_lds_transpose); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MIDDLE_IN_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_in_lds_transpose, cost); + if (middle_in_lds_transpose == current_middle_in_lds_transpose) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_in_lds_transpose = middle_in_lds_transpose; } + } + log("Best MIDDLE_IN_LDS_TRANSPOSE is %u. Default MIDDLE_IN_LDS_TRANSPOSE is 1.\n", best_middle_in_lds_transpose); + configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_IN_LDS_TRANSPOSE", best_middle_in_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(best_middle_in_lds_transpose); + } + + // Find best MIDDLE_OUT_LDS_TRANSPOSE setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_middle_out_lds_transpose = 0; + u32 current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 middle_out_lds_transpose : {0, 1}) { + args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(middle_out_lds_transpose); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MIDDLE_OUT_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_out_lds_transpose, cost); + if (middle_out_lds_transpose == current_middle_out_lds_transpose) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_out_lds_transpose = middle_out_lds_transpose; } + } + log("Best MIDDLE_OUT_LDS_TRANSPOSE is %u. Default MIDDLE_OUT_LDS_TRANSPOSE is 1.\n", best_middle_out_lds_transpose); + configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_OUT_LDS_TRANSPOSE", best_middle_out_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(best_middle_out_lds_transpose); + } + // Find best UNROLL_W setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; From 5684b03d09f8e1ef9aa1ca385d0b0d1017abb6ef Mon Sep 17 00:00:00 2001 From: george Date: Wed, 25 Mar 2026 05:59:21 +0000 Subject: [PATCH 020/214] Revert "I wrongly thought MIDDLE_IN/OUT_LDS_TRANSPOSE settings were not used when INPLACE=1. Tune will now always time these settings." This reverts commit aeb0b571c8c1128aef6e5041c3f0176d9bda2a04. --- src/tune.cpp | 82 ++++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/src/tune.cpp b/src/tune.cpp index 41d445ce..978f485c 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -445,7 +445,7 @@ void Tune::tune() { u32 variant = (defaultShape == &defaultFFTShape) ? 101 : 202; //GW: if fft spec on the command line specifies a variant then we should use that variant (I get some interesting results with 000 vs 101 vs 201 vs 202 likely due to rocm optimizer) - // IN_WG/SIZEX, OUT_WG/SIZEX, PAD apply only if INPLACE=0 + // IN_WG/SIZEX, OUT_WG/SIZEX, PAD, MIDDLE_IN/OUT_LDS_TRANSPOSE apply only if INPLACE=0 u32 current_inplace = args->value("INPLACE", 0); args->flags["INPLACE"] = to_string(0); @@ -518,6 +518,46 @@ void Tune::tune() { args->flags["PAD"] = to_string(best_pad); } + // Find best MIDDLE_IN_LDS_TRANSPOSE setting + if (!time_inplace_only) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_middle_in_lds_transpose = 0; + u32 current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 middle_in_lds_transpose : {0, 1}) { + args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(middle_in_lds_transpose); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MIDDLE_IN_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_in_lds_transpose, cost); + if (middle_in_lds_transpose == current_middle_in_lds_transpose) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_in_lds_transpose = middle_in_lds_transpose; } + } + log("Best MIDDLE_IN_LDS_TRANSPOSE is %u. Default MIDDLE_IN_LDS_TRANSPOSE is 1.\n", best_middle_in_lds_transpose); + configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_IN_LDS_TRANSPOSE", best_middle_in_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(best_middle_in_lds_transpose); + } + + // Find best MIDDLE_OUT_LDS_TRANSPOSE setting + if (!time_inplace_only) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_middle_out_lds_transpose = 0; + u32 current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 middle_out_lds_transpose : {0, 1}) { + args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(middle_out_lds_transpose); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MIDDLE_OUT_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_out_lds_transpose, cost); + if (middle_out_lds_transpose == current_middle_out_lds_transpose) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_out_lds_transpose = middle_out_lds_transpose; } + } + log("Best MIDDLE_OUT_LDS_TRANSPOSE is %u. Default MIDDLE_OUT_LDS_TRANSPOSE is 1.\n", best_middle_out_lds_transpose); + configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_OUT_LDS_TRANSPOSE", best_middle_out_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(best_middle_out_lds_transpose); + } + // If only timing INPLACE=1 options, then set INPLACE if (time_inplace_only) { args->flags["INPLACE"] = to_string(1); @@ -976,46 +1016,6 @@ void Tune::tune() { args->flags["MODM31"] = to_string(best_modm31); } - // Find best MIDDLE_IN_LDS_TRANSPOSE setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_middle_in_lds_transpose = 0; - u32 current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 middle_in_lds_transpose : {0, 1}) { - args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(middle_in_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using MIDDLE_IN_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_in_lds_transpose, cost); - if (middle_in_lds_transpose == current_middle_in_lds_transpose) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_in_lds_transpose = middle_in_lds_transpose; } - } - log("Best MIDDLE_IN_LDS_TRANSPOSE is %u. Default MIDDLE_IN_LDS_TRANSPOSE is 1.\n", best_middle_in_lds_transpose); - configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_IN_LDS_TRANSPOSE", best_middle_in_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(best_middle_in_lds_transpose); - } - - // Find best MIDDLE_OUT_LDS_TRANSPOSE setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_middle_out_lds_transpose = 0; - u32 current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 middle_out_lds_transpose : {0, 1}) { - args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(middle_out_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using MIDDLE_OUT_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_out_lds_transpose, cost); - if (middle_out_lds_transpose == current_middle_out_lds_transpose) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_out_lds_transpose = middle_out_lds_transpose; } - } - log("Best MIDDLE_OUT_LDS_TRANSPOSE is %u. Default MIDDLE_OUT_LDS_TRANSPOSE is 1.\n", best_middle_out_lds_transpose); - configsUpdate(current_cost, best_cost, 0.000, "MIDDLE_OUT_LDS_TRANSPOSE", best_middle_out_lds_transpose, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(best_middle_out_lds_transpose); - } - // Find best UNROLL_W setting if (1) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; From 71ecbdc9cb72498c996a4be5921521d8e6eadaa9 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 26 Mar 2026 02:20:24 +0000 Subject: [PATCH 021/214] Corrected CUDA maxregcount syntax --- src/Gpu.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index d69b78ba..4df2c0f9 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -513,15 +513,15 @@ Gpu::~Gpu() { #define CARRY_SIZE 100000 #if CUDA_BACKEND -#define CARRYFUSED_BLOCKS(x) args.value("CFBLKS", 0) == 0 ? x : args.value("CFBLKS", 0) == 1 ? x " -DCUDA_MIN_BLOCKS=3" : x " -maxregcount 84" -#define MIDDLEIN_BLOCKS args.value("MIBLKS", 0) == 0 ? "" : args.value("MIBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount 84" -#define MIDDLEOUT_BLOCKS args.value("MOBLKS", 0) == 0 ? "" : args.value("MOBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount 84" -#define TAILSQUARE_BLOCKS args.value("TSBLKS", 0) == 0 ? "" : args.value("TSBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount 84" +#define CARRYFUSED_BLOCKS(x) args.value("CFBLKS", 0) == 0 ? x : args.value("CFBLKS", 0) == 1 ? x " -DCUDA_MIN_BLOCKS=2" : x " -maxregcount=128" +#define MIDDLEIN_BLOCKS args.value("MIBLKS", 0) == 0 ? "" : args.value("MIBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount=84" +#define MIDDLEOUT_BLOCKS args.value("MOBLKS", 0) == 0 ? "" : args.value("MOBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount=84" +#define TAILSQUARE_BLOCKS args.value("TSBLKS", 0) == 0 ? "" : args.value("TSBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount=84" #else -#define CARRYFUSED_BLOCKS(x) "" -#define MIDDLEIN_BLOCKS "" -#define MIDDLEOUT_BLOCKS "" -#define TAILSQUARE_BLOCKS "" +#define CARRYFUSED_BLOCKS(x) x +#define MIDDLEIN_BLOCKS "" +#define MIDDLEOUT_BLOCKS "" +#define TAILSQUARE_BLOCKS "" #endif Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : From 23e15f4c2c0ba30f05fd8717120a3b37138c7f1f Mon Sep 17 00:00:00 2001 From: george Date: Fri, 3 Apr 2026 01:43:00 +0000 Subject: [PATCH 022/214] Added support for CUDA programmable dependents launch. Sadly, it does not seem to help. --- src/cl/base.cl | 17 +++++++++ src/cl/carryfused.cl | 78 ++++++++++++++++++++++++++++---------- src/cl/fftmiddlein.cl | 48 +++++++++++++++++++++++ src/cl/fftmiddleout.cl | 32 ++++++++++++++++ src/cl/fftw.cl | 8 ++++ src/cl/tailmul.cl | 16 ++++++++ src/cl/tailsquare.cl | 54 +++++++++++++++++++++++++- src/cuda/cudawrap.cpp | 25 +++++++++++- src/cuda/opencl_compat.cuh | 3 ++ 9 files changed, 258 insertions(+), 23 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 0c41f9fe..545dbb91 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -734,3 +734,20 @@ void OVERLOAD bar(const u32 WG) { // This is used e.g. by the double-wide tailSquare, where LDS is split between the halves. void halfBar() { if (get_enqueued_local_size(0) / 2 > WAVEFRONT) { bar(); } } + +// nVidia GPUs (Hopper architecture sm 9.0 and later) support Programatic Dependent Launch where the tail end execution of one kernel can overlap +// with the beginning of the next kernel. This requires a special launch kernel command that is only available in CUDA 12.0 and later. +// These routines let us take advantage of this CUDA feature. These routines do nothing in OpenCL. + +void dependentLaunch() { +#if CUDA_BACKEND && HAS_PTX >= 900 && ENABLE_PDL + __asm volatile("griddepcontrol.launch_dependents;"); // same as cudaTriggerProgrammaticLaunchCompletion(); +#endif +} + +void dependentLaunchWait() { +#if CUDA_BACKEND && HAS_PTX >= 900 && ENABLE_PDL + __asm volatile("griddepcontrol.wait;"); // same as cudaGridDependencySynchronize(); +#endif +} + diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 7e64f179..dff9bb08 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -130,6 +130,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 3"); #endif + dependentLaunchWait(); // Previous kernel was fftMiddleOutFP64 + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding @@ -282,8 +284,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); } - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInFP64 + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -318,6 +321,8 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 3"); #endif + dependentLaunchWait(); // Previous kernel was fftMiddleOutFP32 + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding @@ -465,8 +470,9 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); } - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -500,6 +506,8 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry __asm("s_setprio 3"); #endif + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding @@ -673,8 +681,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry if (weight_shift > 31) weight_shift -= 31; } - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInGF31 + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -708,6 +717,8 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry __asm("s_setprio 3"); #endif + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + readCarryFusedLine(in, u, line, lowMe); // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding @@ -887,8 +898,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry if (weight_shift > 61) weight_shift -= 61; } - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInGF61 + new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -929,14 +941,17 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 3"); #endif - readCarryFusedLine(in, u, line, lowMe); - readCarryFusedLine(in31, u31, line, lowMe); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(in, u, line, lowMe); new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 + + readCarryFusedLine(in31, u31, line, lowMe); new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; @@ -1124,6 +1139,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u, out, line, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 + new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1168,14 +1185,17 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line, lowMe); - readCarryFusedLine(in31, u31, line, lowMe); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(inF2, uF2, line, lowMe); new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 + + readCarryFusedLine(in31, u31, line, lowMe); new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; @@ -1363,6 +1383,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(uF2, outF2, line, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 + new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1407,14 +1429,17 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line, lowMe); - readCarryFusedLine(in61, u61, line, lowMe); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(inF2, uF2, line, lowMe); new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + + readCarryFusedLine(in61, u61, line, lowMe); new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; @@ -1602,6 +1627,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(uF2, outF2, line, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 + new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1645,14 +1672,17 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 3"); #endif - readCarryFusedLine(in31, u31, line, lowMe); - readCarryFusedLine(in61, u61, line, lowMe); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(in31, u31, line, lowMe); new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + + readCarryFusedLine(in61, u61, line, lowMe); new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; @@ -1848,6 +1878,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInGF31 + new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1897,16 +1929,20 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 3"); #endif - readCarryFusedLine(inF2, uF2, line, lowMe); - readCarryFusedLine(in31, u31, line, lowMe); - readCarryFusedLine(in61, u61, line, lowMe); - // Try this weird FFT_width call that adds a "hidden zero" when unrolling. This prevents the compiler from finding // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; + + readCarryFusedLine(inF2, uF2, line, lowMe); new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + + readCarryFusedLine(in31, u31, line, lowMe); new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + + dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 + + readCarryFusedLine(in61, u61, line, lowMe); new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; @@ -2118,6 +2154,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(uF2, outF2, line, lowMe); + dependentLaunch(); // Next kernel will be fftMiddleInFP32 + new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); diff --git a/src/cl/fftmiddlein.cl b/src/cl/fftmiddlein.cl index d1788799..14b43013 100644 --- a/src/cl/fftmiddlein.cl +++ b/src/cl/fftmiddlein.cl @@ -30,6 +30,10 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT64 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP64 data +#endif + readMiddleInLine(u, in, y, x); middleMul2(u, x, y, 1, trig); @@ -38,6 +42,8 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trig); + dependentLaunch(); // Next kernel will be tailSquareFP64 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local T lds[IN_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -85,6 +91,10 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT32 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP32 data +#endif + readMiddleInLine(u, inF2, y, x); middleMul2(u, x, y, 1, trigF2); @@ -93,6 +103,8 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trigF2); + dependentLaunch(); // Next kernel will be tailSquareFP32 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local F lds[IN_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -140,6 +152,10 @@ KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT31 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in31, y, x); middleMul2(u, x, y, trig31); @@ -148,6 +164,8 @@ KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trig31); + dependentLaunch(); // Next kernel will be tailSquareGF31 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local Z31 lds[IN_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -195,6 +213,10 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; +#if FFT_TYPE == FFT61 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in61, y, x); middleMul2(u, x, y, trig61); @@ -203,6 +225,8 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { middleMul(u, y, trig61); + dependentLaunch(); // Next kernel will be tailSquareGF61 which must dependentLaunchWait before reading data + #if MIDDLE_IN_LDS_TRANSPOSE // Transpose the x and y values local Z61 lds[IN_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -248,6 +272,10 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT64 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP64 data +#endif + readMiddleInLine(u, in, y, x); middleMul2(u, x, y, 1, trig); @@ -256,6 +284,8 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trig); + dependentLaunch(); // Next kernel will be tailSquareFP64 which must dependentLaunchWait before reading data + // Transpose the x and y values local T2 lds[256]; middleShuffle(lds, u); @@ -297,6 +327,10 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT32 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing FP32 data +#endif + readMiddleInLine(u, inF2, y, x); middleMul2(u, x, y, 1, trigF2); @@ -305,6 +339,8 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trigF2); + dependentLaunch(); // Next kernel will be tailSquareFP32 which must dependentLaunchWait before reading data + // Transpose the x and y values local F2 lds[256]; middleShuffle(lds, u); @@ -346,6 +382,10 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT31 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in31, y, x); middleMul2(u, x, y, trig31); @@ -354,6 +394,8 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trig31); + dependentLaunch(); // Next kernel will be tailSquareGF31 which must dependentLaunchWait before reading data + // Transpose the x and y values local GF31 lds[256]; middleShuffle(lds, u); @@ -395,6 +437,10 @@ KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; +#if FFT_TYPE == FFT61 + dependentLaunchWait(); // Previous kernel was carryfused that launched dependents before writing GF31 data +#endif + readMiddleInLine(u, in61, y, x); middleMul2(u, x, y, trig61); @@ -403,6 +449,8 @@ KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { middleMul(u, y, trig61); + dependentLaunch(); // Next kernel will be tailSquareGF61 which must dependentLaunchWait before reading data + // Transpose the x and y values local GF61 lds[256]; middleShuffle(lds, u); diff --git a/src/cl/fftmiddleout.cl b/src/cl/fftmiddleout.cl index a7263dcd..9249c8c3 100644 --- a/src/cl/fftmiddleout.cl +++ b/src/cl/fftmiddleout.cl @@ -34,6 +34,8 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquareFP64 that launched dependents before writing FP64 data + readMiddleOutLine(u, in, y, x); middleMul(u, x, trig); @@ -48,6 +50,8 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { middleMul2(u, y, x, factor, trig); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local T lds[OUT_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -99,6 +103,8 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquareFP32 that launched dependents before writing FP64 data + readMiddleOutLine(u, inF2, y, x); middleMul(u, x, trigF2); @@ -113,6 +119,8 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { middleMul2(u, y, x, factor, trigF2); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local F lds[OUT_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -164,6 +172,8 @@ KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquareGF31 that launched dependents before writing GF31 data + readMiddleOutLine(u, in31, y, x); middleMul(u, x, trig31); @@ -172,6 +182,8 @@ KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { middleMul2(u, y, x, trig31); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local Z31 lds[OUT_WG / 2 * (MIDDLE <= 16 ? 2 * MIDDLE : MIDDLE)]; @@ -223,6 +235,8 @@ KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { u32 x = startx + mx; u32 y = starty + my; + dependentLaunchWait(); // Previous kernel was tailSquare61 that launched dependents before writing GF61 data + readMiddleOutLine(u, in61, y, x); middleMul(u, x, trig61); @@ -231,6 +245,8 @@ KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { middleMul2(u, y, x, trig61); + dependentLaunch(); // Next kernel will be carryfused which must dependentLaunchWait before reading data + #if MIDDLE_OUT_LDS_TRANSPOSE // Transpose the x and y values local Z61 lds[OUT_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)]; @@ -271,6 +287,8 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareFP64 that launched dependents before writing FP64 data + readMiddleOutLine(u, in, y, x); middleMul(u, x, trig); @@ -285,6 +303,8 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { middleMul2(u, y, x, factor, trig); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + // Transpose the x and y values local T2 lds[256]; middleShuffle(lds, u); @@ -324,6 +344,8 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareFP32 that launched dependents before writing FP64 data + readMiddleOutLine(u, inF2, y, x); middleMul(u, x, trigF2); @@ -338,6 +360,8 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { middleMul2(u, y, x, factor, trigF2); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + // Transpose the x and y values local F2 lds[256]; middleShuffle(lds, u); @@ -377,6 +401,8 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareGF31 that launched dependents before writing GF31 data + readMiddleOutLine(u, in31, y, x); middleMul(u, x, trig31); @@ -385,6 +411,8 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { middleMul2(u, y, x, trig31); + dependentLaunch(); // Next kernel will be carryFused which must dependentLaunchWait before reading data + // Transpose the x and y values local GF31 lds[256]; middleShuffle(lds, u); @@ -424,6 +452,8 @@ KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { u32 x = startx + me % 16; u32 y = starty + me / 16; + dependentLaunchWait(); // Previous kernel was tailSquareGF61 that launched dependents before writing GF61 data + readMiddleOutLine(u, in61, y, x); middleMul(u, x, trig61); @@ -432,6 +462,8 @@ KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { middleMul2(u, y, x, trig61); + dependentLaunch(); // Next kernel will be carryfused which must dependentLaunchWait before reading data + // Transpose the x and y values local GF61 lds[256]; middleShuffle(lds, u); diff --git a/src/cl/fftw.cl b/src/cl/fftw.cl index 89277649..789a707d 100644 --- a/src/cl/fftw.cl +++ b/src/cl/fftw.cl @@ -15,6 +15,8 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { u32 g = get_group_id(0); u32 me = get_local_id(0); + dependentLaunchWait(); // Previous kernel was fftMiddleOut + readCarryFusedLine(in, u, g, me); fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); out += WIDTH * g; @@ -42,6 +44,8 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { u32 g = get_group_id(0); u32 me = get_local_id(0); + dependentLaunchWait(); // Previous kernel was fftMiddleOut + readCarryFusedLine(inF2, u, g, me); fft_WIDTH(lds, u, smallTrigF2, 1, SHUFL_BYTES_W, me); outF2 += WIDTH * g; @@ -68,6 +72,8 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { u32 g = get_group_id(0); u32 me = get_local_id(0); + dependentLaunchWait(); // Previous kernel was fftMiddleOut + readCarryFusedLine(in31, u, g, me); fft_WIDTH(lds, u, smallTrig31, 1, SHUFL_BYTES_W, me); out31 += WIDTH * g; @@ -94,6 +100,8 @@ KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { u32 g = get_group_id(0); u32 me = get_local_id(0); + dependentLaunchWait(); // Previous kernel was fftMiddleOut + readCarryFusedLine(in61, u, g, me); fft_WIDTH(lds, u, smallTrig61, 1, SHUFL_BYTES_W, me); out61 += WIDTH * g; diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 51ac9c11..03647fdb 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -64,6 +64,8 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + u32 me = get_local_id(0); readTailFusedLine(in, u, line1, me); readTailFusedLine(in, v, line2, me); @@ -110,6 +112,8 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out, memline2, me); @@ -180,6 +184,8 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + u32 me = get_local_id(0); readTailFusedLine(inF2, u, line1, me); readTailFusedLine(inF2, v, line2, me); @@ -218,6 +224,8 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, outF2, memline2, me); @@ -288,6 +296,8 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + u32 me = get_local_id(0); readTailFusedLine(in31, u, line1, me); readTailFusedLine(in31, v, line2, me); @@ -343,6 +353,8 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out31, memline2, me); @@ -411,6 +423,8 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + u32 me = get_local_id(0); readTailFusedLine(in61, u, line1, me); readTailFusedLine(in61, v, line2, me); @@ -466,6 +480,8 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out61, memline2, me); diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index e30b8012..29832a8b 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -67,6 +67,10 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareFP64 which must dependentLaunchWait before reading data from fftMiddleInFP64 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + readTailFusedLine(in, u, line, me); #if FFT_VARIANT_H != 0 @@ -108,6 +112,9 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + readTailFusedLine(in, u, line1, me); readTailFusedLine(in, v, line2, me); @@ -168,6 +175,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); @@ -221,6 +230,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + // Read lines u and v readTailFusedLine(in, u, line, lowMe); @@ -274,6 +285,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { revCrossLine(lds, u); } + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data + new_fft_HEIGHT2(lds, u, smallTrig, w, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v @@ -346,6 +359,10 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareFP32 which must dependentLaunchWait before reading data from fftMiddleInFP32 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + readTailFusedLine(inF2, u, line, me); F2 trig = slowTrig_N(line + me * H, ND / NH); @@ -383,6 +400,9 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + readTailFusedLine(inF2, u, line1, me); readTailFusedLine(inF2, v, line2, me); @@ -435,6 +455,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); @@ -492,6 +514,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + // Read lines u and v readTailFusedLine(inF2, u, line, lowMe); @@ -537,6 +561,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { revCrossLine(lds, u); } + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data + new_fft_HEIGHT2(lds, u, smallTrigF2, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v @@ -613,6 +639,10 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareGF31 which must dependentLaunchWait before reading data from fftMiddleInGF31 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + readTailFusedLine(in31, u, line, me); // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) @@ -667,6 +697,9 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + readTailFusedLine(in31, u, line1, me); readTailFusedLine(in31, v, line2, me); @@ -715,6 +748,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); @@ -771,6 +806,8 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + // Read lines u and v readTailFusedLine(in31, u, line, lowMe); @@ -812,6 +849,8 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { revCrossLine(lds, u); } + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data + new_fft_HEIGHT2(lds, u, smallTrig31, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v @@ -821,7 +860,7 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - + /**************************************************************************/ /* Similar to above, but for an NTT based on GF(M61^2) */ @@ -888,6 +927,10 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = which ? (H/2) : 0; u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailSquareGF61 which must dependentLaunchWait before reading data from fftMiddleInGF61 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + readTailFusedLine(in61, u, line, me); // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) @@ -942,6 +985,9 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { u32 memline2 = transPos(line2, MIDDLE, WIDTH); u32 me = get_local_id(0); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + readTailFusedLine(in61, u, line1, me); readTailFusedLine(in61, v, line2, me); @@ -990,6 +1036,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { reverseLine(G_H, lds, v); } + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data + fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); @@ -1046,6 +1094,8 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { u32 line = !isSecondHalf ? line_u : line_v; + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + // Read lines u and v readTailFusedLine(in61, u, line, lowMe); @@ -1087,6 +1137,8 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { revCrossLine(lds, u); } + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data + new_fft_HEIGHT2(lds, u, smallTrig61, 2, SHUFL_BYTES_H, lowMe); // Write lines u and v diff --git a/src/cuda/cudawrap.cpp b/src/cuda/cudawrap.cpp index 905e34c7..474d7522 100644 --- a/src/cuda/cudawrap.cpp +++ b/src/cuda/cudawrap.cpp @@ -495,11 +495,32 @@ std::string NvrtcProgram::compile(const std::string& source, const std::string& // ---- Kernel launcher ---- void CudaKernelLauncher::launch(CUstream stream, u32 gridSize, void** args, u32 sharedMem) { - CU_CHECK(cuLaunchKernel(func, +#if CUDA_VERSION >= 12000 && ENABLE_PDL + // enable pdl in kernel launch attributes + CUlaunchAttribute attrs[1]; + attrs[0].id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION; + attrs[0].value.programmaticStreamSerializationAllowed = 1; + // set kernel launch configuration + CUlaunchConfig_st config = {0}; + config.gridDimX = gridSize; + config.gridDimY = 1; + config.gridDimZ = 1; + config.blockDimX = blockSize; + config.blockDimY = 1; + config.blockDimZ = 1; + config.hStream = stream; + config.sharedMemBytes = sharedMem; + config.attrs = attrs; + config.numAttrs = 1; + // launch the kernel + CU_CHECK(cuLaunchKernelEx(&config, func, args, nullptr)); +#else + CU_CHECK(cuLaunchKernel(func, gridSize, 1, 1, // grid dimensions blockSize, 1, 1, // block dimensions sharedMem, // shared memory bytes stream, // stream args, // kernel arguments - nullptr)); // extra + nullptr)); // extra +#endif } diff --git a/src/cuda/opencl_compat.cuh b/src/cuda/opencl_compat.cuh index 99c15378..92b4dc21 100644 --- a/src/cuda/opencl_compat.cuh +++ b/src/cuda/opencl_compat.cuh @@ -7,6 +7,9 @@ #pragma once +// Set flag that OpenCL can access. Let's us add CUDA-only code to the OpenCL sources. +#define CUDA_BACKEND 1 + // ---- Qualifiers ---- // __kernel / kernel → extern "C" __global__ (CUDA kernel launch qualifier) // extern "C" is needed so cuModuleGetFunction() can find kernels by unmangled name From c98aa56b0521dce8b9821fa9c8744634632e3d9f Mon Sep 17 00:00:00 2001 From: george Date: Sat, 4 Apr 2026 07:27:58 +0000 Subject: [PATCH 023/214] Fixed issues with usin --maxrregcount to limit number of registers used by CUDA kernel --- src/cuda/clwrap_cuda.cpp | 48 ++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 0f24cb63..ad69354d 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -235,6 +235,7 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev } } + int maxregcount = 0; if (options) { istringstream iss(options); string tok; @@ -262,6 +263,9 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev // FMA contraction already enabled above via --fmad=true. // Do NOT use -use_fast_math here — it enables flush-to-zero and // reduced-precision division/sqrt which breaks tailMul accuracy. + } else if (tok.substr(0, 14) == "--maxrregcount") { + nvrtcOpts.push_back(tok); + maxregcount = atoi(tok.substr(15, 3).c_str()); } // Skip other -cl-* options (not applicable to NVRTC) } @@ -281,16 +285,17 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev // Preprocess OpenCL source for CUDA compatibility string processedSrc = NvrtcProgram::preprocessOpenCL(headers[i]->source); // Debug: verify KERNEL macro replacement - { - static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); - if (dumpPrefix && string(headerNames[i]) == "base.cl") { - auto pos = processedSrc.find("KERNEL"); - if (pos != string::npos) { - string ctx = processedSrc.substr(pos > 20 ? pos-20 : 0, 120); - fprintf(stderr, "base.cl KERNEL context: [%s]\n", ctx.c_str()); - } - } - } +// I'm not sure what Sherpa was trying to print out here. It prints out nothing useful. +// { +// static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); +// if (dumpPrefix && string(headerNames[i]) == "base.cl") { +// auto pos = processedSrc.find("KERNEL"); +// if (pos != string::npos) { +// string ctx = processedSrc.substr(pos > 20 ? pos-20 : 0, 120); +// fprintf(stderr, "base.cl KERNEL context: [%s]\n", ctx.c_str()); +// } +// } +// } nvrtcHeaders.push_back({headerNames[i], processedSrc}); } } @@ -341,7 +346,6 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev prog->ptx = NvrtcProgram::compile(processedSource, "prpll_kernel.cu", nvrtcOpts, nvrtcHeaders); prog->compiled = true; g_lastBuildLog.clear(); - return CL_SUCCESS; } catch (const exception& e) { g_lastBuildLog = e.what(); prog->compiled = false; @@ -363,6 +367,21 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev } return CL_COMPILE_PROGRAM_FAILURE; } + + // If --maxrregcount is set it seems nvrtc compile ignores the setting. Instead modify the PTX and load the modified PTX. + + if (maxregcount) { + string maxntidPattern = ".maxntid "; + string maxnregPattern = ".maxnreg " + to_string(maxregcount) + "\n"; + for (size_t startpos = 0; ; ) { + size_t pos = prog->ptx.find(maxntidPattern, startpos); + if (pos == string::npos) break; + prog->ptx.insert(pos, maxnregPattern); + startpos = pos + 20; + } + } + + return CL_SUCCESS; } cl_program clLinkProgram(cl_context ctx, unsigned nDevices, const cl_device_id*, @@ -515,14 +534,15 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { { static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); if (dumpPrefix) { - int numRegs = 0, shmem = 0, maxThreads = 0; + int numRegs = 0, shmem = 0, localmem = 0, maxThreads = 0; cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); + cuFuncGetAttribute(&localmem, CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES , k->func); cuFuncGetAttribute(&maxThreads, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, k->func); - fprintf(stderr, " %-25s: %3d regs, %5d shmem, maxThreads=%d\n", name, numRegs, shmem, maxThreads); + fprintf(stderr, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); // Also write to file since WSL2+CUDA swallows stderr FILE* regLog = fopen("kernel_regs.log", "a"); - if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, maxThreads=%d\n", name, numRegs, shmem, maxThreads); fclose(regLog); } + if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); fclose(regLog); } } } From debd808ce65a9038b763b59af8cf347b0664ac02 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 7 Apr 2026 20:01:27 +0000 Subject: [PATCH 024/214] There is now a default limit on the number of GPU registers to use for the most important CUDA kernels. There are command line options to choose a different maximum number of register, a launch bounds with a minimum number of blocks per SM, or a launch_bounds with no limits. In my limited testing, this yields a small benefit, ~1%. --- src/Args.cpp | 4 + src/Gpu.cpp | 193 +++++++++++++++++++++++++++++++++------ src/Gpu.h | 5 +- src/common.h | 3 + src/cuda/clwrap_cuda.cpp | 8 +- src/tune.cpp | 103 +++++---------------- 6 files changed, 202 insertions(+), 114 deletions(-) diff --git a/src/Args.cpp b/src/Args.cpp index cece2fa6..b9084ba3 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -15,6 +15,9 @@ #include #include +// This is a copy of the args.verbose flag. It allows the CUDA wrapper to access the flag. +bool prpll_verbose = 0; + int Args::value(const string& key, int valNotFound) const { auto it = flags.find(key); if (it == flags.end()) { return valNotFound; } @@ -320,6 +323,7 @@ void Args::parse(const string& line) { carryTune = true; } else if (key == "-verbose" || key == "-v") { verbose = true; + prpll_verbose = true; } else if (key == "-time") { profile = true; } else if (key == "-workers") { diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 4df2c0f9..a3983674 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -264,7 +264,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "TABMUL_CHAIN61", "MODM31", "LOADS","STORES", - "CFBLKS","MIBLKS","MOBLKS","TSBLKS", // CUDA - experimental + "NOREG", // CUDA - experimental "WMUL" }); if (!isValid) { @@ -509,20 +509,157 @@ Gpu::~Gpu() { background->waitEmpty(); } -#define ROE_SIZE 100000 -#define CARRY_SIZE 100000 - +// Part of GPU initialization is to compute the default number of registers each kernel should target during compilation. +// Kernel register usage is critical for maximizing GPU occupancy. The default values can be overrriden with command line arguments. +// This feature currently only works for the CUDA compiler. +string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { #if CUDA_BACKEND -#define CARRYFUSED_BLOCKS(x) args.value("CFBLKS", 0) == 0 ? x : args.value("CFBLKS", 0) == 1 ? x " -DCUDA_MIN_BLOCKS=2" : x " -maxregcount=128" -#define MIDDLEIN_BLOCKS args.value("MIBLKS", 0) == 0 ? "" : args.value("MIBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount=84" -#define MIDDLEOUT_BLOCKS args.value("MOBLKS", 0) == 0 ? "" : args.value("MOBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount=84" -#define TAILSQUARE_BLOCKS args.value("TSBLKS", 0) == 0 ? "" : args.value("TSBLKS", 0) == 1 ? " -DCUDA_MIN_BLOCKS=3" : " -maxregcount=84" + int regs = 0; + const char *use_override = ""; + // Allow command line to prefer the CUDA compiler's default number of registers + if (args.value("NOREG", 0)) return string(""); + // Determine a kernel specific default maximum number of GPU registers (values set to -1 have not been tuned for best default value) + switch (which_kernel) { + case CARRYFUSED: // Register usage depends on NW, the FFT/NTT type, and perhaps the long carry setting + switch (fft.shape.fft_type) { + case FFT64: + regs = nW == 8 ? 72 : 64; + use_override = "REGCF64"; + break; + case FFT3161: + regs = nW == 8 ? 96 : 56; + use_override = "REGCF3161"; + break; + case FFT3261: + regs = nW == 8 ? 96 : 56; + use_override = "REGCF3261"; + break; + case FFT61: + regs = nW == 8 ? 80 : 64; + use_override = "REGCF61"; + break; + case FFT323161: + regs = nW == 8 ? 128 : 72; + use_override = "REGCF323161"; + break; + case FFT3231: + regs = -1; + use_override = "REGCF3231"; + break; + case FFT6431: + regs = -1; + use_override = "REGCF6431"; + break; + case FFT31: + regs = -1; + use_override = "REGCF31"; + break; + case FFT32: + regs = -1; + use_override = "REGCF32"; + break; + } + break; + case MIDIN: // Register usage depends on MIDDLE and the FP32/FP64 + if (fft.FFT_FP64) { + if (fft.shape.middle >= 16) regs = 96; + else if (fft.shape.middle >= 14) regs = 88; + else if (fft.shape.middle >= 13) regs = 80; + else if (fft.shape.middle >= 10) regs = 72; + else if (fft.shape.middle >= 7) regs = 64; + else if (fft.shape.middle >= 5) regs = 56; + else if (fft.shape.middle >= 4) regs = 48; + else regs = -1; + use_override = "REGMI64"; + } else { + if (fft.shape.middle == 16) regs = 56; + else if (fft.shape.middle == 8) regs = 40; + else if (fft.shape.middle == 4) regs = 32; + else regs = -1; + use_override = "REGMI32"; + } + break; + case MIDIN31: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 56; + else if (fft.shape.middle == 8) regs = 44; + else if (fft.shape.middle == 4) regs = 32; + else regs = -1; + use_override = "REGMI31"; + break; + case MIDIN61: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 96; + else if (fft.shape.middle == 8) regs = 72; + else if (fft.shape.middle == 4) regs = 64; + else regs = -1; + use_override = "REGMI61"; + break; + case TAIL: // Register usage depends on NH and the FP32/FP64 (assumes double-wide kernel) + if (fft.FFT_FP64) { + regs = nH == 8 ? 80 : 64; + use_override = "REGTS64"; + } else { + regs = nH == 8 ? 64 : 48; + use_override = "REGTS32"; + } + break; + case TAIL31: // Register usage depends on NH (assumes double-wide kernel) + regs = nH == 8 ? 56 : 48; + use_override = "REGTS31"; + break; + case TAIL61: // Register usage depends on NH (assumes double-wide kernel) + regs = nH == 8 ? 96 : 64; + use_override = "REGTS61"; + break; + case MIDOUT: // Register usage depends on MIDDLE and the FFT/NTT type + if (fft.FFT_FP64) { + if (fft.shape.middle >= 15) regs = 96; + else if (fft.shape.middle >= 14) regs = 88; + else if (fft.shape.middle >= 11) regs = 80; + else if (fft.shape.middle >= 10) regs = 72; + else if (fft.shape.middle >= 7) regs = 64; + else if (fft.shape.middle >= 5) regs = 56; + else if (fft.shape.middle >= 4) regs = 48; + else regs = -1; + use_override = "REGMO64"; + } else { + if (fft.shape.middle == 16) regs = 56; + else if (fft.shape.middle == 8) regs = 40; + else if (fft.shape.middle == 4) regs = 32; + else regs = -1; + use_override = "REGMO32"; + } + break; + case MIDOUT31: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 48; + else if (fft.shape.middle == 8) regs = 40; + else if (fft.shape.middle == 4) regs = 32; + else regs = -1; + use_override = "REGMO31"; + break; + case MIDOUT61: // Register usage depends on MIDDLE + if (fft.shape.middle == 16) regs = 96; + else if (fft.shape.middle == 8) regs = 64; + else if (fft.shape.middle == 4) regs = 64; + else regs = -1; + use_override = "REGMO61"; + break; + } + int override_regs = args.value(use_override, 0); + // Allow command line to set CUDA launch_bounds rather than explicit maximum register count + if (override_regs && args.value("REGLB", 0)) return string("-DCUDA_MIN_BLOCKS=") + to_string(override_regs) + " "; + // Return the maximum register count + if (override_regs) regs = override_regs; + // Sometimes the results using default launch_bounds without setting an explicit maxrrregcount can't be beat + if (regs == -1) return string(""); + // Format an explicit register count setting + return string("--maxrregcount=") + to_string(regs) + " "; #else -#define CARRYFUSED_BLOCKS(x) x -#define MIDDLEIN_BLOCKS "" -#define MIDDLEOUT_BLOCKS "" -#define TAILSQUARE_BLOCKS "" + return string(""); #endif +} + +#define ROE_SIZE 100000 +#define CARRY_SIZE 100000 Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : queue(q), @@ -542,45 +679,43 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& #define K(name, ...) name(#name, &compiler, profile.make(#name), queue, __VA_ARGS__) - K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), MIDDLEIN_BLOCKS), + K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDIN).c_str()), K(kfftHin, "ffthin.cl", "fftHin", hN / nH), K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2), K(ktailSquare, "tailsquare.cl", "tailSquare", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, // Single-wide tailSquare with one kernel - TAILSQUARE_BLOCKS), + hN / nH / 2, numCudaRegisters(TAIL).c_str()), // Single-wide tailSquare with one kernel K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2), K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), MIDDLEOUT_BLOCKS), + K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDOUT).c_str()), K(kfftW, "fftw.cl", "fftW", hN / nW), - K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), MIDDLEIN_BLOCKS), + K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDIN31).c_str()), K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH), - K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2, TAILSQUARE_BLOCKS), + K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2), K(ktailSquareGF31, "tailsquare.cl", "tailSquareGF31", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2), // Single-wide tailSquare with one kernel + hN / nH / 2, numCudaRegisters(TAIL31).c_str()), // Single-wide tailSquare with one kernel K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2), K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), MIDDLEOUT_BLOCKS), + K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDOUT31).c_str()), K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW), - K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), MIDDLEIN_BLOCKS), + K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDIN61).c_str()), K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH), K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2), K(ktailSquareGF61, "tailsquare.cl", "tailSquareGF61", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, // Single-wide tailSquare with one kernel - TAILSQUARE_BLOCKS), + hN / nH / 2, numCudaRegisters(TAIL61).c_str()), // Single-wide tailSquare with one kernel K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2), K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), MIDDLEOUT_BLOCKS), + K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDOUT61).c_str()), K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW), K(kfftP, "fftp.cl", "fftP", hN / nW), @@ -589,11 +724,11 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1"), K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1 -DROE=1"), K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, "-DLL=1"), - K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("")), - K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DROE=1")), - K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DMUL3=1")), - K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DMUL3=1 -DROE=1")), - K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, CARRYFUSED_BLOCKS("-DLL=1")), + K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, numCudaRegisters(CARRYFUSED).c_str()), + K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DROE=1").c_str()), + K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DMUL3=1").c_str()), + K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1").c_str()), + K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DLL=1").c_str()), K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN), diff --git a/src/Gpu.h b/src/Gpu.h index 32c0a407..09b6aa16 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -306,7 +306,7 @@ class Gpu { Saver *getSaver(); void writeIn(Buffer& buf, const vector &words); - + u64 dataResidue() { return bufResidue(bufData); } u64 checkResidue() { return bufResidue(bufCheck); } @@ -318,7 +318,6 @@ class Gpu { vector readCheck(); vector readData(); - u32 getFFTSize() { return N; } // return A^h * B @@ -339,6 +338,8 @@ class Gpu { private: u32 getProofPower(u64 k); void doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors); + enum WHICH_KERNEL {CARRYFUSED=0, MIDIN=1, MIDIN31=2, MIDIN61=3, TAIL=4, TAIL31=5, TAIL61=6, MIDOUT=7, MIDOUT31=8, MIDOUT61=9}; + string numCudaRegisters(enum WHICH_KERNEL which_kernel); }; // Compute the size of an FFT/NTT data buffer depending on the FFT/NTT float/prime. Size is returned in units of sizeof(double). diff --git a/src/common.h b/src/common.h index 2a2c0bce..9bd88471 100644 --- a/src/common.h +++ b/src/common.h @@ -6,6 +6,9 @@ #include #include +// This is a copy of the args.verbose flag. It allows the CUDA wrapper to access the flag. +extern bool prpll_verbose; + using u8 = uint8_t; using i32 = int32_t; using u32 = uint32_t; diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index ad69354d..1e3157af 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -533,7 +533,7 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { // Log register and shared memory usage per kernel when PRPLL_DUMP_PTX is set { static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); - if (dumpPrefix) { + if (prpll_verbose || dumpPrefix) { int numRegs = 0, shmem = 0, localmem = 0, maxThreads = 0; cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); @@ -541,8 +541,10 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { cuFuncGetAttribute(&maxThreads, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, k->func); fprintf(stderr, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); // Also write to file since WSL2+CUDA swallows stderr - FILE* regLog = fopen("kernel_regs.log", "a"); - if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); fclose(regLog); } + if (dumpPrefix) { + FILE* regLog = fopen("kernel_regs.log", "a"); + if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); fclose(regLog); } + } } } diff --git a/src/tune.cpp b/src/tune.cpp index 978f485c..43401475 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -659,7 +659,7 @@ void Tune::tune() { log("Time for %12s using Trig frequently used load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } } - log("Best Trig frquently used load is %u. Default is 0.\n", best_trig_load); + log("Best Trig frequently used load is %u. Default is 0.\n", best_trig_load); loads = loads / 1000 * 1000 + best_trig_load * 100 + loads % 100; args->flags["LOADS" ] = to_string(loads); } @@ -707,85 +707,6 @@ void Tune::tune() { args->flags["STORES"] = to_string(stores); } - // Find best CUDA compiler options -#if CUDA_BACKEND - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_cfblks = 0; - u32 current_cfblks = args->value("CFBLKS", 0); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 cfblks : {0, 1, 2}) { - args->flags["CFBLKS"] = to_string(cfblks); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using CFBLKS=%u is %6.1f\n", fft.spec().c_str(), cfblks, cost); - if (cfblks == current_cfblks) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_cfblks = cfblks; } - } - log("Best CFBLKS is %u. Default CFBLKS is 0.\n", best_cfblks); - configsUpdate(current_cost, best_cost, 0.000, "CFBLKS", best_cfblks, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["CFBLKS"] = to_string(best_cfblks); - } - - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_miblks = 0; - u32 current_miblks = args->value("MIBLKS", 0); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 miblks : {0, 1, 2}) { - args->flags["MIBLKS"] = to_string(miblks); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using MIBLKS=%u is %6.1f\n", fft.spec().c_str(), miblks, cost); - if (miblks == current_miblks) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_miblks = miblks; } - } - log("Best MIBLKS is %u. Default MIBLKS is 0.\n", best_miblks); - configsUpdate(current_cost, best_cost, 0.000, "MIBLKS", best_miblks, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["MIBLKS"] = to_string(best_miblks); - } - - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_moblks = 0; - u32 current_moblks = args->value("MOBLKS", 0); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 moblks : {0, 1, 2}) { - args->flags["MOBLKS"] = to_string(moblks); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using MOBLKS=%u is %6.1f\n", fft.spec().c_str(), moblks, cost); - if (moblks == current_moblks) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_moblks = moblks; } - } - log("Best MOBLKS is %u. Default MOBLKS is 0.\n", best_moblks); - configsUpdate(current_cost, best_cost, 0.000, "MOBLKS", best_moblks, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["MOBLKS"] = to_string(best_moblks); - } - - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); - u32 best_tsblks = 0; - u32 current_tsblks = args->value("TSBLKS", 0); - double best_cost = -1.0; - double current_cost = -1.0; - for (u32 tsblks : {0, 1, 2}) { - args->flags["TSBLKS"] = to_string(tsblks); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); - log("Time for %12s using TSBLKS=%u is %6.1f\n", fft.spec().c_str(), tsblks, cost); - if (tsblks == current_tsblks) current_cost = cost; - if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tsblks = tsblks; } - } - log("Best TSBLKS is %u. Default TSBLKS is 0.\n", best_tsblks); - configsUpdate(current_cost, best_cost, 0.000, "TSBLKS", best_tsblks, newConfigKeyVals, suggestedConfigKeyVals); - args->flags["TSBLKS"] = to_string(best_tsblks); - } -#endif - // Find best FAST_BARRIER setting if (1 /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; @@ -1116,6 +1037,28 @@ void Tune::tune() { args->flags["WMUL"] = to_string(best_wmul); } + // Find best CUDA compiler options +#if CUDA_BACKEND + if (0) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_noreg = 0; + u32 current_noreg = args->value("NOREG", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 noreg : {0, 1}) { + args->flags["NOREG"] = to_string(noreg); + double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using NOREG=%u is %6.1f\n", fft.spec().c_str(), noreg, cost); + if (noreg == current_noreg) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_noreg = noreg; } + } + log("Best NOREG is %u. Default NOREG is 0.\n", best_noreg); + configsUpdate(current_cost, best_cost, 0.000, "NOREG", best_noreg, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["NOREG"] = to_string(best_noreg); + } +#endif + // Find best BIGLIT setting if (0 && time_FFTs) { // Deprecated FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; From 922a31b086e72b97c57532b10410c7572dc5b3b7 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 10 Apr 2026 00:54:45 +0000 Subject: [PATCH 025/214] Halved the number of bar() calls in middleShuffle --- src/cl/fft-middle.cl | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/cl/fft-middle.cl b/src/cl/fft-middle.cl index 03203a89..2f75478c 100644 --- a/src/cl/fft-middle.cl +++ b/src/cl/fft-middle.cl @@ -352,12 +352,13 @@ void OVERLOAD middleShuffle(local T2 *lds, T2 *u) { u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } @@ -626,12 +627,13 @@ void OVERLOAD middleShuffle(local F2 *lds, F2 *u) { u32 y = me / 16; u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } #endif @@ -745,12 +747,13 @@ void OVERLOAD middleShuffle(local GF31 *lds, GF31 *u) { u32 y = me / 16; u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } @@ -886,12 +889,13 @@ void OVERLOAD middleShuffle(local GF61 *lds, GF61 *u) { u32 y = me / 16; u32 x = me % 16; for (int i = 0; i < MIDDLE; ++i) { -// lds[x * 16 + y] = u[i]; - lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts + lds[x * 16 + y ^ x] = u[i]; // Swizzling with XOR should reduce LDS bank conflicts, formerly "lds[x * 16 + y] = u[i];" bar(); -// u[i] = lds[me]; - u[i] = lds[y * 16 + x ^ y]; + u[i] = lds[y * 16 + x ^ y]; // Formerly "u[i] = lds[me];" + if (++i == MIDDLE) break; + lds[y * 16 + x ^ y] = u[i]; bar(); + u[i] = lds[x * 16 + y ^ x]; } } From 84d1d33dc2b80bfe606514e74464632d93b357e1 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 10 Apr 2026 19:41:56 +0000 Subject: [PATCH 026/214] Fixed bug where an NTT shape that nedded to interpolate from the maxbpw table incorrectly used the FP64 bpw table. --- src/FFTConfig.cpp | 8 ++------ src/FFTConfig.h | 3 +-- src/tune.cpp | 8 ++++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index 5441392d..c1009845 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -126,10 +126,6 @@ FFTShape::FFTShape(enum FFT_TYPES t, const string& w, const string& m, const str FFTShape{t, parseInt(w), parseInt(m), parseInt(h)} {} -FFTShape::FFTShape(u32 w, u32 m, u32 h) : - FFTShape(FFT64, w, m, h) { -} - FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : fft_type{t}, width{w}, middle{m}, height{h} { assert(w && m && h); @@ -142,7 +138,7 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : bpw = it->second; } else { if (height > width) { - bpw = FFTShape{h, m, w}.bpw; + bpw = FFTShape{t, h, m, w}.bpw; } else { // Make up some defaults @@ -156,7 +152,7 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : while (w < h || w < 256 || w == 2048) { w *= 2; h /= 2; } while (h < 256) { h *= 2; m /= 2; } if (m == 1) m = 2; - bpw = FFTShape{w, m, h}.bpw; + bpw = FFTShape{t, w, m, h}.bpw; for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) bpw[j] -= 0.05f; // Assume this fft spec is worse than measured fft specs if (this->isFavoredShape()) { // Don't output this warning message for non-favored shapes (we expect the BPW info to be missing) printf("BPW info for %s not found, defaults={", s.c_str()); diff --git a/src/FFTConfig.h b/src/FFTConfig.h index 1b139ea6..e16d610a 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -35,8 +35,7 @@ class FFTShape { u32 height = 0; array bpw; - FFTShape(u32 w = 1, u32 m = 1, u32 h = 1); - FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h); + FFTShape(enum FFT_TYPES t = FFT64, u32 w = 1, u32 m = 1, u32 h = 1); FFTShape(enum FFT_TYPES t, const string& w, const string& m, const string& h); explicit FFTShape(const string& spec); diff --git a/src/tune.cpp b/src/tune.cpp index 43401475..c00956f0 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -192,7 +192,7 @@ void Tune::ztune() { // Study a specific shape and variant if (0) { - FFTShape shape = FFTShape(512, 15, 512); + FFTShape shape = FFTShape(FFT64, 512, 15, 512); u32 variant = 202; u32 sample_size = 5; FFTConfig fft{shape, variant, CARRY_AUTO}; @@ -296,7 +296,7 @@ void Tune::ctune() { // log("tuning %10s with exponent %" PRIu64 "\n", fft.shape.spec().c_str(), exponent); vector bestPos(configsVect.size()); - Entry best{{1, 1, 1}, {}, 1e9}; + Entry best{{}, {}, 1e9}; for (u32 i = 0; i < configsVect.size(); ++i) { for (u32 pos = i ? 1 : 0; pos < configsVect[i].size(); ++pos) { @@ -1192,7 +1192,7 @@ skip_1K_256 = 0; if (auto it = fastest_width_variants.find(shape.width); it != fastest_width_variants.end()) { fastest_width = it->second; } else { - FFTShape test = FFTShape(shape.width, 12, 256); + FFTShape test = FFTShape(FFT64, shape.width, 12, 256); double cost, min_cost = -1.0; for (u32 w = 0; w < N_VARIANT_W; w++) { if (w == 0 && !AMDGPU) continue; @@ -1214,7 +1214,7 @@ skip_1K_256 = 0; if (auto it = fastest_height_variants.find(shape.height); it != fastest_height_variants.end()) { fastest_height = it->second; } else { - FFTShape test = FFTShape(shape.height, 12, shape.height); + FFTShape test = FFTShape(FFT64, shape.height, 12, shape.height); double cost, min_cost = -1.0; for (u32 h = 0; h < N_VARIANT_H; h++) { if (h == 0 && !AMDGPU) continue; From 60a3b10988c949037aa5d9030749db5048669d81 Mon Sep 17 00:00:00 2001 From: george Date: Sat, 11 Apr 2026 16:36:45 +0000 Subject: [PATCH 027/214] Implemented variant 2 width and height for FP32. Unfortunately, it was not faster on a nVidia 5070Ti mobile. For now, the new code is walled off with ENABLE_FP32_VARIANT_2. --- src/Gpu.cpp | 18 ++-- src/TrigBufCache.cpp | 120 ++++++++++++++++++++++ src/TrigBufCache.h | 6 +- src/cl/base.cl | 22 ++++ src/cl/fftbase.cl | 237 +++++++++++++++++++++++++++++++++++++++++++ src/cl/fftheight.cl | 105 ++++++++++++++++++- src/cl/fftwidth.cl | 145 +++++++++++++++++++++++++- src/cl/math.cl | 4 +- src/cl/tailsquare.cl | 64 ++++++------ 9 files changed, 671 insertions(+), 50 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index a3983674..6012cfc7 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -114,17 +114,23 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { vector weightsIF32; // Inverse + Forward for (u32 thread = 0; thread < groupWidth; ++thread) { - auto iw = invWeight32(N, E, H, 0, thread, 0); - auto w = weight32(N, E, H, 0, thread, 0); + auto iw = invWeight32(N, E, H, 0, thread, 0) ;// * 17592186044416.0f; + auto w = weight32(N, E, H, 0, thread, 0) ;// * 0.0000002384185791015625f; + // Play with the weight so that optionalDouble and optionalHalve work + iw = 2.0f * boundUnderOne(iw); + w = 2.0f * w; + // Weights are scaled by 2^-24 and 2^48 so that multiplicaton by 1/epsilon does not generate infinty results (width and height variant 2). + iw = iw * 281474976710656.0f; + w = w * 0.000000059604644775390625f; // nVidia GPUs have a constant cache that only works on buffer sizes less than 64KB. Create a smaller buffer // that is a copy of the first part of weightsIF. There are several kernels that need the combined weightsIF // buffer, so there is an unfortunate duplication of these weights. if (!AmdGpu) { - weightsConstIF32.push_back(2 * boundUnderOne(iw)); - weightsConstIF32.push_back(2 * w); + weightsConstIF32.push_back(iw); + weightsConstIF32.push_back(w); } - weightsIF32.push_back(2 * boundUnderOne(iw)); - weightsIF32.push_back(2 * w); + weightsIF32.push_back(iw); + weightsIF32.push_back(w); } // the group order matches CarryA/M (not fftP/CarryFused). diff --git a/src/TrigBufCache.cpp b/src/TrigBufCache.cpp index 0edb1f29..67c53d73 100644 --- a/src/TrigBufCache.cpp +++ b/src/TrigBufCache.cpp @@ -337,6 +337,58 @@ float2 root1FP32(u32 N, u32 k) { } } +// Epsilon value, 2^-50, should have an exact representation as a float. Used to avoid divide-by-zero in root1overFP32. +const double epsilonFP32 = 8.8817841970012523233890533447266e-16; // Protect against divide by zero + +// Returns the primitive root of unity of order N, to the power k. Returned format is cosine, sine/cosine. +float2 root1overFP32(u32 N, u32 k) { + assert(k < N); + + double angle = M_PI * k / (N / 2); + double c = cos(angle); + double s = sin(angle); + + if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; + s = s / c; + return {float(c), float(s)}; +} + +// Returns the primitive root of unity of order N, to the power k. Returns only the cosine value. +float root1cosFP32(u32 N, u32 k) { + assert(k < N); + + double angle = M_PI * k / (N / 2); + double c = cos(angle); + + if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; + return float(c); +} + +// Returns the primitive root of unity of order N, to the power k. Returns only the cosine value divided by another cosine value. +float root1cosoverFP32(u32 N, u32 k, double over) { + assert(k < N); + + double angle = M_PI * k / (N / 2); + double c = cos(angle); + + if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; + return float(c / over); +} + +// Interleave two lines of trig values so that AMD GPUs can use global_load_dwordx4 instructions +void F2shuffle(u32 size, u32 radix, u32 line, vector &tab) { + vector line1, line2; + u32 line_size = size / radix; + for (u32 col = 0; col < line_size; ++col) { + line1.push_back(tab[line*line_size + col]); + line2.push_back(tab[(line+1)*line_size + col]); + } + for (u32 col = 0; col < line_size; ++col) { + tab[line*line_size + 2*col] = line1[col]; + tab[line*line_size + 2*col + 1] = line2[col]; + } +} + vector genSmallTrigFP32(u32 size, u32 radix) { u32 WG = size / radix; vector tab; @@ -348,6 +400,74 @@ vector genSmallTrigFP32(u32 size, u32 radix) { } } tab.resize(size); + +// New fft_WIDTH and fft_HEIGHT +// We need two versions of trig values. One where we save one more mul and one where we don't. +// In theory, we should always use save one more mul but the rocm optimizer is doing something weird in fft_WIDTH. + + for (u32 save_one_more_mul = 0; save_one_more_mul <= 1; ++save_one_more_mul) { + vector tab1; + if (save_one_more_mul) tab.resize(3*size); + + // Sine/cosine values for first fft4 or fft8 + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + float2 root = root1overFP32(size, col * line); + tab1.push_back(root.second); + } + } + + // Sine/cosine values for later fft4 or fft8 + for (u32 line = 0; line < radix; ++line) { + for (u32 col = 0; col < WG; col += radix) { + float2 root = root1overFP32(size, col * line); + tab1.push_back(root.second); + } + } + + // Cosine values for first fft4 or fft8 (output in post-shufl order) + for (u32 grp = 0; grp < WG; ++grp) { + u32 line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. + for (u32 col = 0; col < radix; ++col) { + float divide_by = 1.0; + // Compute cosine3 / cosine1 + if ((radix == 4 && line == 3) || (radix == 8 && save_one_more_mul && line == 3)) { + divide_by = root1cosFP32(size, col * (grp - 2*(WG/radix))); + } + // Compute cosine5 / cosine1, cosine6 / cosine2, cosine7 / cosine3 + if (radix == 8 && ((save_one_more_mul && line == 5) || line == 6 || line == 7)) { + divide_by = root1cosFP32(size, col * (grp - 4*(WG/radix))); + } + tab1.push_back(root1cosoverFP32(size, col * grp, divide_by)); + } + } + + // Cosine values for later fft4 or fft8 (output in post-shufl order). Similar to cosines above but output every radix-th value. + for (u32 grp = 0; grp < radix; ++grp) { + for (u32 col = 0; col < WG; col += radix) { + u32 line = col / (WG/radix); + double divide_by = 1.0; + // Compute cosine3 / cosine1 + if ((radix == 4 && line == 3) || (radix == 8 && save_one_more_mul && line == 3)) { + divide_by = root1cosFP32(size, grp * (col - 2*(WG/radix))); + } + // Compute cosine5 / cosine1, cosine6 / cosine2, cosine7 / cosine3 + if (radix == 8 && ((save_one_more_mul && line == 5) || line == 6 || line == 7)) { + divide_by = root1cosFP32(size, grp * (col - 4*(WG/radix))); + } + tab1.push_back(root1cosoverFP32(size, grp * col, divide_by)); + } + } + + // Interleave first fft4 or fft8 trig values for faster AMD GPU access + for (u32 i = 0; i < radix-2; i += 2) F2shuffle(size, radix, i, tab1); + for (u32 i = radix; i < 2*radix; i += 2) F2shuffle(size, radix, i, tab1); + + // Convert to a vector of float2 + for (u32 i = 0; i < tab1.size(); i += 2) tab.push_back({tab1[i], tab1[i+1]}); + } + + tab.resize(5*size); return tab; } diff --git a/src/TrigBufCache.h b/src/TrigBufCache.h index d5e5317a..1808b355 100644 --- a/src/TrigBufCache.h +++ b/src/TrigBufCache.h @@ -57,14 +57,14 @@ float2 root1FP32(u32 N, u32 k); uint2 root1GF31(u32 N, u32 k); ulong2 root1GF61(u32 N, u32 k); -// Compute the size of the largest possible trig buffer given width, middle, height (in number of float2 values) +// Compute the size of the largest possible trig buffer given width, middle, height (in number of double2 values) #define SMALLTRIG_FP64_SIZE(W,M,H,nH) (W != H || H == 0 ? W * 5 : SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH)) // See genSmallTrigFP64 #define SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH) (H * 5 + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboFP64 #define MIDDLETRIG_FP64_SIZE(W,M,H) (H + W + H) // See genMiddleTrigFP64 // Compute the size of the largest possible trig buffer given width, middle, height (in number of float2 values) -#define SMALLTRIG_FP32_SIZE(W,M,H,nH) (W != H || H == 0 ? W : SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH)) // See genSmallTrigFP32 -#define SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH) (H + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboFP32 +#define SMALLTRIG_FP32_SIZE(W,M,H,nH) (W != H || H == 0 ? W * 5 : SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH)) // See genSmallTrigFP32 +#define SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH) (H * 5 + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboFP32 #define MIDDLETRIG_FP32_SIZE(W,M,H) (H + W + H) // See genMiddleTrigFP32 // Compute the size of the largest possible trig buffer given width, middle, height (in number of uint2 values) diff --git a/src/cl/base.cl b/src/cl/base.cl index 545dbb91..2fc32c06 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -268,12 +268,14 @@ ulong2 OVERLOAD U2(unsigned long long a, unsigned long long b) { return (ulong2) typedef constant const T2* Trig; typedef constant const T* TrigSingle; typedef constant const F2* TrigFP32; +typedef constant const F* TrigSingleFP32; typedef constant const GF31* TrigGF31; typedef constant const GF61* TrigGF61; #else typedef global const T2* Trig; typedef global const T* TrigSingle; typedef global const F2* TrigFP32; +typedef global const F* TrigSingleFP32; typedef global const GF31* TrigGF31; typedef global const GF61* TrigGF61; #endif @@ -334,6 +336,11 @@ F2 OVERLOAD L2LOAD(CP(F2) mem) { __asm("ld.global.cg.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); return retval; } +F OVERLOAD L2LOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.cg.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} i64 OVERLOAD L2LOAD(i64 *mem) { i64 retval; __asm("ld.global.cg.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); @@ -395,6 +402,11 @@ F2 OVERLOAD EFLOAD(CP(F2) mem) { __asm("ld.global.cs.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); return retval; } +F OVERLOAD EFLOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.cs.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} i64 OVERLOAD EFLOAD(i64 *mem) { i64 retval; __asm("ld.global.cs.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); @@ -456,6 +468,11 @@ F2 OVERLOAD LULOAD(TrigFP32 mem) { __asm("ld.global.lu.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); return retval; } +F OVERLOAD LULOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.lu.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} i64 OVERLOAD LULOAD(i64 *mem) { i64 retval; __asm("ld.global.lu.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); @@ -498,6 +515,11 @@ F2 OVERLOAD NCLOAD(TrigFP32 mem) { __asm("ld.global.nc.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); return retval; } +F OVERLOAD NCLOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.nc.f32 %0}, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} i64 OVERLOAD NCLOAD(i64 *mem) { i64 retval; __asm("ld.global.nc.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 88cd3380..7f581728 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -581,6 +581,243 @@ void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { } } +//************************************************************************************ +// New fft WIDTH and HEIGHT macros to support radix-4 FFTs with more FMA instructions +//************************************************************************************ + +// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. +// We're trying to calculate u * U2(cosine,sine). +// real = (u.x - u.y*sine_over_cosine) * cosine +// imag = (u.x*sine_over_cosine + u.y) * cosine +F2 partial_cmul(F2 u, F sine_over_cosine) { + return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); +} + +// Copy of macro from fft4 and fft8 with FMAs added +#define X2_via_FMA(a, c, b) { F2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } + +// Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. +void preload_tabMul4_trig(u32 WG, TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + + // Read 3 lines of sine/cosine values for the first fft4. Read two of the lines as a pair as AMD likes T2 global memory reads + TrigFP32 trig2 = (TrigFP32) trig1; + F2 sine_over_cosines = TFLOAD(&trig2[me]); + preloads[0] = sine_over_cosines.x; + preloads[1] = sine_over_cosines.y; + // Read 3rd line + preloads[2] = TFLOAD(&trig1[2*WG + me]); +} + +// Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. +void partial_tabMul4(u32 WG, local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { + local F *lds1 = (local F *) lds; + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + trig1 += 4*WG; // Skip past sine_over_cosine values + + // Use LDS memory to distribute preloaded trig values. + if (f > 1) { + bar(WG); + lds1[me] = preloads[4]; // Preloaded sine/cosine values + lds1[WG+me] = preloads[5]; // Preloaded cosine values + } + + // Apply sine/cosines + bar(WG); + for (u32 i = 1; i < 4; ++i) { + F sine_over_cosine; + if (f == 1) sine_over_cosine = preloads[i-1]; + else sine_over_cosine = lds1[i*(WG/4) + (me/f)*(f/4)]; + u[i] = partial_cmul(u[i], sine_over_cosine); + } + + // Preload cosines for finishing first tabMul (done after using up preloaded sine/cosine values). Hopefully, shufl will hide the latency. + if (f == 1) { + // Read pairs of lines to make AMD happy with T2 global memory loads + for (u32 i = 0; i < 4; i += 2) { + TrigFP32 trig2 = (TrigFP32) (trig1 + i*WG); + F2 cosines = TFLOAD(&trig2[me]); + preloads[i] = cosines.x; + preloads[i+1] = cosines.y; + } + } + else { + // Load cosine1, cosine2, cosine3/cosine1 + if (f < WG/4) preloads[0] = lds1[WG + ((me/f) & 3) * WG/4 + (0 * WG + me)/(4*f) * f/4]; + preloads[2] = lds1[WG + ((me/f) & 3) * WG/4 + (2 * WG + me)/(4*f) * f/4]; + preloads[3] = lds1[WG + ((me/f) & 3) * WG/4 + (3 * WG + me)/(4*f) * f/4]; + preloads[1] = lds1[WG + ((me/f) & 3) * WG/4 + (1 * WG + me)/(4*f) * f/4]; + } +} + +// Finish off a partial tabMul while doing next fft4 making more use of FMA. +void finish_tabMul4_fft4(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + + // + // Mimic a traditional fft4 but use FMA instructions to apply the cosine multiplies. + // + + // Apply cosine0 to u[0] + if (f < WG/4) u[0] = u[0] * preloads[0]; + + // Apply cosine2, cosine3/cosine1 to u[2] and u[3] using FMA + X2_via_FMA(u[0], preloads[2], u[2]); + X2_via_FMA(u[1], preloads[3], u[3]); u[3] = mul_t4(u[3]); + + // Preload one line of sine/cosines and one line of cosines for later tabMuls. We'll later broadcast these values as needed using LDS. + if (f == 1) { + preloads[4] = TFLOAD(&trig1[3*WG + me]); // Sine/cosines for later tabMuls + preloads[5] = TFLOAD(&trig1[4*WG + 4*WG + me]); // Cosines for later tabMuls + } + + // Do the last level of fft4 applying cosine1 + X2_via_FMA(u[0], preloads[1], u[1]); + X2_via_FMA(u[2], preloads[1], u[3]); + + // revbin [0, 2, 1, 3] undo + SWAP(u[1], u[2]); +} + +//************************************************************************************ +// New fft WIDTH and HEIGHT macros to support radix-8 FFTs with more FMA instructions +//************************************************************************************ + +// Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. +void preload_tabMul8_trig(u32 WG, TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + + // Read 7 lines of sine/cosine values for the first fft8. Read six of the lines as pairs as AMD likes T2 global memory reads + for (u32 i = 1; i < 7; i += 2) { + TrigFP32 trig2 = (TrigFP32) (trig1 + (i-1)*WG); + F2 sine_over_cosines = TFLOAD(&trig2[me]); + preloads[i-1] = sine_over_cosines.x; + preloads[i] = sine_over_cosines.y; + } + // Read 7th line + preloads[6] = TFLOAD(&trig1[6*WG + me]); +} + +// Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. +void partial_tabMul8(u32 WG, local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { + local F *lds1 = (local F *) lds; + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + trig1 += 8*WG; // Skip past sine_over_cosine values + + // Use LDS memory to distribute preloaded trig values. + if (f > 1) { + bar(WG); + lds1[me] = preloads[8]; // Preloaded sine/cosine values + lds1[WG+me] = preloads[9]; // Preloaded cosine values + } + + // Apply sine/cosines + bar(WG); + for (u32 i = 1; i < 8; ++i) { + F sine_over_cosine; + if (f == 1) sine_over_cosine = preloads[i-1]; + else sine_over_cosine = lds1[i*(WG/8) + (me/f)*(f/8)]; + u[i] = partial_cmul(u[i], sine_over_cosine); + } + + // Preload cosines for finishing first tabMul (done after using up preloaded sine/cosine values). Hopefully, shufl will hide the latency. + if (f == 1) { + // Read pairs of lines to make AMD happy with T2 global memory loads + for (u32 i = 0; i < 8; i += 2) { + TrigFP32 trig2 = (TrigFP32) (trig1 + i*WG); + F2 cosines = TFLOAD(&trig2[me]); + preloads[i] = cosines.x; + preloads[i+1] = cosines.y; + } + } + else { + // Load cosine4, cosine5/cosine1, cosine6/cosine2, cosine7/cosine3, cosine2, cosine3/cosine1, cosine1 + // Load them in the order they will be used, though it probably won't matter. + if (f < WG/8) preloads[0] = lds1[WG + ((me/f) & 7) * WG/8 + (0 * WG + me)/(8*f) * f/8]; + preloads[1] = lds1[WG + ((me/f) & 7) * WG/8 + (1 * WG + me)/(8*f) * f/8]; + preloads[4] = lds1[WG + ((me/f) & 7) * WG/8 + (4 * WG + me)/(8*f) * f/8]; + preloads[5] = lds1[WG + ((me/f) & 7) * WG/8 + (5 * WG + me)/(8*f) * f/8]; + preloads[6] = lds1[WG + ((me/f) & 7) * WG/8 + (6 * WG + me)/(8*f) * f/8]; + preloads[7] = lds1[WG + ((me/f) & 7) * WG/8 + (7 * WG + me)/(8*f) * f/8]; + preloads[2] = lds1[WG + ((me/f) & 7) * WG/8 + (2 * WG + me)/(8*f) * f/8]; + preloads[3] = lds1[WG + ((me/f) & 7) * WG/8 + (3 * WG + me)/(8*f) * f/8]; + } +} + +// Finish off a partial tabMul while doing next fft8 making more use of FMA. +void finish_tabMul8_fft8(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { + TrigSingleFP32 trig1 = (TrigSingleFP32) trig; + + // + // Mimic a traditional fft8 but use FMA instructions to apply the cosine multiplies. + // + + // Apply cosine0 to u[0] + if (f < WG/8) u[0] = u[0] * preloads[0]; + + if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in new_fft_WIDTH case. + + // Apply cosine4, cosine5/cosine1, cosine6/cosine2, cosine7/cosine3 to u[4] through u[7] using FMA + X2_via_FMA(u[0], preloads[4], u[4]); + X2_via_FMA(u[1], preloads[5], u[5]); u[5] = mul_t8_delayed(u[5]); + X2_via_FMA(u[2], preloads[6], u[6]); u[6] = mul_t4(u[6]); + X2_via_FMA(u[3], preloads[7], u[7]); u[7] = mul_3t8_delayed(u[7]); + + // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. + if (f == 1) { + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul + } + + // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3/cosine1 + X2_via_FMA(u[0], preloads[2], u[2]); + X2_via_FMA(u[4], preloads[2], u[6]); + X2_via_FMA(u[1], preloads[3], u[3]); u[3] = mul_t4(u[3]); + X2_via_FMA(u[5], preloads[3], u[7]); u[7] = mul_t4(u[7]); + + // Do last level of fft8 applying cosine1 +//TODO: Save this MUL by SQRT(1/2) by pre-computing cosine1*SQRTHALF + F cosine1_SQRT1_2 = preloads[1] * (float) M_SQRT1_2; + X2_via_FMA(u[0], preloads[1], u[1]); + X2_via_FMA(u[2], preloads[1], u[3]); + X2_via_FMA(u[4], cosine1_SQRT1_2, u[5]); + X2_via_FMA(u[6], cosine1_SQRT1_2, u[7]); + + } else { + + // Apply cosine to u[1] + u[1] = u[1] * preloads[1]; + + // Apply cosine4, cosine5, cosine6/cosine2, cosine7/cosine3 to u[4] through u[7] using FMA + X2_via_FMA(u[0], preloads[4], u[4]); + X2_via_FMA(u[1], preloads[5], u[5]); u[5] = mul_t8_delayed(u[5]); + X2_via_FMA(u[2], preloads[6], u[6]); u[6] = mul_t4(u[6]); + X2_via_FMA(u[3], preloads[7], u[7]); u[7] = mul_3t8_delayed(u[7]); + + // Preload one line of sine/cosines and one line of cosines for second tabMul. We'll later broadcast these values as needed using LDS. + if (f == 1) { + preloads[8] = TFLOAD(&trig1[7*WG + me]); // Sine/cosines for second tabMul + preloads[9] = TFLOAD(&trig1[8*WG + 8*WG + me]); // Cosines for second tabMul + } + + // Do the fft4Core and fft4CoreSpecial applying cosine2, cosine3 + X2_via_FMA(u[0], preloads[2], u[2]); + X2_via_FMA(u[4], preloads[2], u[6]); + X2_via_FMA(u[1], preloads[3], u[3]); u[3] = mul_t4(u[3]); + X2_via_FMA(u[5], preloads[3], u[7]); u[7] = mul_t4(u[7]); + + // Do last level of fft8 + X2(u[0], u[1]); + X2(u[2], u[3]); + X2_apply_delay(u[4], u[5]); + X2_apply_delay(u[6], u[7]); + } + + // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + #endif diff --git a/src/cl/fftheight.cl b/src/cl/fftheight.cl index 49323782..80e148db 100644 --- a/src/cl/fftheight.cl +++ b/src/cl/fftheight.cl @@ -203,8 +203,109 @@ void OVERLOAD fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u fft_NH(u); } -void new_fft_HEIGHT1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } -void new_fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } +void OVERLOAD new_fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe, int callnum) { + u32 WG = SMALL_HEIGHT / NH; + + // This line mimics shufl -- partition lds + local F2* partitioned_lds = lds; + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * SMALL_HEIGHT * sb / sizeof(F2); + +// Custom code for various SMALL_HEIGHT values + +#if ENABLE_FP32_VARIANT_2 && SMALL_HEIGHT == 256 && NH == 4 && FFT_VARIANT_H == 2 + +// Custom code for SMALL_HEIGHT=256, NH=4 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); + + // Finish third tabMul and perform final fft4. + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); + +#elif ENABLE_FP32_VARIANT_2 && SMALL_HEIGHT == 512 && NH == 8 && FFT_VARIANT_H == 2 + +// Custom code for SMALL_HEIGHT=512, NH=8 + + F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8 + 2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(WG, lds, u, NH, 8, numWG, sb, lowMe); + + // Finish second tabMul and perform final fft8. + finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 1); + +#elif ENABLE_FP32_VARIANT_2 && SMALL_HEIGHT == 1024 && NH == 4 && FFT_VARIANT_H == 2 + +// Custom code for SMALL_HEIGHT=1024, NH=4 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); + + // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(WG, lds, u, NH, 64, numWG, sb, lowMe); + + // Finish fourth tabMul and perform final fft4. + finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); + +#else + + // Old version + fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); + +#endif +} + +void new_fft_HEIGHT1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, numWG, sb, lowMe, 1); } +void new_fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, numWG, sb, lowMe, 2); } #endif diff --git a/src/cl/fftwidth.cl b/src/cl/fftwidth.cl index 03935149..11c5d1cf 100644 --- a/src/cl/fftwidth.cl +++ b/src/cl/fftwidth.cl @@ -73,7 +73,7 @@ void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb // To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. // The downside is sine/cosine cannot be computed with chained multiplies. -void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe, const u32 sb, int callnum) { +void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe, int callnum) { u32 WG = WIDTH / NW; // This line mimics shufl -- partition lds @@ -203,8 +203,8 @@ void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowM } // There are two version of new_fft_WIDTH in case we want to try saving some trig values from new_fft_WIDTH1 in LDS memory for later use in new_fft_WIDTH2. -void OVERLOAD new_fft_WIDTH1(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, lowMe, sb, 1); } -void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, lowMe, sb, 2); } +void OVERLOAD new_fft_WIDTH1(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 1); } +void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 2); } #endif @@ -239,8 +239,143 @@ void OVERLOAD fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u3 fft_NW(u); } -void OVERLOAD new_fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } -void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } +// New fft_WIDTH that uses more FMA instructions than the old fft_WIDTH. +// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. +// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. +// The downside is sine/cosine cannot be computed with chained multiplies. + +void OVERLOAD new_fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe, int callnum) { + u32 WG = WIDTH / NW; + + // This line mimics shufl -- partition lds + local F2* partitioned_lds = lds; + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * WIDTH * sb / sizeof(F2); + +// Custom code for various WIDTH values + +#if ENABLE_FP32_VARIANT_2 && WIDTH == 256 && NW == 4 && FFT_VARIANT_W == 2 + +// Custom code for WIDTH=256, NW=4 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); + + // Finish third tabMul and perform final fft4. + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); + +#elif ENABLE_FP32_VARIANT_2 && WIDTH == 512 && NW == 8 && FFT_VARIANT_W == 2 + +// Custom code for WIDTH=512, NW=8 + + F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8; // Skip past old FFT_width trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); + + // Finish second tabMul and perform final fft8. + finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + +#elif ENABLE_FP32_VARIANT_2 && WIDTH == 1024 && NW == 4 && FFT_VARIANT_W == 2 + +// Custom code for WIDTH=1024, NW=4 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); + + // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. + finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); + + // Finish fourth tabMul and perform final fft4. + finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); + +#elif ENABLE_FP32_VARIANT_2 && WIDTH == 4096 && NW == 8 && FFT_VARIANT_W == 2 + +// Custom code for WIDTH=4K, NW=8 + + F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); + + // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. + finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); + + // Finish third tabMul and perform final fft8. + finish_tabMul8_fft8(WG, trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + +#else + + // Old version + fft_WIDTH(lds, u, trig, numWG, sb, lowMe); + +#endif +} + +// There are two version of new_fft_WIDTH in case we want to try saving some trig values from new_fft_WIDTH1 in LDS memory for later use in new_fft_WIDTH2. +void OVERLOAD new_fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 1); } +void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 2); } #endif diff --git a/src/cl/math.cl b/src/cl/math.cl index 28401a7b..51829cb6 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -454,7 +454,7 @@ void cmul_a_by_fancyb_and_conjfancyb(F2 *res1, F2 *res2, F2 a, F2 b) { F2 OVERLOAD mul_t4(F2 a) { return U2(-a.y, a.x); } // i.e. a * i -F2 OVERLOAD mul_t8(F2 a) { // mul(a, U2(1, 1)) * (T)(M_SQRT1_2); } +F2 OVERLOAD mul_t8(F2 a) { // mul(a, U2(1, 1)) * (F)(M_SQRT1_2); } // One mul, two FMAs F ay = a.y * (float) M_SQRT1_2; return U2(fma(a.x, (float) M_SQRT1_2, -ay), fma(a.x, (float) M_SQRT1_2, ay)); @@ -462,7 +462,7 @@ F2 OVERLOAD mul_t8(F2 a) { // mul(a, U2(1, 1)) * (T)(M_SQRT1_2); } // return U2(a.x - a.y, a.x + a.y) * M_SQRT1_2; } -F2 OVERLOAD mul_3t8(F2 a) { // mul(a, U2(-1, 1)) * (T)(M_SQRT1_2); } +F2 OVERLOAD mul_3t8(F2 a) { // mul(a, U2(-1, 1)) * (F)(M_SQRT1_2); } // One mul, two FMAs F ay = a.y * (float) M_SQRT1_2; return U2(fma(-a.x, (float) M_SQRT1_2, -ay), fma(a.x, (float) M_SQRT1_2, -ay)); diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 29832a8b..48f49aa5 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -83,12 +83,12 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { T2 trig = slowTrig_N(line + me * H, ND / NH); - fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), me); } @@ -127,8 +127,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #endif u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds + zerohack, v, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, v, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS == 2 @@ -177,8 +177,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out, memline2, me); writeTailFusedLine(u, out, memline1, me); @@ -367,12 +367,12 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { F2 trig = slowTrig_N(line + me * H, ND / NH); - fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), me); } @@ -407,18 +407,18 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(inF2, v, line2, me); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds + zerohack, v, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, v, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); - // Compute trig values from scratch. Good on GPUs with high DP throughput. + // Compute trig values from scratch. Good on GPUs with high FP throughput. #if TAIL_TRIGS32 == 2 F2 trig = slowTrig_N(line1 + me * H, ND / NH); - // Do a little bit of memory access and a little bit of DP math. Good on a Radeon VII. + // Do a little bit of memory access and a little bit of FP math. #elif TAIL_TRIGS32 == 1 // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached F2 per line F2 trig = TFLOAD(&smallTrigF2[height_trigs + me]); // Trig values for line zero, should be cached F2 mult = TSLOAD(&smallTrigF2[height_trigs + G_H + line1]); // Line multiplier @@ -428,7 +428,7 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #else // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values F2 trig = TOLOAD(&smallTrigF2[height_trigs + line1*G_H + me]); #endif @@ -457,8 +457,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, outF2, memline2, me); writeTailFusedLine(u, outF2, memline1, me); @@ -522,15 +522,15 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; new_fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 2, SHUFL_BYTES_H, lowMe); - // Compute trig values from scratch. Good on GPUs with high DP throughput. + // Compute trig values from scratch. Good on GPUs with high FP throughput. #if TAIL_TRIGS32 == 2 F2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); - // Do a little bit of memory access and a little bit of DP math. Good on a Radeon VII. + // Do a little bit of memory access and a little bit of FP math. #elif TAIL_TRIGS32 == 1 // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read a hopefully cached line of data and one non-cached F2 per line F2 trig = TFLOAD(&smallTrigF2[height_trigs + lowMe]); // Trig values for line zero, should be cached F2 mult = TSLOAD(&smallTrigF2[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. @@ -540,7 +540,7 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #else // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. - u32 height_trigs = SMALL_HEIGHT*1; + u32 height_trigs = SMALL_HEIGHT*5; // Read pre-computed trig values F2 trig = TOLOAD(&smallTrigF2[height_trigs + line_u*G_H*2 + me]); #endif @@ -664,12 +664,12 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), me); } @@ -704,8 +704,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in31, v, line2, me); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds + zerohack, v, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, v, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS31 >= 1 @@ -750,8 +750,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out31, memline2, me); writeTailFusedLine(u, out31, memline1, me); @@ -952,12 +952,12 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); reverse(G_H, lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); reverse(G_H, lds, u + NH/2, !which); - fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), me); } @@ -992,8 +992,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in61, v, line2, me); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - fft_HEIGHT(lds + zerohack, u, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds + zerohack, v, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT1(lds + zerohack, v, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS61 >= 1 @@ -1038,8 +1038,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); + new_fft_HEIGHT2(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); writeTailFusedLine(v, out61, memline2, me); writeTailFusedLine(u, out61, memline1, me); From fef7c3adb9808edb034c6573de5977bf33cf1a1b Mon Sep 17 00:00:00 2001 From: george Date: Sat, 11 Apr 2026 20:44:10 +0000 Subject: [PATCH 028/214] If type 2 NTTs (GF31 + GF61), replaced two 64-bit shifts with one 64-bit shift and one 32-bit shift. It probably makes little speed difference. --- src/cl/carryutil.cl | 4 ++-- src/cl/math.cl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 4e04812b..f31fdace 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -427,8 +427,8 @@ i96 weightAndCarryOne(Z31 u31, Z61 u61, u32 m31_invWeight, u32 m61_invWeight, bo *maxROE = max(*maxROE, roundoff); // Compute the value using i96 math - i64 vhi = n61 >> 33; - u64 vlo = ((u64)n61 << 31) | n31; + i64 vhi = n61 >> 1; + u32 vlo = ((u32)n61 << 31) | n31; i96 value = make_i96(vhi, vlo); // (n61 << 31) + n31 value = sub(value, n61); // n61 * M31 + n31 diff --git a/src/cl/math.cl b/src/cl/math.cl index 51829cb6..74f343e2 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -18,6 +18,7 @@ i32 OVERLOAD hi32(i64 x) { uint2 x2 = as_uint2(x); return (i32)x2.y; } typedef struct { i32 hi32; u32 mid32; u32 lo32; } i96; i96 OVERLOAD make_i96(i64 v) { i96 val; val.lo32 = lo32(v); val.mid32 = hi32(v); val.hi32 = (i32)val.mid32 >> 31; return val; } i96 OVERLOAD make_i96(i32 v) { i96 val; val.lo32 = v; val.mid32 = val.hi32 = (i32)val.lo32 >> 31; return val; } +i96 OVERLOAD make_i96(i64 hi, u32 lo) { i96 val; val.hi32 = hi32(hi); val.mid32 = lo32(hi); val.lo32 = lo; return val; } i96 OVERLOAD make_i96(i64 hi, u64 lo) { i96 val; val.hi32 = hi; val.mid32 = hi32(lo); val.lo32 = lo32(lo); return val; } i96 OVERLOAD make_i96(i32 hi, u64 lo) { i96 val; val.hi32 = hi; val.mid32 = hi32(lo); val.lo32 = lo32(lo); return val; } u32 i96_hi32(i96 val) { return val.hi32; } From f4bbd40e0a9c5c2364760c75b848ffa8e1df7374 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 13 Apr 2026 08:41:06 +0000 Subject: [PATCH 029/214] Cleaned up all code that dealt with splitting 64-bit values or combining 32-bit values. No more >> 32 or << 32 code. This generates cleaner PTX code. Faster carry propagation in type-4 FFTs. --- src/cl/carryutil.cl | 50 ++++++++++++++++++++++++-------------- src/cl/math.cl | 49 ++++++++++++++++++++++++++++++------- src/cuda/opencl_compat.cuh | 44 +++++++++------------------------ 3 files changed, 84 insertions(+), 59 deletions(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index f31fdace..5a77e511 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -285,7 +285,7 @@ i64 weightAndCarryOne(Z61 u, u32 invWeight, i64 inCarry, u32* maxROE) { i64 value = get_balanced_Z61(u); // Optionally calculate roundoff error as proximity to M61/2. 28 bits of accuracy should be sufficient. - u32 roundoff = (u32) abs((i32) (value >> 32)); + u32 roundoff = (u32) abs((i32) hi32(value)); *maxROE = max(*maxROE, roundoff); // Mul by 3 and add carry @@ -375,9 +375,9 @@ i96 weightAndCarryOne(float uF2, Z61 u61, float F2_invWeight, u32 m61_invWeight, u61 = shr(u61, m61_invWeight); u64 n61 = get_Z61(u61); - // The final result must be n61 mod M61. Use FP32 data to calculate this value. - float n61f = (float)((u32)(n61 >> 32)) * -4294967296.0f; // Conversion from u64 to float might be slow, this might be faster - uF2 = fma(uF2, F2_invWeight, n61f); // This should be close to a multiple of M61 + // The final result mod M61 must be n61. Use FP32 data to calculate how many multiples of M61 need to be added to n61. + float n61f = (float)hi32(n61) * -4294967296.0f; // Estimate -n61 as a float. + uF2 = fma(uF2, F2_invWeight, n61f); // This should be close to an integer multiple of M61 float uF2int = fma(uF2, 4.3368086899420177360298112034798e-19f, RNDVAL); // Divide by M61 and round to int i32 nF2 = RNDVALfloatToInt(uF2int); @@ -423,7 +423,7 @@ i96 weightAndCarryOne(Z31 u31, Z61 u61, u32 m31_invWeight, u32 m61_invWeight, bo i64 n61 = get_balanced_Z61(u61); // Optionally calculate roundoff error as proximity to M61/2. 28 bits of accuracy should be sufficient. - u32 roundoff = (u32) abs((i32)(n61 >> 32)); + u32 roundoff = (u32) abs((i32)hi32(n61)); *maxROE = max(*maxROE, roundoff); // Compute the value using i96 math @@ -458,20 +458,34 @@ i128 weightAndCarryOne(float uF2, Z31 u31, Z61 u61, float F2_invWeight, u32 m31_ u61 = subq(u61, make_Z61(n31), 2); // u61 - u31 u61 = add(u61, shl(u61, 31)); // u61 + (u61 << 31) u64 n61 = get_Z61(u61); + // Let's call the 92-bit CRT result n3161. At this point, n3161 = n61 * M31 + n31. - i128 n3161 = make_i128(n61 >> 33, (n61 << 31) | n31); // n61 << 31 + n31 - n3161 = sub(n3161, n61); // n61 * M31 + n31 - - // The final result must be n3161 mod M31*M61. Use FP32 data to calculate this value. - float n3161f = (float)((u32)(n61 >> 32)) * -9223372036854775808.0f; // Converting n3161 from i128 to float might be slow, this might be faster - uF2 = fma(uF2, F2_invWeight, n3161f); // This should be close to a multiple of M31*M61 + // The final result mod M31*M61 must be n3161. Use FP32 data to calculate how many multiples of M31*M61 need to be added to n31n61. + float n3161f = (float)hi32(n61) * -9223372036854775808.0f; // Estimate -n3161 as a float. -n61 << 31 should be close enough. + uF2 = fma(uF2, F2_invWeight, n3161f); // This should be close to an integer multiple of M31*M61 float uF2int = fma(uF2, 2.0194839183061857038255724444152e-28f, RNDVAL); // Divide by M31*M61 and round to int i32 nF2 = RNDVALfloatToInt(uF2int); - i64 nF2m31 = ((i64)nF2 << 31) - nF2; // nF2 * M31 - i128 v = make_i128(nF2m31 >> 3, (u64)nF2m31 << 61); // nF2m31 << 61 - v = sub(v, nF2m31); // nF2m31 * M61 - v = add(v, n3161); // nF2m31 * M61 + n3161 + // The final result will be nF2 * M31*M61 + n3161. Rearranging to use as few 128-bit and 64-bit ops as possible: + // = nF2 * M61 * M31 + n61 * M31 + n31 + // = (nF2 * M61 + n61) * M31 + n31 + // = ((nF2 << 61) - nF2 + n61) * M31 + n31 + // = (((nF2 << 61) - nF2 + n61) << 31) - ((nF2 << 61) - nF2 + n61) + n31 + // = (nF2 << 92) + ((n61 - nF2) << 31) - (nF2 << 61) - (n61 - nF2) + n31 + // = (nF2 << 92) - (nF2 << 61) + ((n61 - nF2) << 31) - (n61 - nF2) + n31 + // = (((nF2 << 31) - nF2) << 61) + ((n61 - nF2) << 31) - (n61 - nF2) + n31 + // = (((nF2 << 32) - nF2*2) << 60) + ((n61 - nF2) << 31) - (n61 - nF2) + n31 + + // Compute x = (n61 - nF2) + i64 x = (i64)n61 - nF2; + // Compute y = ((n61 - nF2) << 31) + n31 + i128 y = make_i128(x >> 33, (x << 31) | n31); + // Compute z = ((nF2 << 32) - nF2*2) << 60 + i64 tmp = make_i64(nF2, 0) - (i64)(nF2 + nF2); + i128 z = make_i128(tmp >> 4, tmp << 60); + + // Put the parts together + i128 v = sub(add(z, y), x); // Optionally calculate roundoff error float roundoff = fabs(fma(uF2, 2.0194839183061857038255724444152e-28f, RNDVAL - uF2int)); @@ -538,7 +552,7 @@ Word OVERLOAD carryStep(i64 x, i64 *outCarry, bool isBigWord) { #elif EXP / NWORDS == 32 i32 xhi = hi32(x); i64 w = lowBits(x, nBits); - xhi -= (i32)(w >> 32); + xhi -= (i32)hi32(w); *outCarry = xhi >> (nBits - 32); return w; #elif EXP / NWORDS == 31 @@ -717,7 +731,7 @@ Word OVERLOAD carryStepSignedSloppy(i64 x, i32 *outCarry, bool isBigWord) { #if EXP / NWORDS >= 32 // nBits is 32 or more u64 x_topbit = x & ((u64)1 << (bigwordBits - 1)); i64 w = ulowFixedBits(x, bigwordBits - 1) - x_topbit; - i32 xhi = (i32)(x >> 32) + (i32)(x_topbit >> 32); + i32 xhi = (i32)hi32(x) + (i32)hi32(x_topbit); *outCarry = xhi >> (nBits - 32); return w; // nBits = 31 or 32, bigwordBits = 32 (or allowed to create 32-bit word for better performance). For reasons I don't fully understand the sloppy @@ -725,7 +739,7 @@ Word OVERLOAD carryStepSignedSloppy(i64 x, i32 *outCarry, bool isBigWord) { // Not a major concern as end users should avoid small BPW as there is probably a more efficient NTT that could be used. #elif EXP / NWORDS == 31 || (EXP / NWORDS >= 23 && SLOPPY_MAXBPW >= 3200) i32 w = x; // lowBits(x, bigwordBits = 32); - *outCarry = ((i32)(x >> 32) + (w < 0)) << (32 - nBits); + *outCarry = ((i32)hi32(x) + (w < 0)) << (32 - nBits); return w; #else // nBits less than 32 //GWBUG - is there a faster version? Is this faster than plain old carryStep? No // u32 x_topbit = (u32) x & (1 << (bigwordBits - 1)); diff --git a/src/cl/math.cl b/src/cl/math.cl index 74f343e2..b53fccbb 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -6,10 +6,13 @@ // Access parts of a 64-bit value -u32 OVERLOAD lo32(u64 x) { uint2 x2 = as_uint2(x); return (u32)x2.x; } -u32 OVERLOAD hi32(u64 x) { uint2 x2 = as_uint2(x); return (u32)x2.y; } -u32 OVERLOAD lo32(i64 x) { uint2 x2 = as_uint2(x); return (u32)x2.x; } -i32 OVERLOAD hi32(i64 x) { uint2 x2 = as_uint2(x); return (i32)x2.y; } +u32 OVERLOAD lo32(u64 x) { return (u32)x; } +u32 OVERLOAD hi32(u64 x) { union { uint2 ui2; u64 ul; } u; u.ul = x; return u.ui2.y; } +u32 OVERLOAD lo32(i64 x) { return (u32)x; } +u32 OVERLOAD hi32(i64 x) { union { uint2 ui2; u64 ul; } u; u.ul = x; return u.ui2.y; } + +u64 OVERLOAD make_u64(u32 hi, u32 lo) { union { uint2 ui2; u64 ul; } u; u.ui2.x = lo; u.ui2.y = hi; return u.ul; } +i64 OVERLOAD make_i64(i32 hi, u32 lo) { union { uint2 ui2; u64 ul; } u; u.ui2.x = lo; u.ui2.y = hi; return u.ul; } // A primitive partial implementation of an i96 integer type #if 1 @@ -65,7 +68,7 @@ i96 OVERLOAD make_i96(i32 v) { i96 val; val.x = v; return val; } i96 OVERLOAD make_i96(i64 hi, u64 lo) { i96 val; val.x = ((unsigned __int128)hi << 64) + lo; return val; } i96 OVERLOAD make_i96(i32 hi, u64 lo) { return make_i96((i64)hi, lo); } u32 i96_hi32(i96 val) { return (unsigned __int128)val.x >> 64; } -u32 i96_mid32(i96 val) { return (u64)val.x >> 32; } +u32 i96_mid32(i96 val) { return hi32((u64)val.x); } u32 i96_lo32(i96 val) { return val.x; } u64 i96_lo64(i96 val) { return val.x; } u64 i96_hi64(i96 val) { return (unsigned __int128)val.x >> 32; } @@ -86,7 +89,7 @@ u32 i96_hi32(i96 val) { return val.hi32; } u32 i96_mid32(i96 val) { return hi32(val.lo64); } u32 i96_lo32(i96 val) { return val.lo64; } u64 i96_lo64(i96 val) { return val.lo64; } -u64 i96_hi64(i96 val) { return ((u64) val.hi32 << 32) | i96_mid32(val); } +u64 i96_hi64(i96 val) { return make_64(val.hi32, i96_mid32(val)); } i96 OVERLOAD add(i96 a, i96 b) { i96 val; val.lo64 = a.lo64 + b.lo64; val.hi32 = a.hi32 + b.hi32 + (val.lo64 < a.lo64); return val; } i96 OVERLOAD add(i96 a, i64 b) { return add(a, make_i96(b)); } i96 OVERLOAD sub(i96 a, i96 b) { i96 val; val.lo64 = a.lo64 - b.lo64; val.hi32 = a.hi32 - b.hi32 - (val.lo64 > a.lo64); return val; } @@ -172,7 +175,7 @@ i32 optional_sub(i32 a, const i32 b) { // Optionally subtract a value if first arg is greater than value. i32 optional_mod(i32 a, const i32 b) { -#if ENABLE_OPTIONAL_MOD && HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (not sure why) +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (too small a gain to measure??) __asm("{.reg .pred %%p;\n\t" " setp.ge.s32 %%p, %0, %1;\n\t" // a > b " @%%p sub.s32 %0, %0, %1;}" // if (a > b) a = a - b @@ -183,6 +186,34 @@ i32 optional_mod(i32 a, const i32 b) { return a; } +#if 0 // These are not used because there is no asm constraint to indicate a 64-bit constant +// Optionally subtract c from a if a >= b. +u64 OVERLOAD optional_sub(u64 a, const u64 b, const u64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.u64 %%p, %0, %1;\n\t" // a >= b + " @%%p sub.u64 %0, %0, %2;}" // if (a >= b) a = a - c + : "+l"(a) : "l"(b), "l"(c)); +#else + if (a >= b) a = a - c; +#endif + return a; +} + +// Optionally subtract c from a if hi32(a) >= b. +u64 OVERLOAD optional_sub(u64 a, const u32 b, const u64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.u32 %%p, %1, %2;\n\t" // hi32(a) >= b + " @%%p sub.u64 %0, %0, %3;}" // if (hi32(a) >= b) a = a - c + : "+l"(a) : "r"(hi32(a)), "n"(b), "l"(c)); +#else + if (hi32(a) >= b) a = a - c; +#endif + return a; +} +#endif + // Multiply and add primitives u64 OVERLOAD mad32(u32 a, u32 b, u32 c) { @@ -912,8 +943,8 @@ void OVERLOAD X2s_conjb(GF61 *a, GF61 *b, u32 m61_count) { X2_conjb_internal(a, #elif 1 // Faster version that keeps results in the range 0 .. M61+epsilon -u64 OVERLOAD get_Z61(Z61 a) { Z61 m = a - M61; return (m & 0x8000000000000000ULL) ? a : m; } // Get value in range 0 to M61-1 -i64 OVERLOAD get_balanced_Z61(Z61 a) { return (a >= 0x1000000000000000ULL) ? (i64) a - (i64) M61 : (i64) a; } // Get balanced value in range -M61/2 to M61/2 +u64 OVERLOAD get_Z61(Z61 a) { Z61 m = a - M61; return (m & 0x8000000000000000ULL) ? a : m; } // Get value in range 0 to M61-1 +i64 OVERLOAD get_balanced_Z61(Z61 a) { return (hi32(a) >= 0x10000000) ? (i64)(a - M61) : (i64)a; } // Get balanced value in range -M61/2 to M61/2 // Internal routine to bring Z61 value into the range 0..M61+epsilon Z61 OVERLOAD modM61(Z61 a) { return (a & M61) + (a >> 61); } diff --git a/src/cuda/opencl_compat.cuh b/src/cuda/opencl_compat.cuh index 92b4dc21..b72ef5e4 100644 --- a/src/cuda/opencl_compat.cuh +++ b/src/cuda/opencl_compat.cuh @@ -176,48 +176,28 @@ __device__ __forceinline__ ulong2 operator*(ulong2 v, int s) { return {v.x*(ulon // Scalar ↔ vector bitwise reinterpretations (OpenCL as_type functions) // as_uint2: split 64-bit value into two 32-bit halves -__device__ __forceinline__ uint2 as_uint2(double v) { - ulong bits = __double_as_longlong(v); - return make_uint2((uint)(bits), (uint)(bits >> 32)); -} -__device__ __forceinline__ uint2 as_uint2(ulong v) { - return make_uint2((uint)(v), (uint)(v >> 32)); -} -__device__ __forceinline__ uint2 as_uint2(i64 v) { - return make_uint2((uint)((ulong)v), (uint)((ulong)v >> 32)); -} +__device__ __forceinline__ uint2 as_uint2(double v) { union { uint2 ui2; double d; } u; u.d = v; return u.ui2; } +__device__ __forceinline__ uint2 as_uint2(ulong v) { union { uint2 ui2; ulong ul; } u; u.ul = v; return u.ui2; } +__device__ __forceinline__ uint2 as_uint2(i64 v) { union { uint2 ui2; i64 l; } u; u.l = v; return u.ui2; } // as_int2: split 64-bit value into two signed 32-bit halves -__device__ __forceinline__ int2 as_int2(double v) { - ulong bits = __double_as_longlong(v); - return make_int2((int)(uint)(bits), (int)(uint)(bits >> 32)); -} -__device__ __forceinline__ int2 as_int2(i64 v) { - return make_int2((int)(uint)((ulong)v), (int)(uint)((ulong)v >> 32)); -} +__device__ __forceinline__ int2 as_int2(double v) { union { int2 i2; double d; } u; u.d = v; return u.i2; } +__device__ __forceinline__ int2 as_int2(i64 v) { union { int2 i2; i64 l; } u; u.l = v; return u.i2; } // as_double: reinterpret bits as double -__device__ __forceinline__ double as_double(int2 v) { - ulong bits = ((ulong)(uint)v.y << 32) | (uint)v.x; - return __longlong_as_double(bits); -} -__device__ __forceinline__ double as_double(uint2 v) { - ulong bits = ((ulong)v.y << 32) | v.x; - return __longlong_as_double(bits); -} -__device__ __forceinline__ double as_double(ulong v) { - return __longlong_as_double(v); -} +__device__ __forceinline__ double as_double(int2 v) { union { int2 i2; double d; } u; u.i2 = v; return u.d; } +__device__ __forceinline__ double as_double(uint2 v) { union { uint2 ui2; double d; } u; u.ui2 = v; return u.d; } +__device__ __forceinline__ double as_double(ulong v) { return __longlong_as_double(v); } __device__ __forceinline__ double as_double(i64 v) { return __longlong_as_double(v); } // as_ulong: reinterpret as unsigned 64-bit -__device__ __forceinline__ ulong as_ulong(uint2 v) { return (((ulong)v.y << 32) | v.x); } -__device__ __forceinline__ ulong as_ulong(int2 v) { return (((ulong)(uint)v.y << 32) | (uint)v.x); } +__device__ __forceinline__ ulong as_ulong(uint2 v) { union { uint2 ui2; ulong ul; } u; u.ui2 = v; return u.ul; } +__device__ __forceinline__ ulong as_ulong(int2 v) { union { int2 i2; ulong ul; } u; u.i2 = v; return u.ul; } __device__ __forceinline__ ulong as_ulong(double v) { return (ulong)__double_as_longlong(v); } // as_long: reinterpret as signed 64-bit -__device__ __forceinline__ i64 as_long(int2 v) { return (i64)(((ulong)(uint)v.y << 32) | (uint)v.x); } -__device__ __forceinline__ i64 as_long(uint2 v) { return (i64)(((ulong)v.y << 32) | v.x); } +__device__ __forceinline__ i64 as_long(int2 v) { union { int2 i2; i64 l; } u; u.i2 = v; return u.l; } +__device__ __forceinline__ i64 as_long(uint2 v) { union { uint2 ui2; i64 l; } u; u.ui2 = v; return u.l; } __device__ __forceinline__ i64 as_long(double v) { return (i64)__double_as_longlong(v); } // as_float / as_int / as_uint: 32-bit reinterprets From 3ec69b91465acdd98c185683188aa9a554cd11cd Mon Sep 17 00:00:00 2001 From: george Date: Tue, 14 Apr 2026 19:56:31 +0000 Subject: [PATCH 030/214] Improved "quick" routines in GF61 math. --- src/cl/carryutil.cl | 4 +-- src/cl/fft4.cl | 47 +++++++++++++---------------- src/cl/fft8.cl | 66 +++++++++++++++++------------------------ src/cl/math.cl | 70 ++++++++++++++++---------------------------- src/cl/tailmul.cl | 8 +++-- src/cl/tailsquare.cl | 43 +++++++++++++++++++-------- 6 files changed, 110 insertions(+), 128 deletions(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 5a77e511..2f9f7f65 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -414,7 +414,7 @@ i96 weightAndCarryOne(Z31 u31, Z61 u61, u32 m31_invWeight, u32 m61_invWeight, bo // Use chinese remainder theorem to create a 92-bit result. Loosely copied from Yves Gallot's mersenne2 program. u32 n31 = get_Z31(u31); - u61 = subq(u61, make_Z61(n31), 2); // u61 - u31 + u61 += make_u64(hi32(M61), lo32(M61) - n31); // u61 - u31 u61 = add(u61, shl(u61, 31)); // u61 + (u61 << 31) // The resulting value will be get_Z61(u61) * M31 + n31 and if larger than ~M31*M61/2 return a negative value by subtracting M31 * M61. @@ -455,7 +455,7 @@ i128 weightAndCarryOne(float uF2, Z31 u31, Z61 u61, float F2_invWeight, u32 m31_ // Use chinese remainder theorem to create a 92-bit result. Loosely copied from Yves Gallot's mersenne2 program. u32 n31 = get_Z31(u31); - u61 = subq(u61, make_Z61(n31), 2); // u61 - u31 + u61 += make_u64(hi32(M61), lo32(M61) - n31); // u61 - u31 u61 = add(u61, shl(u61, 31)); // u61 + (u61 << 31) u64 n61 = get_Z61(u61); // Let's call the 92-bit CRT result n3161. At this point, n3161 = n61 * M31 + n31. diff --git a/src/cl/fft4.cl b/src/cl/fft4.cl index 8b422ca9..1aafbb69 100644 --- a/src/cl/fft4.cl +++ b/src/cl/fft4.cl @@ -194,44 +194,37 @@ void OVERLOAD fft4(GF31 *u) { fft4by(u, 0, 1, 4); } #if NTT_GF61 -void OVERLOAD fft4Core(GF61 *u) { // Starts with all u[i] having maximum values of M61+epsilon. - X2q(&u[0], &u[2], 2); // X2(u[0], u[2]); No reductions mod M61. Will require 3 M61s additions to make positives. - X2q_mul_t4(&u[1], &u[3], 2); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); - X2s(&u[0], &u[1], 3); - X2s(&u[2], &u[3], 3); -} - // 16 ADD void OVERLOAD fft4by(GF61 *u, u32 base, u32 step, u32 M) { #define A(k) u[(base + step * k) % M] - Z61 x0 = addq(A(0).x, A(2).x); // Max value is 2*M61+epsilon - Z61 x2 = subq(A(0).x, A(2).x, 2); // Max value is 3*M61+epsilon - Z61 y0 = addq(A(0).y, A(2).y); - Z61 y2 = subq(A(0).y, A(2).y, 2); + Z61 x0 = A(0).x + A(2).x; // 0..2+ + Z61 x2 = A(0).x - A(2).x; // -1-..1+ + Z61 y0 = A(0).y + A(2).y; // 0..2+ + Z61 y2 = A(0).y - A(2).y; // -1-..1+ - Z61 x1 = addq(A(1).x, A(3).x); - Z61 y3 = subq(A(1).x, A(3).x, 2); - Z61 y1 = addq(A(1).y, A(3).y); - Z61 x3 = subq(A(3).y, A(1).y, 2); + Z61 x1 = A(1).x + A(3).x; // 0..2+ + Z61 y3 = A(1).x - A(3).x; // -1-..1+ + Z61 y1 = A(1).y + A(3).y; // 0..2+ + Z61 x3 = A(3).y - A(1).y; // -1-..1+ - Z61 a0 = add(x0, x1); - Z61 a1 = subs(x0, x1, 3); + Z61 a0 = x0 + x1; // 0..4+ + Z61 a1 = x0 - x1; // -2-..2+ - Z61 b0 = add(y0, y1); - Z61 b1 = subs(y0, y1, 3); + Z61 b0 = y0 + y1; // 0..4+ + Z61 b1 = y0 - y1; // -2-..2+ - Z61 a2 = add(x2, x3); - Z61 a3 = subs(x2, x3, 4); + Z61 a2 = x2 + x3; // -2-..2+ + Z61 a3 = x2 - x3; // -2-..2+ - Z61 b2 = add(y2, y3); - Z61 b3 = subs(y2, y3, 4); + Z61 b2 = y2 + y3; // -2-..2+ + Z61 b3 = y2 - y3; // -2-..2+ - A(0) = U2(a0, b0); - A(1) = U2(a2, b2); - A(2) = U2(a1, b1); - A(3) = U2(a3, b3); + A(0) = modM61q(U2(a0, b0), 0); + A(1) = modM61q(U2(a2, b2), 3); + A(2) = modM61q(U2(a1, b1), 3); + A(3) = modM61q(U2(a3, b3), 3); #undef A diff --git a/src/cl/fft8.cl b/src/cl/fft8.cl index 56e2fc94..ec4fca48 100644 --- a/src/cl/fft8.cl +++ b/src/cl/fft8.cl @@ -108,47 +108,37 @@ void OVERLOAD fft8(GF31 *u) { #if NTT_GF61 -#if 0 // Working code. - -void OVERLOAD fft8Core(GF61 *u) { - X2(u[0], u[4]); //GWBUG: Delay some mods using extra 3 bits of Z61 - X2_mul_t8(u[1], u[5]); // X2(u[1], u[5]); u[5] = mul_t8(u[5]); - X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); - X2_mul_3t8(u[3], u[7]); // X2(u[3], u[7]); u[7] = mul_3t8(u[7]); - fft4Core(u); - fft4Core(u + 4); -} - -void OVERLOAD fft8(GF61 *u) { - fft8Core(u); - // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo - SWAP(u[1], u[4]); - SWAP(u[3], u[6]); -} - -#else // Carefully track the size of numbers to reduce the number of mod M61 reductions - -void OVERLOAD fft4CoreSpecial1(GF61 *u) { // Starts with u[0,1,2,3] having maximum values of (2,2,3,2)*M61+epsilon. - X2q(&u[0], &u[2], 4); // X2(u[0], u[2]); No reductions mod M61. u[0,2] max value is 5,6*M61+epsilon. - X2q_mul_t4(&u[1], &u[3], 3); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] max value is 5,4*M61+epsilon. - u[1] = modM61(u[1]); u[2] = modM61(u[2]); // Reduce the worst offenders. u[0,1,2,3] have maximum values of (5,1,1,4)*M61+epsilon. - X2s(&u[0], &u[1], 2); // u[0,1] max value before reduction is 6,7*M61+epsilon - X2s(&u[2], &u[3], 5); // u[2,3] max value before reduction is 5,6*M61+epsilon +void OVERLOAD fft4CoreSpecial1(GF61 *u) { // Starts with u[0,1,2,3] in range of 0..2*M61+epsilon. + X2q(&u[0], &u[2]); // X2(u[0], u[2]); No reductions mod M61. u[0,2] range is 0..4+, -2-..2+ + X2q_mul_t4(&u[1], &u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] range is 0..4+, -2-..2+ + u[1] = optsubq(u[1], 2, 2); // Partially reduce. If u[1] > 2*M61, sub 2*M61. u[1] now has range 0..2+ + u[3] = optsubq(u[3], 0, 2); // Partially reduce. If u[3] > 0*M61, sub 2*M61. u[3] now has range -2-..0+ + X2q(&u[0], &u[1]); // X2(u[0], u[1]); u[0,1] range is 0..6+, -2-..4+ + X2q(&u[2], &u[3]); // X2(u[2], u[3]); u[2,3] range is -4-..2+, -2-..4+ + u[0] = modM61q(u[0], 0); + u[1] = modM61q(u[1], 3); + u[2] = modM61q(u[2], 5); + u[3] = modM61q(u[3], 3); } -void OVERLOAD fft4CoreSpecial2(GF61 *u) { // Similar to above. Starts with u[0,1,2,3] having maximum values of (3,1,2,1)*M61+epsilon. - X2q(&u[0], &u[2], 3); // u[0,2] max value is 5,6*M61+epsilon. - X2q_mul_t4(&u[1], &u[3], 2); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] max value is 3,2*M61+epsilon. - u[0] = modM61(u[0]); u[2] = modM61(u[2]); // Reduce the worst offenders u[0,1,2,3] have maximum values of (1,3,1,2)*M61+epsilon. - X2s(&u[0], &u[1], 4); // u[0,1] max value before reduction is 4,5*M61+epsilon - X2s(&u[2], &u[3], 3); // u[2,3] max value before reduction is 3,4*M61+epsilon +void OVERLOAD fft4CoreSpecial2(GF61 *u) { // Bottom half of an fft8. Starts with u[0,1,2,3] in range of -1*M61-epsilon..1*M61+epsilon + X2q(&u[0], &u[2]); // X2(u[0], u[2]); No reductions mod M61. u[0,2] range is -2-..2+, -2-..2+ + u[1] = mul_t8q(u[1], 3); // Perform delayed mul_t8. u[1] range is 0..1+ + u[3] = mul_t8q(u[3], 3); // Perform delayed mul_t8. u[3] range is 0..1+ + X2q_mul_t4(&u[1], &u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] range is 0..2+, -1-..1+ + X2q(&u[0], &u[1]); // X2(u[0], u[1]); u[0,1] range is -2-..4+, -4-..2+ + X2q(&u[2], &u[3]); // X2(u[2], u[3]); u[2,3] range is -3-..3+, -3-..3+ + u[0] = modM61q(u[0], 3); + u[1] = modM61q(u[1], 5); + u[2] = modM61q(u[2], 4); + u[3] = modM61q(u[3], 4); } -void OVERLOAD fft8Core(GF61 *u) { // Starts with all u[i] having maximum values of M61+epsilon. - X2q(&u[0], &u[4], 2); // X2(u[0], u[4]); No reductions mod M61. u[0,4] max value is 2,3*M61+epsilon. - X2q_mul_t8(&u[1], &u[5], 2); // X2(u[1], u[5]); u[5] = mul_t8(u[5]); u[1,5] max value is 2,1*M61+epsilon. - X2q_mul_t4(&u[2], &u[6], 2); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); u[2,6] max value is 3,2*M61+epsilon. - X2q_mul_3t8(&u[3], &u[7], 2); // X2(u[3], u[7]); u[7] = mul_3t8(u[7]); u[3,7] max value is 2,1*M61+epsilon. +void OVERLOAD fft8Core(GF61 *u) { // Starts with all u[i] values in range of 0..M61+epsilon (shorthand notation is 0..1+) + X2q(&u[0], &u[4]); // X2(u[0], u[4]); No reductions mod M61. u[0,4] range is 0..2+, -1-..1+ + X2q(&u[1], &u[5]); // X2(u[1], u[5]); Delay mul_t8 on u[5]. u[1,5] range is 0..2+, -1-..1+ + X2q_mul_t4(&u[2], &u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); u[2,6] range is 0..2+, -1-..1+ + X2q_mul_t4(&u[3], &u[7]); // X2(u[3], u[7]); u[7] = mul_t4(u[7]); u[3,7] range is 0..2+, -1-..1+ Delay mul_t8 on u[7]. fft4CoreSpecial1(u); fft4CoreSpecial2(u + 4); } @@ -161,5 +151,3 @@ void OVERLOAD fft8(GF61 *u) { } #endif - -#endif diff --git a/src/cl/math.cl b/src/cl/math.cl index b53fccbb..76ec9429 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -186,14 +186,13 @@ i32 optional_mod(i32 a, const i32 b) { return a; } -#if 0 // These are not used because there is no asm constraint to indicate a 64-bit constant // Optionally subtract c from a if a >= b. -u64 OVERLOAD optional_sub(u64 a, const u64 b, const u64 c) { +i64 OVERLOAD optional_sub(i64 a, const i64 b, const i64 c) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" - " setp.ge.u64 %%p, %0, %1;\n\t" // a >= b - " @%%p sub.u64 %0, %0, %2;}" // if (a >= b) a = a - c - : "+l"(a) : "l"(b), "l"(c)); + " setp.ge.s64 %%p, %0, %1;\n\t" // a >= b + " @%%p sub.s64 %0, %0, %2;}" // if (a >= b) a = a - c + : "+l"(a) : "l"(b), "l"(c)); // It would be nice if there was an asm constraint to indicate a 64-bit constant #else if (a >= b) a = a - c; #endif @@ -201,18 +200,17 @@ u64 OVERLOAD optional_sub(u64 a, const u64 b, const u64 c) { } // Optionally subtract c from a if hi32(a) >= b. -u64 OVERLOAD optional_sub(u64 a, const u32 b, const u64 c) { +i64 OVERLOAD optional_sub(i64 a, const i32 b, const i64 c) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" - " setp.ge.u32 %%p, %1, %2;\n\t" // hi32(a) >= b - " @%%p sub.u64 %0, %0, %3;}" // if (hi32(a) >= b) a = a - c - : "+l"(a) : "r"(hi32(a)), "n"(b), "l"(c)); + " setp.ge.s32 %%p, %1, %2;\n\t" // hi32(a) >= b + " @%%p sub.s64 %0, %0, %3;}" // if (hi32(a) >= b) a = a - c + : "+l"(a) : "r"((i32)hi32(a)), "n"(b), "l"(c)); #else - if (hi32(a) >= b) a = a - c; + if ((i32)hi32(a) >= b) a = a - c; #endif return a; } -#endif // Multiply and add primitives @@ -918,27 +916,14 @@ Z61 OVERLOAD modM61(Z61 a) { return a; } GF61 OVERLOAD modM61(GF61 a) { return a; } Z61 OVERLOAD neg(Z61 a, u32 m61_count) { return neg(a); } GF61 OVERLOAD neg(GF61 a, u32 m61_count) { return neg(a); } -Z61 OVERLOAD addq(Z61 a, Z61 b) { return add(a, b); } -GF61 OVERLOAD addq(GF61 a, GF61 b) { return add(a, b); } -Z61 OVERLOAD subq(Z61 a, Z61 b, u32 m61_count) { return sub(a, b); } -GF61 OVERLOAD subq(GF61 a, GF61 b, u32 m61_count) { return sub(a, b); } -Z61 OVERLOAD subs(Z61 a, Z61 b, u32 m61_count) { return sub(a, b); } -GF61 OVERLOAD subs(GF61 a, GF61 b, u32 m61_count) { return sub(a, b); } -void OVERLOAD X2q(GF61 *a, GF61 *b, u32 m61_count) { X2_internal(a, b); } -void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b, u32 m61_count) { X2_mul_t4_internal(a, b); } -void OVERLOAD X2s(GF61 *a, GF61 *b, u32 m61_count) { X2_internal(a, b); } -void OVERLOAD X2s_conjb(GF61 *a, GF61 *b, u32 m61_count) { X2_conjb_internal(a, b); } - - // Philosophy: This Z61/GF61 implementation uses faster, sloppier mod M61 reduction where the end result is in the range 0..M61+epsilon. // This implementation also handles subtractions by adding enough M61s to make a value positive. This allows us to always deal with positive -// intermediate results. The downside is that a caller using the sloppy/quick routines must keep track of how large unreduced values can get. -// An alternative implementation is to have Z61 be an i64 (costs us a precious bit of precision) but is surprisingly slower (at least on TitanV) because +// intermediate results. An alternative implementation is to have Z61 be an i64 (costs us a precious bit of precision) and is surprisingly slower (at least on TitanV) because // mod(a - b), where the mod routinue uses a signed right shift is slower than // mod(a + (M61*2 - b)) where the mod routine uses an unsigned shift right. -// However, a long string of subtracts (example, fft8 does 3 subtracts before mod M61 might be better off using negative intermediate results. +// However, a long string of subtracts (example, fft8 does 3 subtracts before mod M61 will be better off using the quick routines and negative intermediate results). // The mul routine (and obviously csq and cmul) must use only positive values as __int128 multiply is very slow. #elif 1 // Faster version that keeps results in the range 0 .. M61+epsilon @@ -1092,28 +1077,25 @@ GF61 OVERLOAD addsub(GF61 a) { return U2(add(a.x, a.y), sub(a.x, a.y)); } GF61 OVERLOAD foo2(GF61 a, GF61 b) { a = addsub(a); b = addsub(b); return addsub(U2(mul(RE(a), RE(b)), mul(IM(a), IM(b)))); } GF61 OVERLOAD foo(GF61 a) { return foo2(a, a); } -// The following routines can be used to reduce mod M61 operations. Caller must track how many M61s need to be added to make positive -// values for subtractions. In function names, "q" stands for quick (no modM61), "s" stands for slow (i.e. does modM61). - -Z61 OVERLOAD addq(Z61 a, Z61 b) { return a + b; } -GF61 OVERLOAD addq(GF61 a, GF61 b) { return U2(addq(a.x, b.x), addq(a.y, b.y)); } - -Z61 OVERLOAD subq(Z61 a, Z61 b, const u32 m61_count) { return a + neg(b, m61_count); } -GF61 OVERLOAD subq(GF61 a, GF61 b, const u32 m61_count) { return U2(subq(a.x, b.x, m61_count), subq(a.y, b.y, m61_count)); } +// The following routines can be used to reduce mod M61 operations by carefully tracking the range of intermediate results which can be negative. +// This reduces the number of m61_count*M61 addition operations too. By tracking ranges of intermediate results, the caller knows how many M61s +// need to be added to make result positive prior to the final modM61. In function names, "q" stands for quick (no modM61). -Z61 OVERLOAD subs(Z61 a, Z61 b, const u32 m61_count) { return modM61(a + neg(b, m61_count)); } -GF61 OVERLOAD subs(GF61 a, GF61 b, const u32 m61_count) { return U2(subs(a.x, b.x, m61_count), subs(a.y, b.y, m61_count)); } +GF61 OVERLOAD addq(GF61 a, GF61 b) { return a + b; } +GF61 OVERLOAD subq(GF61 a, GF61 b) { return a - b; } +GF61 OVERLOAD addiq(GF61 a, GF61 b) { return U2(a.x - b.y, a.y + b.x); } +GF61 OVERLOAD subiq(GF61 a, GF61 b) { return U2(a.x + b.y, a.y - b.x); } -GF61 OVERLOAD addiq(GF61 a, GF61 b, const u32 m61_count) { return U2(subq(a.x, b.y, m61_count), addq(a.y, b.x)); } -GF61 OVERLOAD subiq(GF61 a, GF61 b, const u32 m61_count) { return U2(addq(a.x, b.y), subq(a.y, b.x, m61_count)); } +void OVERLOAD X2q(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; *b = t - *b; } +void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; t.x = t.x - b->x; b->x = b->y - t.y; b->y = t.x; } +void OVERLOAD X2q_conjb(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } -void OVERLOAD X2q(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; *b = t + neg(*b, m61_count); } -void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; t.x = t.x + neg(b->x, m61_count); b->x = b->y + neg(t.y, m61_count); b->y = t.x; } -void OVERLOAD X2q_mul_t8(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; t = *b + neg(t, m61_count); *b = shl(U2(t.x + neg(t.y, m61_count * 2), t.x + t.y), 30); } -void OVERLOAD X2q_mul_3t8(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = t + *b; t = t + neg(*b, m61_count); *b = mul_3t8(t, m61_count * 2); } +GF61 OVERLOAD mul_t8q(GF61 a, const u32 m61_count) { return shl(U2(m61_count * M61 + (a.y - a.x), m61_count * M61 - (a.x + a.y)), 30); } -void OVERLOAD X2s(GF61 *a, GF61 *b, const u32 m61_count) { GF61 t = *a; *a = add(t, *b); *b = subs(t, *b, m61_count); } -void OVERLOAD X2s_conjb(GF61 *a, GF61 *b, const u32 a_m61_count, const u32 b_m61_count) { GF61 t = *a; *a = add(t, *b); b->x = subs(t.x, b->x, b_m61_count); b->y = subs(b->y, t.y, a_m61_count); } +Z61 OVERLOAD optsubq(Z61 a, const u32 m61_limit, const u32 m61_count) { return optional_sub((i64)a, (i32)(m61_limit << (61 - 32)), (i64)(m61_count * M61)); } +GF61 OVERLOAD optsubq(GF61 a, const u32 m61_limit, const u32 m61_count) { return U2(optsubq(a.x, m61_limit, m61_count), optsubq(a.y, m61_limit, m61_count)); } +GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count) { if (m61_count) { a.x += m61_count * M61; a.y += m61_count * M61; } return modM61(a); } +GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count_x, const u32 m61_count_y) { if (m61_count_x) a.x += m61_count_x * M61; if (m61_count_y) a.y += m61_count_y * M61; return modM61(a); } #endif diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 03647fdb..b871021b 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -375,9 +375,11 @@ void OVERLOAD onePairMul(GF61* pa, GF61* pb, GF61* pc, GF61* pd, GF61 t_squared) X2conjb(a, b); X2conjb(c, d); - GF61 e = subq(cmul(a, c), cmul(cmul(b, d), t_squared), 2); // Max value is 3*M61+epsilon - GF61 f = addq(cmul(b, c), cmul(a, d)); // Max value is 2*M61+epsilon - X2s_conjb(&e, &f, 4, 3); + GF61 e = subq(cmul(a, c), cmul(cmul(b, d), t_squared)); // Range is -1-..1+ + GF61 f = addq(cmul(b, c), cmul(a, d)); // Range is 0..2+ + X2q_conjb(&e, &f); // e range is -1..3+, f.x range is -3-..1+, f.y range is -1-..3+ + e = modM61q(e, 2); + f = modM61q(f, 4, 2); *pa = SWAP_XY(e), *pb = SWAP_XY(f); } diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 48f49aa5..c727859c 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -870,19 +870,36 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_type) { GF61 a = *pa, b = *pb; - GF61 c, d; - - X2conjb(a, b); - if (t_squared_type == 0) // mul t_squared by 1 - c = subq(csqq(a, 2), cmul(csq(b), t_squared), 2); // max c value is 4*M61+epsilon - if (t_squared_type == 1) // mul t_squared by i - c = subiq(csqq(a, 2), cmul(csq(b), t_squared), 2); // max c value is 4*M61+epsilon - if (t_squared_type == 2) // mul t_squared by -1 - c = addq(csqq(a, 2), cmul(csq(b), t_squared)); // max c value is 3*M61+epsilon - if (t_squared_type == 3) // mul t_squared by -i - c = addiq(csqq(a, 2), cmul(csq(b), t_squared), 2); // max c value is 4*M61+epsilon - d = 2 * cmul(a, b); // max d value is 2*M61+epsilon - X2s_conjb(&c, &d, 5, 3); + GF61 a2, b2t2, c, d; + + X2conjb(a, b); // X2(a, conjugate(b)) + a2 = csqq(a, 2); // a2 = a^2, a2.x range is 0..2+, a2.y range is 0..3+ + b2t2 = cmul(csq(b), t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ + d = cmul(a, b); d = d + d; // d = 2ab, d range is 0..2+ + if (t_squared_type == 0) { // mul t_squared by 1 + c = subq(a2, b2t2); // c.x range is -1..2+, c.y range is -1-..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is -1-..5+, d.x range is -3-..2+, d.y range is -3-..3+ + c = modM61q(c, 2); + d = modM61q(d, 4); + } + if (t_squared_type == 1) { // mul t_squared by i + c = subiq(a2, b2t2); // c.x range is 0..3+, c.y range is -1-..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is -1..5+, d.x range is -2-..3+, d.y range is -3-..3+ + c = modM61q(c, 0, 2); + d = modM61q(d, 4); + } + if (t_squared_type == 2) { // mul t_squared by -1 + c = addq(a2, b2t2); // c.x range is 0..3+, c.y range is 0..4+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is 0..6+, d.x range is -2-..3+, d.y range is -4-..2+ + c = modM61q(c, 0); + d = modM61q(d, 3, 5); + } + if (t_squared_type == 3) { // mul t_squared by -i + c = addiq(a2, b2t2); // c.x range is -1-..2+, c.y range is 0..4+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is 0..6+, d.x range is -3-..2+, d.y range is -4-..2+ + c = modM61q(c, 2, 0); + d = modM61q(d, 4, 5); + } *pa = SWAP_XY(c), *pb = SWAP_XY(d); } From df98f93c813f49ffeffcf9242ad9941e6c8ada1f Mon Sep 17 00:00:00 2001 From: george Date: Wed, 15 Apr 2026 04:03:40 +0000 Subject: [PATCH 031/214] Allow "PRP=1,2,n,-1" in worktodo.txt without the no-factor-to-bits value. There is probably a more elegant fix. I'm not sure mine work in Windows. --- src/Worktodo.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 1628a391..0801ffec 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -58,7 +58,7 @@ std::optional parse(const std::string& line) { parts.erase(parts.begin()); } - string s = (parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && parts[3] == "-1") ? parts[2] + string s = (parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && (parts[3] == "-1" || parts[3] == "-1\n")) ? parts[2] : (!parts.empty() ? parts[0] : ""); const char *end = s.c_str() + s.size(); From 279cc83db057c2d8b4bb39725fbaffea2485fec0 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 15 Apr 2026 19:53:32 +0000 Subject: [PATCH 032/214] Sanity check a user-supplied FFT spec. --- src/FFTConfig.cpp | 23 +++++++++++++++++++++++ src/Task.cpp | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index c1009845..2e45afd0 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -208,6 +208,29 @@ FFTConfig::FFTConfig(const string& spec) { v.resize(v.size() - 1); } + // Sanity check the spec + if (v.size() >= 3) { + u32 w = parseInt(v[0]); + u32 m = parseInt(v[1]); + u32 h = parseInt(v[2]); + if (w != 256 && w != 512 && w != 1024 && w != 4096) { + log("Width must be 256, 512, 1024, or 4096.\n"); + throw "Invalid FFT spec"; + } + if (m < 2 || m > 16) { + log("Middle must be between 1 and 16.\n"); + throw "Invalid FFT spec"; + } + if (h != 256 && h != 512 && h != 1024) { + log("Height must be 256, 512, 1024.\n"); + throw "Invalid FFT spec"; + } + if (fft_type != FFT64 && fft_type != FFT32 && (m & (m - 1))) { + log("NTT middle must be a power of two.\n"); + throw "Invalid FFT spec"; + } + } + if (v.size() == 1) { *this = {FFTShape::multiSpec(spec).front(), LAST_VARIANT, CARRY_AUTO}; } if (v.size() == 3) { diff --git a/src/Task.cpp b/src/Task.cpp index 9a49654f..95c23e98 100644 --- a/src/Task.cpp +++ b/src/Task.cpp @@ -215,7 +215,7 @@ void Task::execute(GpuCommon shared, Queue *q, u32 instance) { assert(exponent); // Testing exponent 140000001 using FFT 512:15:512 fails with severe round off errors. - // I'm guessing this is because bot the exponent and FFT size are divisible by 3. + // I'm guessing this is because both the exponent and FFT size are divisible by 3. // Here we make sure the exponent is prime. If not we do not raise an error because it // is very common to use command line argument "-prp some-random-exponent" to get a quick // timing. Instead, we output a warning and test a smaller prime exponent. From 14021aba733f20fda5da6fd8d37b78c71339d398 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 16 Apr 2026 02:09:53 +0000 Subject: [PATCH 033/214] Minor tweaks to calculating weights --- src/Gpu.cpp | 29 +++++++++++++++++++---------- src/Gpu.h | 1 - 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 6012cfc7..1c743c70 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -37,6 +37,14 @@ #define M_PI 3.141592653589793238462643383279502884 #endif +#ifndef M_LN2l +#define M_LN2l 0.69314718055994530941723212145818L +#endif + +#ifndef M_LN2 +#define M_LN2 0.69314718055994530941723212145818 +#endif + #define CARRY_LEN 8 namespace { @@ -51,12 +59,13 @@ double invWeight(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return exp2l(-(long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } +// MSVC does not truly support long double. Use expm1 rather than exp2 and subtracting one. double weightM1(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2l((long double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; + return expm1l(M_LN2l * (long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } double invWeightM1(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2l(- (long double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; + return expm1l(M_LN2l * - (long double)(extra(N, E, kAt(H, line, col) + rep)) / N); } double boundUnderOne(double x) { return std::min(x, nexttoward(1, 0)); } @@ -70,16 +79,16 @@ float invWeight32(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { } float weightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; + return expm1(M_LN2 * (double)(extra(N, E, kAt(H, line, col) + rep)) / N); } float invWeightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2(- (double)(extra(N, E, kAt(H, line, col) + rep)) / N) - 1; + return expm1(M_LN2 * - (double)(extra(N, E, kAt(H, line, col) + rep)) / N); } float boundUnderOne(float x) { return std::min(x, nexttowardf(1, 0)); } -Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { +Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { u32 N = 2u * W * H; u32 groupWidth = W / nW; @@ -94,7 +103,7 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { // nVidia GPUs have a constant cache that only works on buffer sizes less than 64KB. Create a smaller buffer // that is a copy of the first part of weightsIF. There are several kernels that need the combined weightsIF // buffer, so there is an unfortunate duplication of these weights. - if (!AmdGpu) { + if (nvidiaGpu) { weightsConstIF.push_back(2 * boundUnderOne(iw)); weightsConstIF.push_back(2 * w); } @@ -114,8 +123,8 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { vector weightsIF32; // Inverse + Forward for (u32 thread = 0; thread < groupWidth; ++thread) { - auto iw = invWeight32(N, E, H, 0, thread, 0) ;// * 17592186044416.0f; - auto w = weight32(N, E, H, 0, thread, 0) ;// * 0.0000002384185791015625f; + auto iw = invWeight32(N, E, H, 0, thread, 0) ; + auto w = weight32(N, E, H, 0, thread, 0) ; // Play with the weight so that optionalDouble and optionalHalve work iw = 2.0f * boundUnderOne(iw); w = 2.0f * w; @@ -125,7 +134,7 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool AmdGpu) { // nVidia GPUs have a constant cache that only works on buffer sizes less than 64KB. Create a smaller buffer // that is a copy of the first part of weightsIF. There are several kernels that need the combined weightsIF // buffer, so there is an unfortunate duplication of these weights. - if (!AmdGpu) { + if (nvidiaGpu) { weightsConstIF32.push_back(iw); weightsConstIF32.push_back(w); } @@ -761,7 +770,7 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& bufTrigM{shared.bufCache->middleTrig(shared.args, fft, SMALL_H, BIG_H / SMALL_H, WIDTH)}, bufTrigW{shared.bufCache->smallTrig(shared.args, fft, WIDTH, nW, fft.shape.middle, SMALL_H, nH, tail_single_wide)}, - weights{genWeights(fft, E, WIDTH, BIG_H, nW, isAmdGpu(q->context->deviceId()))}, + weights{genWeights(fft, E, WIDTH, BIG_H, nW, isNvidiaGpu(q->context->deviceId()))}, bufConstWeights{q->context, std::move(weights.weightsConstIF)}, bufWeights{q->context, std::move(weights.weightsIF)}, diff --git a/src/Gpu.h b/src/Gpu.h index 09b6aa16..2c0554a0 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -80,7 +80,6 @@ class RoeInfo { struct Weights { vector weightsConstIF; vector weightsIF; - vector bitsCF; }; class Gpu { From a9c34396f7f3e9ed852b51c0b75e21b2c407b823 Mon Sep 17 00:00:00 2001 From: george Date: Sat, 18 Apr 2026 00:50:02 +0000 Subject: [PATCH 034/214] Worked on CONST_THREAD_WEIGHTS. __constant__ can be implemented in a CUDA build but it is a bit klunky and the benefit is iffy. I did change the way OpenCL's CONST_THREAD_WEIGHTS are used which is more in line with nVidia documentation of how constant should be used. --- src/Gpu.cpp | 44 ++++++++++++++++++++++++++++---------------- src/cl/carryfused.cl | 18 ++++++++++++------ 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 1c743c70..197b13a8 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -100,13 +100,6 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { for (u32 thread = 0; thread < groupWidth; ++thread) { auto iw = invWeight(N, E, H, 0, thread, 0); auto w = weight(N, E, H, 0, thread, 0); - // nVidia GPUs have a constant cache that only works on buffer sizes less than 64KB. Create a smaller buffer - // that is a copy of the first part of weightsIF. There are several kernels that need the combined weightsIF - // buffer, so there is an unfortunate duplication of these weights. - if (nvidiaGpu) { - weightsConstIF.push_back(2 * boundUnderOne(iw)); - weightsConstIF.push_back(2 * w); - } weightsIF.push_back(2 * boundUnderOne(iw)); weightsIF.push_back(2 * w); } @@ -116,6 +109,19 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { weightsIF.push_back(invWeightM1(N, E, H, gy, 0, 0)); weightsIF.push_back(weightM1(N, E, H, gy, 0, 0)); } + + // nVidia GPUs have a fast constant cache that only works on buffer sizes less than 64KB. Create two smaller buffers + // that can be used to create the large group order buffer with a single multiply. + if (nvidiaGpu) { + for (u32 gy = 0; gy < 64; ++gy) { + weightsConstIF.push_back(invWeightM1(N, E, H, gy, 0, 0)); + weightsConstIF.push_back(weightM1(N, E, H, gy, 0, 0)); + } + for (u32 gy = 0; gy < H; gy += 64) { + weightsConstIF.push_back(invWeightM1(N, E, H, gy, 0, 0)); + weightsConstIF.push_back(weightM1(N, E, H, gy, 0, 0)); + } + } } else if (fft.FFT_FP32) { @@ -131,13 +137,6 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { // Weights are scaled by 2^-24 and 2^48 so that multiplicaton by 1/epsilon does not generate infinty results (width and height variant 2). iw = iw * 281474976710656.0f; w = w * 0.000000059604644775390625f; - // nVidia GPUs have a constant cache that only works on buffer sizes less than 64KB. Create a smaller buffer - // that is a copy of the first part of weightsIF. There are several kernels that need the combined weightsIF - // buffer, so there is an unfortunate duplication of these weights. - if (nvidiaGpu) { - weightsConstIF32.push_back(iw); - weightsConstIF32.push_back(w); - } weightsIF32.push_back(iw); weightsIF32.push_back(w); } @@ -148,11 +147,24 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { weightsIF32.push_back(weightM132(N, E, H, gy, 0, 0)); } + // nVidia GPUs have a fast constant cache that only works on buffer sizes less than 64KB. Create two smaller buffers + // that can be used to create the large group order buffer with a single multiply. + if (nvidiaGpu) { + for (u32 gy = 0; gy < 64; ++gy) { + weightsConstIF32.push_back(invWeightM132(N, E, H, gy, 0, 0)); + weightsConstIF32.push_back(weightM132(N, E, H, gy, 0, 0)); + } + for (u32 gy = 0; gy < H; gy += 64) { + weightsConstIF32.push_back(invWeightM132(N, E, H, gy, 0, 0)); + weightsConstIF32.push_back(weightM132(N, E, H, gy, 0, 0)); + } + } + // Copy the float vectors to the double vectors - weightsConstIF.resize(weightsConstIF32.size() / 2); - memcpy((double *) weightsConstIF.data(), weightsConstIF32.data(), weightsConstIF32.size() * sizeof(float)); weightsIF.resize(weightsIF32.size() / 2); memcpy((double *) weightsIF.data(), weightsIF32.data(), weightsIF32.size() * sizeof(float)); + weightsConstIF.resize(weightsConstIF32.size() / 2); + memcpy((double *) weightsConstIF.data(), weightsConstIF32.data(), weightsConstIF32.size() * sizeof(float)); } return Weights{weightsConstIF, weightsIF}; diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index dff9bb08..90892cb8 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -141,10 +141,13 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; -#if AMDGPU - T2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - T2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif #if MUL3 @@ -332,10 +335,13 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif P(CFcarry) carryShuttlePtr = (P(CFcarry)) carryShuttle; From 2da665d3d1f6ee1f3b312046567993b1370527a2 Mon Sep 17 00:00:00 2001 From: george Date: Sat, 18 Apr 2026 17:11:59 +0000 Subject: [PATCH 035/214] Fixed rare bug in GF61 fft8 code committed last week. --- src/cl/fft8.cl | 4 ++-- src/cl/math.cl | 28 ++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/cl/fft8.cl b/src/cl/fft8.cl index ec4fca48..99ffa23c 100644 --- a/src/cl/fft8.cl +++ b/src/cl/fft8.cl @@ -111,8 +111,8 @@ void OVERLOAD fft8(GF31 *u) { void OVERLOAD fft4CoreSpecial1(GF61 *u) { // Starts with u[0,1,2,3] in range of 0..2*M61+epsilon. X2q(&u[0], &u[2]); // X2(u[0], u[2]); No reductions mod M61. u[0,2] range is 0..4+, -2-..2+ X2q_mul_t4(&u[1], &u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); u[1,3] range is 0..4+, -2-..2+ - u[1] = optsubq(u[1], 2, 2); // Partially reduce. If u[1] > 2*M61, sub 2*M61. u[1] now has range 0..2+ - u[3] = optsubq(u[3], 0, 2); // Partially reduce. If u[3] > 0*M61, sub 2*M61. u[3] now has range -2-..0+ + u[1] = optsubqu(u[1], 2, 2); // Partially reduce. If u[1] > 2*M61, sub 2*M61. u[1] now has range 0..2+ + u[3] = optsubqs(u[3], 0, 2); // Partially reduce. If u[3] > 0*M61, sub 2*M61. u[3] now has range -2-..0+ X2q(&u[0], &u[1]); // X2(u[0], u[1]); u[0,1] range is 0..6+, -2-..4+ X2q(&u[2], &u[3]); // X2(u[2], u[3]); u[2,3] range is -4-..2+, -2-..4+ u[0] = modM61q(u[0], 0); diff --git a/src/cl/math.cl b/src/cl/math.cl index 76ec9429..e1530762 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -187,6 +187,17 @@ i32 optional_mod(i32 a, const i32 b) { } // Optionally subtract c from a if a >= b. +u64 OVERLOAD optional_sub(u64 a, const u64 b, const u64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.u64 %%p, %0, %1;\n\t" // a >= b + " @%%p sub.u64 %0, %0, %2;}" // if (a >= b) a = a - c + : "+l"(a) : "l"(b), "l"(c)); // It would be nice if there was an asm constraint to indicate a 64-bit constant +#else + if (a >= b) a = a - c; +#endif + return a; +} i64 OVERLOAD optional_sub(i64 a, const i64 b, const i64 c) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" @@ -200,6 +211,17 @@ i64 OVERLOAD optional_sub(i64 a, const i64 b, const i64 c) { } // Optionally subtract c from a if hi32(a) >= b. +u64 OVERLOAD optional_sub(u64 a, const u32 b, const u64 c) { +#if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.ge.u32 %%p, %1, %2;\n\t" // hi32(a) >= b + " @%%p sub.u64 %0, %0, %3;}" // if (hi32(a) >= b) a = a - c + : "+l"(a) : "r"((u32)hi32(a)), "n"(b), "l"(c)); +#else + if ((u32)hi32(a) >= b) a = a - c; +#endif + return a; +} i64 OVERLOAD optional_sub(i64 a, const i32 b, const i64 c) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" @@ -1092,8 +1114,10 @@ void OVERLOAD X2q_conjb(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; b->x = t.x GF61 OVERLOAD mul_t8q(GF61 a, const u32 m61_count) { return shl(U2(m61_count * M61 + (a.y - a.x), m61_count * M61 - (a.x + a.y)), 30); } -Z61 OVERLOAD optsubq(Z61 a, const u32 m61_limit, const u32 m61_count) { return optional_sub((i64)a, (i32)(m61_limit << (61 - 32)), (i64)(m61_count * M61)); } -GF61 OVERLOAD optsubq(GF61 a, const u32 m61_limit, const u32 m61_count) { return U2(optsubq(a.x, m61_limit, m61_count), optsubq(a.y, m61_limit, m61_count)); } +Z61 OVERLOAD optsubqu(Z61 a, const u32 m61_limit, const u32 m61_count) { return optional_sub((u64)a, (u32)(m61_limit << (61 - 32)), (u64)(m61_count * M61)); } +GF61 OVERLOAD optsubqu(GF61 a, const u32 m61_limit, const u32 m61_count) { return U2(optsubqu(a.x, m61_limit, m61_count), optsubqu(a.y, m61_limit, m61_count)); } +Z61 OVERLOAD optsubqs(Z61 a, const u32 m61_limit, const u32 m61_count) { return optional_sub((i64)a, (i32)(m61_limit << (61 - 32)), (i64)(m61_count * M61)); } +GF61 OVERLOAD optsubqs(GF61 a, const u32 m61_limit, const u32 m61_count) { return U2(optsubqs(a.x, m61_limit, m61_count), optsubqs(a.y, m61_limit, m61_count)); } GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count) { if (m61_count) { a.x += m61_count * M61; a.y += m61_count * M61; } return modM61(a); } GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count_x, const u32 m61_count_y) { if (m61_count_x) a.x += m61_count_x * M61; if (m61_count_y) a.y += m61_count_y * M61; return modM61(a); } From 2bbb6918fd989206f4fca23353f857e7c5685f0e Mon Sep 17 00:00:00 2001 From: george Date: Sat, 18 Apr 2026 18:09:38 +0000 Subject: [PATCH 036/214] Micro-optimization. Eliminated a modulo when MIDDLE is not a power of 2. --- src/cl/carryfused.cl | 45 ++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 90892cb8..0b1b5f3b 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -120,11 +120,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); @@ -314,11 +315,12 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); @@ -502,11 +504,12 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); @@ -713,11 +716,12 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; #if HAS_ASM __asm("s_setprio 3"); @@ -933,11 +937,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -1174,11 +1179,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1418,11 +1424,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -1661,11 +1668,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -1915,11 +1923,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 H = BIG_HEIGHT; #if WMUL == 1 u32 lowMe = me; - u32 line = gr % H; + u32 line = gr; #else u32 lowMe = me % G_W; // lane-id in one of the WMUL sub-workgroups. - u32 line = (gr * WMUL + me / G_W) % H; + u32 line = gr * WMUL + me / G_W; #endif + if (line >= H) line -= H; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; From e5f5e18cb000ac1b6ac9c32349f023eceacd34d5 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 19 Apr 2026 00:29:11 +0000 Subject: [PATCH 037/214] Fixed the remaining CONST_THREAD_WEIGHTS that were overlooked in last CONST_THREAD_WEIGHTS change --- src/cl/carryfused.cl | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 0b1b5f3b..b28d5de2 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -966,10 +966,13 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; -#if AMDGPU - T2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - T2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + T2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif P(i64) carryShuttlePtr = (P(i64)) carryShuttle; @@ -1211,10 +1214,13 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif P(i32) carryShuttlePtr = (P(i32)) carryShuttle; @@ -1456,10 +1462,13 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif P(i64) carryShuttlePtr = (P(i64)) carryShuttle; @@ -1961,10 +1970,13 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); Word2 wu[NW]; -#if AMDGPU - F2 weights = fancyMul(THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); +#if !NVIDIAGPU || CUDA_BACKEND + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); #else - F2 weights = fancyMul(CONST_THREAD_WEIGHTS[lowMe], THREAD_WEIGHTS[G_W + line]); // On nVidia, don't pollute the constant cache with line weights + F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); + weights.x = optionalDouble(weights.x); + weights.y = optionalHalve(weights.y); + weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); #endif P(i64) carryShuttlePtr = (P(i64)) carryShuttle; From 83b36feda37d674b4185ee2ade4f8e90139a14cf Mon Sep 17 00:00:00 2001 From: george Date: Sun, 19 Apr 2026 05:11:18 +0000 Subject: [PATCH 038/214] Slightly simpler make_Z31(i64) --- src/cl/math.cl | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index e1530762..09b45c5b 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -675,21 +675,6 @@ Z31 OVERLOAD modM31(i64 a) { // abs(a) u32 ahi = a >> 62; // Sign extend the top bits return modM31(ahi + amid + alo); // This is where caller must assure a 32-bit overflow does not occur } -Z31 OVERLOAD modM31q(u64 a) { // Quick version, a < 2^62 - u32 alo = a & M31; - u32 ahi = a >> 31; - return modM31(ahi + alo); -} -#if 0 // GWBUG - which is faster? -Z31 OVERLOAD modM31q(i64 a) { // Quick version, abs(a) must be 61 bits - u32 alo = a & M31; - i32 ahi = a >> 31; // Sign extend the top bits - if (ahi < 0) ahi = ahi + M31; - return modM31((u32) ahi + alo); -} -#else -Z31 OVERLOAD modM31q(i64 a) { return modM31(a); } // Quick version, abs(a) must be 61 bits -#endif Z31 OVERLOAD neg(Z31 a) { return M31 - a; } // GWBUG: Examine all callers to see if neg call can be avoided GF31 OVERLOAD neg(GF31 a) { return U2(neg(a.x), neg(a.y)); } @@ -702,7 +687,12 @@ GF31 OVERLOAD sub(GF31 a, GF31 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } Z31 OVERLOAD make_Z31(i32 a) { return (Z31) (a < 0 ? a + M31 : a); } // Handles signed values of a Z31 OVERLOAD make_Z31(u32 a) { return (Z31) (a); } // a must be in range of 0 .. M31-1 -Z31 OVERLOAD make_Z31(i64 a) { return modM31q(a); } // Handles range -2^61..2^61 +Z31 OVERLOAD make_Z31(i64 a) { // Handles range -2^61..2^61 + u32 alo = (u32)a & M31; + i32 ahi = (i32)((u64)a >> 31); // Unsigned shift might be faster than signed shift + ahi = optional_add(ahi, M31); // Make ahi positive + return modM31((u32)ahi + alo); +} u32 get_Z31(Z31 a) { return a == M31 ? 0 : a; } // Get value in range 0 to M31-1 i32 get_balanced_Z31(Z31 a) { return (a & 0xC0000000) ? (i32) a - M31 : (i32) a; } // Get balanced value in range -M31/2 to M31/2 From a6530f51e272f9047230ef5faa519587d08d93c2 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 19 Apr 2026 10:52:25 +0000 Subject: [PATCH 039/214] Explicitly calll Z61's modM61 rather than assuming add() will. The add routine does not expect inputs out of the range 0..M61+epsilon. --- src/cl/carryutil.cl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 2f9f7f65..dfaf3ee3 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -415,12 +415,12 @@ i96 weightAndCarryOne(Z31 u31, Z61 u61, u32 m31_invWeight, u32 m61_invWeight, bo // Use chinese remainder theorem to create a 92-bit result. Loosely copied from Yves Gallot's mersenne2 program. u32 n31 = get_Z31(u31); u61 += make_u64(hi32(M61), lo32(M61) - n31); // u61 - u31 - u61 = add(u61, shl(u61, 31)); // u61 + (u61 << 31) + u61 += shl(u61, 31); // u61 + (u61 << 31) // The resulting value will be get_Z61(u61) * M31 + n31 and if larger than ~M31*M61/2 return a negative value by subtracting M31 * M61. // We can save a little work by determining if the result will be large using just u61 and returning (get_Z61(u61) - M61) * M31 + n31. // This simplifies to get_balanced_Z61(u61) * M31 + n31. - i64 n61 = get_balanced_Z61(u61); + i64 n61 = get_balanced_Z61(modM61(u61)); // Optionally calculate roundoff error as proximity to M61/2. 28 bits of accuracy should be sufficient. u32 roundoff = (u32) abs((i32)hi32(n61)); @@ -452,15 +452,14 @@ i128 weightAndCarryOne(float uF2, Z31 u31, Z61 u61, float F2_invWeight, u32 m31_ // Apply inverse weights u31 = shr(u31, m31_invWeight); u61 = shr(u61, m61_invWeight); - // Use chinese remainder theorem to create a 92-bit result. Loosely copied from Yves Gallot's mersenne2 program. u32 n31 = get_Z31(u31); u61 += make_u64(hi32(M61), lo32(M61) - n31); // u61 - u31 - u61 = add(u61, shl(u61, 31)); // u61 + (u61 << 31) - u64 n61 = get_Z61(u61); + u61 += shl(u61, 31); // u61 + (u61 << 31) + u64 n61 = get_Z61(modM61(u61)); // Let's call the 92-bit CRT result n3161. At this point, n3161 = n61 * M31 + n31. - // The final result mod M31*M61 must be n3161. Use FP32 data to calculate how many multiples of M31*M61 need to be added to n31n61. + // The final result mod M31*M61 must be n3161. Use FP32 data to calculate how many multiples of M31*M61 need to be added to n3161. float n3161f = (float)hi32(n61) * -9223372036854775808.0f; // Estimate -n3161 as a float. -n61 << 31 should be close enough. uF2 = fma(uF2, F2_invWeight, n3161f); // This should be close to an integer multiple of M31*M61 float uF2int = fma(uF2, 2.0194839183061857038255724444152e-28f, RNDVAL); // Divide by M31*M61 and round to int From 036061ea1a54e1c54e740e6de4d2be5f419c7f89 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Apr 2026 01:58:02 +0000 Subject: [PATCH 040/214] Added shufl code to eliminate LDS bank conflicts using either padding or swizzling. Padding seems better than swizzling in all tests. Both add to register pressure, swizzling more than padding. Benefit ranges from 0% to ~2%. Oddly Radeon VII benefits, but only in fftheight. The padding code has apparently makes the rocm optimizer unhappy with carryfused. Also started reorganization to allow moving more code from fftheight and fftwidth to fftbase. --- src/Gpu.cpp | 12 +- src/cl/base.cl | 9 + src/cl/carry.cl | 3 + src/cl/carryb.cl | 2 + src/cl/carryfused.cl | 8 +- src/cl/carryutil.cl | 3 - src/cl/fft-middle.cl | 1 + src/cl/fft7.cl | 2 - src/cl/fftbase.cl | 439 ++++++++++++++++++++++++++++++++++++++--- src/cl/fftheight.cl | 16 +- src/cl/ffthin.cl | 4 - src/cl/fftmiddlein.cl | 1 - src/cl/fftmiddleout.cl | 1 - src/cl/fftp.cl | 21 +- src/cl/fftw.cl | 9 +- src/cl/fftwidth.cl | 14 +- src/cl/math.cl | 2 - src/cl/tailmul.cl | 7 +- src/cl/tailsquare.cl | 8 +- src/cl/tailutil.cl | 20 +- src/cl/trig.cl | 2 - 21 files changed, 489 insertions(+), 95 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 197b13a8..446b9638 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -550,15 +550,15 @@ string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { case CARRYFUSED: // Register usage depends on NW, the FFT/NTT type, and perhaps the long carry setting switch (fft.shape.fft_type) { case FFT64: - regs = nW == 8 ? 72 : 64; + regs = nW == 8 ? 80 : 64; use_override = "REGCF64"; break; case FFT3161: - regs = nW == 8 ? 96 : 56; + regs = nW == 8 ? 96 : 64; use_override = "REGCF3161"; break; case FFT3261: - regs = nW == 8 ? 96 : 56; + regs = nW == 8 ? 96 : 64; use_override = "REGCF3261"; break; case FFT61: @@ -566,7 +566,7 @@ string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { use_override = "REGCF61"; break; case FFT323161: - regs = nW == 8 ? 128 : 72; + regs = nW == 8 ? 128 : 80; use_override = "REGCF323161"; break; case FFT3231: @@ -622,7 +622,7 @@ string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { break; case TAIL: // Register usage depends on NH and the FP32/FP64 (assumes double-wide kernel) if (fft.FFT_FP64) { - regs = nH == 8 ? 80 : 64; + regs = nH == 8 ? 88 : 64; use_override = "REGTS64"; } else { regs = nH == 8 ? 64 : 48; @@ -630,7 +630,7 @@ string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { } break; case TAIL31: // Register usage depends on NH (assumes double-wide kernel) - regs = nH == 8 ? 56 : 48; + regs = nH == 8 ? 64 : 48; use_override = "REGTS31"; break; case TAIL61: // Register usage depends on NH (assumes double-wide kernel) diff --git a/src/cl/base.cl b/src/cl/base.cl index 2fc32c06..0b1feb02 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -145,6 +145,15 @@ G_H "group height" == SMALL_HEIGHT / NH #define SHUFL_BYTES_H 8 #endif +// Shufl can pad (or swizzle) to avoid LDS bank conflicts. See fftbase.cl. You would think this option would be good (or bad) +// for both fft_width and fft_height, but the rocm optimizer is super-finicky. Default to using LDS padding. +#if !defined(LDSPAD_W) +#define LDSPAD_W 1 +#endif +#if !defined(LDSPAD_H) +#define LDSPAD_H 1 +#endif + #if !defined(TABMUL_CHAIN) #define TABMUL_CHAIN 0 #endif diff --git a/src/cl/carry.cl b/src/cl/carry.cl index 28863dd8..88a0b2b2 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -1,5 +1,8 @@ // Copyright (C) Mihai Preda +#include "base.cl" +#include "math.cl" +#include "trig.cl" #include "carryutil.cl" #include "weight.cl" diff --git a/src/cl/carryb.cl b/src/cl/carryb.cl index d6033e19..c08004cd 100644 --- a/src/cl/carryb.cl +++ b/src/cl/carryb.cl @@ -1,5 +1,7 @@ // Copyright (C) Mihai Preda +#include "base.cl" +#include "math.cl" #include "carryutil.cl" KERNEL(G_W) carryB(P(Word2) io, CP(CarryABM) carryIn) { diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index b28d5de2..1d1feaca 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -1,8 +1,9 @@ // Copyright (C) Mihai Preda +#include "base.cl" +#include "fftwidth.cl" #include "carryutil.cl" #include "weight.cl" -#include "fftwidth.cl" #include "middle.cl" void spin() { @@ -16,10 +17,7 @@ void spin() { #endif } -// LDS bytes used by shufl for each line processed in fft_WIDTH -#define LDS_BYTES (WIDTH * SHUFL_BYTES_W) - -// Increasing WMUL to 2 will reduce carryShuttle activity. This led to a 1% speedup on Titan V. Testing on other GPUs is needed. +// Increasing WMUL to 2 reduces carryShuttle activity. This led to a 1% speedup on Titan V. Testing on other GPUs is needed. #ifndef WMUL #define WMUL 2 #endif diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index dfaf3ee3..4ae4b576 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -1,8 +1,5 @@ // Copyright (C) Mihai Preda -#include "base.cl" -#include "math.cl" - #if CARRY64 typedef i64 CFcarry; #else diff --git a/src/cl/fft-middle.cl b/src/cl/fft-middle.cl index 2f75478c..c0d96ad5 100644 --- a/src/cl/fft-middle.cl +++ b/src/cl/fft-middle.cl @@ -1,5 +1,6 @@ // Copyright (C) Mihai Preda +#include "math.cl" #include "trig.cl" #if MIDDLE == 3 diff --git a/src/cl/fft7.cl b/src/cl/fft7.cl index 450cfa7d..96e2827e 100644 --- a/src/cl/fft7.cl +++ b/src/cl/fft7.cl @@ -2,8 +2,6 @@ #pragma once -#include "base.cl" - #if FFT_FP64 #define A(i) u[(base + i * step) % M] diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 7f581728..dd0d5449 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -2,44 +2,310 @@ #include "fft4.cl" #include "fft8.cl" -#include "trig.cl" -// #include "math.cl" +// Calculate the LDS bytes used by shufl +#if LDSPAD && SHUFL_BYTES == 16 && RADIX == 8 +#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 72 / 64) +#elif LDSPAD && SHUFL_BYTES == 16 && RADIX == 4 +#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 20 / 16) +#elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 8 +#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 72 / 64) +#elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 4 +#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 20 / 16) +#else +#define LDS_BYTES (WGSZ * RADIX * SHUFL_BYTES) +#endif #if FFT_FP64 | NTT_GF61 -// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values. Each WG uses WG * sb bytes of LDS memory. +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. // Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- // even when operating on differernt sized data elements as can happen in an M31+M61 NTT. // WG = workgroup size of a single fft_WIDTH or fft_HEIGHT // n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT -// sb = The number of bytes to write to LDS memory at a time. SHUFL_BYTES_W or SHUFL_BYTES_H // numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously // lowMe = me % WG // NOTE: shufl routines perform a bar(WG) at the start but not at the end. After calling shufl, a bar(WG) is required // before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and // only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively -// minor optimization would be to spedial case the first shufl call. -void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { +// minor optimization would be to special case the first shufl call. +void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u32 lowMe) { u32 mask = f - 1; assert((mask & (mask + 1)) == 0); - if (sb == 16) { + int force_default = 0; +#if NOWG2 // For timing tests only. Option to not turn off LDS bank conflict code when numWG > 1. I've not found a GPU where this is beneficial. + if (numWG > 1) force_default = 1; +#endif +#if NOLDS2 // For timing tests only. Option to not turn off LDS bank for second shufl calls. I've not found a GPU where this is beneficial. + if (f != 1) force_default = 1; +#endif + + // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. + if (SHUFL_BYTES == 16) { local T2* lds = ((local T2*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(T2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); + +#if LDSPAD + // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Pad after every 8th value to eliminate bank conflicts. + if (!force_default && f == 1 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[lowMe * 9 + i] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } + return; + } + + // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // No padding of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and + // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. + // We can however save a bar() by writing to same locations that previous shufl wrote to. + if (!force_default && f == 8 && n == 8) { + for (u32 i = 0; i < n; ++i) { lds[(i * WG + lowMe) * 9 / 8] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[((lowMe & ~7) + i) * 9 + (lowMe & 7)]; } + return; + } + + // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Pad after every 8th value to eliminate bank conflicts. + if (!force_default && f == 1 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 9 / 8] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } + return; + } + + // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Pad 4 values after every 16th value to eliminate bank conflicts. + if (!force_default && f == 4 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). + if (!force_default && f == 1 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } + return; + } + // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and + // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. + // We can however save a bar() by writing to same locations that previous shufl wrote to. + if (!force_default && f == 8 && n == 8) { + for (u32 i = 0; i < n; ++i) { lds[i * WG + lowMe] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } + return; + } + + // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). + if (!force_default && f == 1 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } + return; + } + + // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). + if (!force_default && f == 4 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } + return; + } +#endif + + // Otherwise, execute the original shufl code bar(WG); for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i]; } bar(WG); for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + lowMe]; } } - else if (sb == 8) { - // Accessing lds memory as doubles is faster than T2 accesses on Radeon VII (halving LDS memory requirements) + // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. + else if (SHUFL_BYTES == 8) { local T* lds = ((local T*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(T); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T); + +#if LDSPAD + // Special case first n == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Pad after every 16th value to eliminate bank conflicts. + if (!force_default && f == 1 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } + return; + } + + // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (!force_default && f == 8 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * 72 + lowMe]; } + else for (u32 i = 0; i < n; ++i) { u32 idx = (i * WG + lowMe); u[i].x = lds[idx + idx / 64 * 8]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * 72 + lowMe]; } + else for (u32 i = 0; i < n; ++i) { u32 idx = (i * WG + lowMe); u[i].y = lds[idx + idx / 64 * 8]; } + return; + } + + // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Pad after every 16th value to eliminate bank conflicts. + if (!force_default && f == 1 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } + return; + } + + // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Pad 4 values after every 16th value to eliminate bank conflicts. + if (!force_default && f == 4 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i].x = lds[idx + idx / 16 * 4]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i].y = lds[idx + idx / 16 * 4]; } + return; + } +#endif +#if LDSSWIZ + // Special case first n == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). + if (!force_default && f == 1 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + return; + } + + // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). + if (!force_default && f == 8 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + return; + } + + // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (!force_default && f == 1 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + return; + } + + // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (!force_default && f == 4 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + return; + } +#endif + + // Execute the original shufl code bar(WG); for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].x; } bar(WG); @@ -50,12 +316,14 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, co for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + lowMe]; } } - else if (sb == 4) { + // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. + // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + else if (SHUFL_BYTES == 4) { // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. local int* lds = (local int*) lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(int); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); bar(WG); for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).x; } @@ -82,27 +350,146 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, co #if FFT_FP32 | NTT_GF31 // Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats. -void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { +void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u32 lowMe) { u32 mask = f - 1; assert((mask & (mask + 1)) == 0); - //GW - would a 16 byte implementation be useful? + //GW - would a 16 byte implementation be useful? Less LDS conflict work? + + int force_default = 0; +#if NOWG2 + if (numWG > 1) force_default = 1; +#endif +#if NOLDS2 + if (f != 1) force_default = 1; +#endif - if (sb >= 8) { + // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. + if (SHUFL_BYTES >= 8) { local F2* lds = ((local F2*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(F2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); + +#if LDSPAD + // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Pad after every 16th value to eliminate bank conflicts. + if (!force_default && f == 1 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } + return; + } + + // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (!force_default && f == 8 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i] = lds[i * 72 + lowMe]; } + else for (u32 i = 0; i < n; ++i) { u32 idx = (i * WG + lowMe); u[i] = lds[idx + idx / 64 * 8]; } + return; + } + + // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Pad after every 16th value to eliminate bank conflicts. + if (!force_default && f == 1 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } + return; + } + + // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 + // Pad 4 values after every 16th value to eliminate bank conflicts. + if (!force_default && f == 4 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). + if (!force_default && f == 1 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + return; + } + + // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). + if (!force_default && f == 8 && n == 8) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + return; + } + + // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (!force_default && f == 1 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + return; + } + + // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (!force_default && f == 4 && n == 4) { + bar(WG); + for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } + bar(WG); + for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + return; + } +#endif + // Execute the original shufl code bar(WG); for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i]; } bar(WG); for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + lowMe]; } } - else if (sb == 4) { + // If SHUFL_BYTES is 4 we split the F2 values into 2 int values. These are written to LDS memory using two instructions. + // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + else if (SHUFL_BYTES == 4) { // Accessing lds memory as ints might be faster than F2 accesses (halving LDS memory requirements) local F* lds = ((local F*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * n * WG * sb / sizeof(F); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F); bar(WG); for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].x; } @@ -220,7 +607,7 @@ T2 bcast(T2 src, u32 span) { #endif void OVERLOAD shufl(u32 WG, local T2 *lds, T2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl64(WG, lds, u, n, f, numWG, sb, lowMe); + shufl64(WG, lds, u, n, f, numWG, lowMe); } void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { @@ -530,7 +917,7 @@ void OVERLOAD chainMul4(F2 *u, F2 w) { void OVERLOAD chainMul8(F2 *u, F2 w, u32 tailSquareBcast) { u[1] = cmulFancy(u[1], w); - //GWBUG - see FP64 version for many possible optimizations + //GWBUG - see FP64 version for many possible optimizations F2 w2 = csqTrigFancy(w); u[2] = cmulFancy(u[2], w2); @@ -553,7 +940,7 @@ void OVERLOAD chainMul(u32 len, F2 *u, F2 w, u32 tailSquareBcast) { } void OVERLOAD shufl(u32 WG, local F2 *lds, F2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl32(WG, lds, u, n, f, numWG, sb, lowMe); + shufl32(WG, lds, u, n, f, numWG, lowMe); } void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { @@ -858,7 +1245,7 @@ void OVERLOAD chainMul(u32 len, GF31 *u, GF31 w) { } void OVERLOAD shufl(u32 WG, local GF31 *lds, GF31 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl32(WG, (local F2 *) lds, (local F2 *) u, n, f, numWG, sb, lowMe); + shufl32(WG, (local F2 *) lds, (local F2 *) u, n, f, numWG, lowMe); } void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { @@ -896,7 +1283,7 @@ void OVERLOAD chainMul4(GF61 *u, GF61 w) { GF61 base = csq(w); u[2] = cmul(u[2], base); - base = cmul(base, w); //GWBUG - see FP64 version for possible optimization + base = cmul(base, w); //GWBUG - see FP64 version for possible optimization u[3] = cmul(u[3], base); } @@ -906,7 +1293,7 @@ void OVERLOAD chainMul8(GF61 *u, GF61 w, u32 tailSquareBcast) { GF61 w2 = csq(w); u[2] = cmul(u[2], w2); - GF61 base = cmul(w2, w); //GWBUG - see FP64 version for many possible optimizations + GF61 base = cmul(w2, w); //GWBUG - see FP64 version for many possible optimizations for (int i = 3; i < 8; ++i) { u[i] = cmul(u[i], base); base = cmul(base, w); @@ -921,7 +1308,7 @@ void OVERLOAD chainMul(u32 len, GF61 *u, GF61 w, u32 tailSquareBcast) { } void OVERLOAD shufl(u32 WG, local GF61 *lds, GF61 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl64(WG, (local T2 *) lds, (T2 *) u, n, f, numWG, sb, lowMe); + shufl64(WG, (local T2 *) lds, (T2 *) u, n, f, numWG, lowMe); } void OVERLOAD tabMul(u32 WG, TrigGF61 trig, GF61 *u, u32 n, u32 f, u32 me) { diff --git a/src/cl/fftheight.cl b/src/cl/fftheight.cl index 80e148db..a528bf5e 100644 --- a/src/cl/fftheight.cl +++ b/src/cl/fftheight.cl @@ -1,8 +1,16 @@ // Copyright (C) Mihai Preda -#include "base.cl" +// #defines that allow fft_height and fft_width share common code in fftbase.cl +#define VARIANT FFT_VARIANT_H +#define LDSPAD LDSPAD_H +#define LDSSWIZ LDSSWIZ_H +#define SHUFL_BYTES SHUFL_BYTES_H +#define WGSZ G_H // Change this to WG!!! +#define RADIX NH + +#include "math.cl" +#include "trig.cl" #include "fftbase.cl" -#include "middle.cl" #if SMALL_HEIGHT != 256 && SMALL_HEIGHT != 512 && SMALL_HEIGHT != 1024 && SMALL_HEIGHT != 4096 #error SMALL_HEIGHT must be one of: 256, 512, 1024, 4096 @@ -71,7 +79,7 @@ void OVERLOAD new_fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, c // This line mimics shufl -- partition lds local T2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * SMALL_HEIGHT * sb / sizeof(T2); + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); // Custom code for various SMALL_HEIGHT values @@ -208,7 +216,7 @@ void OVERLOAD new_fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, con // This line mimics shufl -- partition lds local F2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * SMALL_HEIGHT * sb / sizeof(F2); + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); // Custom code for various SMALL_HEIGHT values diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index 233aa92f..d9af5c5b 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -1,12 +1,8 @@ // Copyright (C) Mihai Preda #include "base.cl" -#include "math.cl" #include "fftheight.cl" -// LDS bytes used by shufl for each line processed in fft_HEIGHT -#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) - #if FFT_FP64 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) diff --git a/src/cl/fftmiddlein.cl b/src/cl/fftmiddlein.cl index 14b43013..1c97e612 100644 --- a/src/cl/fftmiddlein.cl +++ b/src/cl/fftmiddlein.cl @@ -1,7 +1,6 @@ // Copyright (C) Mihai Preda and George Woltman #include "base.cl" -#include "math.cl" #include "fft-middle.cl" #include "middle.cl" diff --git a/src/cl/fftmiddleout.cl b/src/cl/fftmiddleout.cl index 9249c8c3..6e05b712 100644 --- a/src/cl/fftmiddleout.cl +++ b/src/cl/fftmiddleout.cl @@ -1,7 +1,6 @@ // Copyright (C) Mihai Preda and George Woltman #include "base.cl" -#include "math.cl" #include "fft-middle.cl" #include "middle.cl" diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index 21cacf77..e7cdbd35 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -1,16 +1,15 @@ // Copyright (C) Mihai Preda #include "base.cl" -#include "math.cl" -#include "weight.cl" #include "fftwidth.cl" +#include "weight.cl" #include "middle.cl" #if FFT_TYPE == FFT64 // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[WIDTH * SHUFL_BYTES_W / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NW]; u32 g = get_group_id(0); @@ -41,7 +40,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 lds[WIDTH * SHUFL_BYTES_W / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; F2 u[NW]; u32 g = get_group_id(0); @@ -72,7 +71,7 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { - local GF31 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; GF31 u[NW]; u32 g = get_group_id(0); @@ -127,7 +126,7 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { - local GF61 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; GF61 u[NW]; u32 g = get_group_id(0); @@ -184,7 +183,7 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[WIDTH * SHUFL_BYTES_W / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; T2 u[NW]; GF31 u31[NW]; @@ -252,7 +251,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 ldsF2[WIDTH * SHUFL_BYTES_W / sizeof(F2)]; + local F2 ldsF2[LDS_BYTES / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; F2 uF2[NW]; GF31 u31[NW]; @@ -322,7 +321,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; F2 uF2[NW]; GF61 u61[NW]; @@ -392,7 +391,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { - local GF61 lds61[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; GF31 u31[NW]; GF61 u61[NW]; @@ -475,7 +474,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; F2 uF2[NW]; diff --git a/src/cl/fftw.cl b/src/cl/fftw.cl index 789a707d..6b54faf7 100644 --- a/src/cl/fftw.cl +++ b/src/cl/fftw.cl @@ -1,7 +1,6 @@ // Copyright (C) Mihai Preda #include "base.cl" -#include "math.cl" #include "fftwidth.cl" #include "middle.cl" @@ -9,7 +8,7 @@ // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[WIDTH * SHUFL_BYTES_W / sizeof(T2)]; + local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NW]; u32 g = get_group_id(0); @@ -34,7 +33,7 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[WIDTH * SHUFL_BYTES_W / sizeof(F2)]; + local F2 lds[LDS_BYTES / sizeof(F2)]; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -62,7 +61,7 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF31 KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF31)]; + local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -90,7 +89,7 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF61 KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[WIDTH * SHUFL_BYTES_W / sizeof(GF61)]; + local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); diff --git a/src/cl/fftwidth.cl b/src/cl/fftwidth.cl index 11c5d1cf..e5ef26e0 100644 --- a/src/cl/fftwidth.cl +++ b/src/cl/fftwidth.cl @@ -1,5 +1,15 @@ // Copyright (C) Mihai Preda +// #defines that allow fft_height and fft_width share common code in fftbase.cl +#define VARIANT FFT_VARIANT_W +#define LDSPAD LDSPAD_W +#define LDSSWIZ LDSSWIZ_W +#define SHUFL_BYTES SHUFL_BYTES_W +#define WGSZ G_W // Change this to WG!!! +#define RADIX NW + +#include "math.cl" +#include "trig.cl" #include "fftbase.cl" #if WIDTH != 256 && WIDTH != 512 && WIDTH != 1024 && WIDTH != 4096 && WIDTH != 625 @@ -78,7 +88,7 @@ void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u3 // This line mimics shufl -- partition lds local T2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * WIDTH * sb / sizeof(T2); + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); // Custom code for various WIDTH values @@ -249,7 +259,7 @@ void OVERLOAD new_fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, cons // This line mimics shufl -- partition lds local F2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * WIDTH * sb / sizeof(F2); + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); // Custom code for various WIDTH values diff --git a/src/cl/math.cl b/src/cl/math.cl index 09b45c5b..8408f397 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -2,8 +2,6 @@ #pragma once -#include "base.cl" - // Access parts of a 64-bit value u32 OVERLOAD lo32(u64 x) { return (u32)x; } diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index b871021b..9728edc7 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -1,12 +1,9 @@ // Copyright (C) Mihai Preda and George Woltman #include "base.cl" -#include "tailutil.cl" -#include "trig.cl" #include "fftheight.cl" - -// LDS bytes used by shufl for each line processed in fft_HEIGHT -#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) +#include "tailutil.cl" +#include "middle.cl" #if FFT_FP64 diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index c727859c..970baa32 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -1,11 +1,9 @@ // Copyright (C) Mihai Preda and George Woltman -#include "tailutil.cl" -#include "trig.cl" +#include "base.cl" #include "fftheight.cl" - -// LDS bytes used by shufl for each line processed in fft_HEIGHT -#define LDS_BYTES (SMALL_HEIGHT * SHUFL_BYTES_H) +#include "tailutil.cl" +#include "middle.cl" #if FFT_FP64 diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index 9234f858..43604852 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -1,7 +1,5 @@ // Copyright (C) Mihai Preda -#include "math.cl" - // TAIL_TRIGS setting: // 2 = No memory accesses, trig values computed from scratch. Good for excellent DP GPUs such as Titan V or Radeon VII Pro. // 1 = Limited memory accesses and some DP computation. Tuned for Radeon VII a GPU with good DP performance. @@ -141,7 +139,7 @@ void OVERLOAD reverse2(local T2 *lds2, T2 *u) { if (SHUFL_BYTES_H >= 8) { local T2 *lds = lds2; - if (me >= G_H) lds += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T2); + if (me >= G_H) lds += LDS_BYTES / sizeof(T2); // For NH=8, u[0] to u[3] are left unchanged. Write to lds: // u[7]rev u[6]rev u[5]rev u[4]rev // v[7]rev v[6]rev v[5]rev v[4]rev @@ -158,7 +156,7 @@ void OVERLOAD reverse2(local T2 *lds2, T2 *u) { else if (SHUFL_BYTES_H == 4) { local T *lds = (local T *) lds2; - if (me >= G_H) lds += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T); + if (me >= G_H) lds += LDS_BYTES / sizeof(T); bar(G_H); for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i].x; } bar(G_H); @@ -179,8 +177,8 @@ void OVERLOAD revCrossLine(local T2* lds2, T2 *u) { if (SHUFL_BYTES_H >= 8) { local T2 *ldsOut = lds2; local T2 *ldsIn = lds2; - if (me < G_H) ldsOut += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T2); // Crossing LDS halves - else ldsIn += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T2); // Staying within LDS halves (just like shufl) + if (me < G_H) ldsOut += LDS_BYTES / sizeof(T2); // Crossing LDS halves + else ldsIn += LDS_BYTES / sizeof(T2); // Staying within LDS halves (just like shufl) bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. @@ -190,8 +188,8 @@ void OVERLOAD revCrossLine(local T2* lds2, T2 *u) { else if (SHUFL_BYTES_H == 4) { local T *ldsOut = (local T *) lds2; local T *ldsIn = (local T *) lds2; - if (me < G_H) ldsOut += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T); - else ldsIn += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(T); + if (me < G_H) ldsOut += LDS_BYTES / sizeof(T); + else ldsIn += LDS_BYTES / sizeof(T); bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].x; } bar(); // we need a full bar because we just crossed halves @@ -353,7 +351,7 @@ void OVERLOAD reverse2(local F2 *lds, F2 *u) { u32 lowMe = me % G_H; if (SHUFL_BYTES_H >= 4) { - if (me >= G_H) lds += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(F2); + if (me >= G_H) lds += LDS_BYTES / sizeof(F2); // For NH=8, u[0] to u[3] are left unchanged. Write to lds: // u[7]rev u[6]rev u[5]rev u[4]rev // v[7]rev v[6]rev v[5]rev v[4]rev @@ -378,8 +376,8 @@ void OVERLOAD revCrossLine(local F2* lds2, F2 *u) { if (SHUFL_BYTES_H >= 4) { local F2 *ldsOut = lds2; local F2 *ldsIn = lds2; - if (me < G_H) ldsOut += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(F2); - else ldsIn += SMALL_HEIGHT * SHUFL_BYTES_H / sizeof(F2); + if (me < G_H) ldsOut += LDS_BYTES / sizeof(F2); + else ldsIn += LDS_BYTES / sizeof(F2); bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. diff --git a/src/cl/trig.cl b/src/cl/trig.cl index ebdd2af9..c03b544e 100644 --- a/src/cl/trig.cl +++ b/src/cl/trig.cl @@ -2,8 +2,6 @@ #pragma once -#include "math.cl" - #if FFT_FP64 T2 reducedCosSin(int k, double cosBase) { From 8592e36dbbcc820bf899147c9aad5764ad8943eb Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Apr 2026 14:57:15 +0000 Subject: [PATCH 041/214] From last commit, repaired a missing include file. --- src/cl/ffthin.cl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index d9af5c5b..faea14dd 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -2,6 +2,7 @@ #include "base.cl" #include "fftheight.cl" +#include "middle.cl" #if FFT_FP64 From 16361e928533f9b0a0594d8662b3a0f3746abdaf Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Apr 2026 17:48:26 +0000 Subject: [PATCH 042/214] Moved lots of common code from fftwith.cl and fftheight.cl into fftbase.cl. Much cleaner. --- src/cl/base.cl | 2 + src/cl/carryfused.cl | 60 ++-- src/cl/fftbase.cl | 747 ++++++++++++++++++++++++++++++++----------- src/cl/fftheight.cl | 360 ++------------------- src/cl/ffthin.cl | 8 +- src/cl/fftp.cl | 30 +- src/cl/fftw.cl | 8 +- src/cl/fftwidth.cl | 434 ++----------------------- src/cl/selftest.cl | 1 + src/cl/tailmul.cl | 136 ++++---- src/cl/tailsquare.cl | 128 ++++---- src/cl/tailutil.cl | 222 ++++++------- 12 files changed, 903 insertions(+), 1233 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 0b1feb02..e0168f8d 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -34,6 +34,8 @@ G_H "group height" == SMALL_HEIGHT / NH #define STR(x) XSTR(x) #define XSTR(x) #x +#pragma clang diagnostic ignored "-Wconstant-logical-operand" + #define OVERLOAD __attribute__((overloadable)) #pragma OPENCL FP_CONTRACT ON diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 1d1feaca..5d169d41 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -137,7 +137,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; #if !NVIDIAGPU || CUDA_BACKEND @@ -288,7 +288,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut dependentLaunch(); // Next kernel will be fftMiddleInFP64 - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -332,7 +332,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; #if !NVIDIAGPU || CUDA_BACKEND @@ -478,7 +478,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -521,7 +521,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; @@ -690,7 +690,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry dependentLaunch(); // Next kernel will be fftMiddleInGF31 - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -733,7 +733,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // common sub-expressions to re-use in the second fft_WIDTH call. Re-using this data requires dozens of VGPRs // which causes a terrible reduction in occupancy. u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; @@ -908,7 +908,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry dependentLaunch(); // Next kernel will be fftMiddleInGF61 - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); writeCarryFusedLine(u, out, line, lowMe); } @@ -956,12 +956,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; readCarryFusedLine(in, u, line, lowMe); - new_fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 readCarryFusedLine(in31, u31, line, lowMe); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); Word2 wu[NW]; #if !NVIDIAGPU || CUDA_BACKEND @@ -1148,12 +1148,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (weight_shift > 31) weight_shift -= 31; } - new_fft_WIDTH2(lds, u, smallTrig, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds, u, smallTrig, WMUL, lowMe); writeCarryFusedLine(u, out, line, lowMe); dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1204,12 +1204,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; readCarryFusedLine(inF2, uF2, line, lowMe); - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, lowMe); dependentLaunchWait(); // Previous kernel was fftMiddleOutGF31 readCarryFusedLine(in31, u31, line, lowMe); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); Word2 wu[NW]; #if !NVIDIAGPU || CUDA_BACKEND @@ -1396,12 +1396,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (weight_shift > 31) weight_shift -= 31; } - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, lowMe); writeCarryFusedLine(uF2, outF2, line, lowMe); dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); } @@ -1452,12 +1452,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; readCarryFusedLine(inF2, uF2, line, lowMe); - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, lowMe); dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 readCarryFusedLine(in61, u61, line, lowMe); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; #if !NVIDIAGPU || CUDA_BACKEND @@ -1644,12 +1644,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (weight_shift > 61) weight_shift -= 61; } - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, lowMe); writeCarryFusedLine(uF2, outF2, line, lowMe); dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds61, u61, smallTrig61, WMUL, lowMe); writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1699,12 +1699,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; readCarryFusedLine(in31, u31, line, lowMe); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 readCarryFusedLine(in61, u61, line, lowMe); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; @@ -1896,12 +1896,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); dependentLaunch(); // Next kernel will be fftMiddleInGF31 - new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds61, u61, smallTrig61, WMUL, lowMe); writeCarryFusedLine(u61, out61, line, lowMe); } @@ -1957,15 +1957,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 zerohack = ZEROHACK_W * (u32) get_group_id(0) / 131072; readCarryFusedLine(inF2, uF2, line, lowMe); - new_fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(ldsF2 + zerohack, uF2, smallTrigF2 + zerohack, WMUL, lowMe); readCarryFusedLine(in31, u31, line, lowMe); - new_fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); dependentLaunchWait(); // Previous kernel was fftMiddleOutGF61 readCarryFusedLine(in61, u61, line, lowMe); - new_fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; #if !NVIDIAGPU || CUDA_BACKEND @@ -2176,15 +2176,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - new_fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(ldsF2, uF2, smallTrigF2, WMUL, lowMe); writeCarryFusedLine(uF2, outF2, line, lowMe); dependentLaunch(); // Next kernel will be fftMiddleInFP32 - new_fft_WIDTH2(lds31, u31, smallTrig31, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds31, u31, smallTrig31, WMUL, lowMe); writeCarryFusedLine(u31, out31, line, lowMe); - new_fft_WIDTH2(lds61, u61, smallTrig61, WMUL, SHUFL_BYTES_W, lowMe); + fft_WIDTH2(lds61, u61, smallTrig61, WMUL, lowMe); writeCarryFusedLine(u61, out61, line, lowMe); } diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index dd0d5449..498d18f0 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -5,15 +5,15 @@ // Calculate the LDS bytes used by shufl #if LDSPAD && SHUFL_BYTES == 16 && RADIX == 8 -#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 72 / 64) +#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 72 / 64) #elif LDSPAD && SHUFL_BYTES == 16 && RADIX == 4 -#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 20 / 16) +#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 20 / 16) #elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 8 -#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 72 / 64) +#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 72 / 64) #elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 4 -#define LDS_BYTES ((WGSZ * RADIX * SHUFL_BYTES) * 20 / 16) +#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 20 / 16) #else -#define LDS_BYTES (WGSZ * RADIX * SHUFL_BYTES) +#define LDS_BYTES (WG * RADIX * SHUFL_BYTES) #endif #if FFT_FP64 | NTT_GF61 @@ -29,7 +29,7 @@ // before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and // only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively // minor optimization would be to special case the first shufl call. -void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u32 lowMe) { +void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { u32 mask = f - 1; assert((mask & (mask + 1)) == 0); @@ -52,11 +52,11 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 // Pad after every 8th value to eliminate bank conflicts. - if (!force_default && f == 1 && n == 8) { + if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[lowMe * 9 + i] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe * 9 + i] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } return; } @@ -66,10 +66,10 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // No padding of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. // We can however save a bar() by writing to same locations that previous shufl wrote to. - if (!force_default && f == 8 && n == 8) { - for (u32 i = 0; i < n; ++i) { lds[(i * WG + lowMe) * 9 / 8] = u[i]; } + if (!force_default && f == 8 && RADIX == 8) { + for (u32 i = 0; i < RADIX; ++i) { lds[(i * WG + lowMe) * 9 / 8] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[((lowMe & ~7) + i) * 9 + (lowMe & 7)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[((lowMe & ~7) + i) * 9 + (lowMe & 7)]; } return; } @@ -77,11 +77,11 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 // Pad after every 8th value to eliminate bank conflicts. - if (!force_default && f == 1 && n == 4) { + if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 9 / 8] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 9 / 8] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } return; } @@ -89,11 +89,11 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 // Pad 4 values after every 16th value to eliminate bank conflicts. - if (!force_default && f == 4 && n == 4) { + if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } + for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } return; } #endif @@ -103,11 +103,11 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). - if (!force_default && f == 1 && n == 8) { + if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } return; } @@ -117,10 +117,10 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. // We can however save a bar() by writing to same locations that previous shufl wrote to. - if (!force_default && f == 8 && n == 8) { - for (u32 i = 0; i < n; ++i) { lds[i * WG + lowMe] = u[i]; } + if (!force_default && f == 8 && RADIX == 8) { + for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } return; } @@ -131,9 +131,9 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). if (!force_default && f == 1 && n == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } return; } @@ -142,20 +142,20 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). - if (!force_default && f == 4 && n == 4) { + if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } return; } #endif // Otherwise, execute the original shufl code bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + lowMe]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } } // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. @@ -168,15 +168,15 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 // Pad after every 16th value to eliminate bank conflicts. - if (!force_default && f == 1 && n == 8) { + if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].x; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].y; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } return; } @@ -184,17 +184,17 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 // Pad 8 values after every 64 values to eliminate bank conflicts. - if (!force_default && f == 8 && n == 8) { + if (!force_default && f == 8 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].x; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * 72 + lowMe]; } - else for (u32 i = 0; i < n; ++i) { u32 idx = (i * WG + lowMe); u[i].x = lds[idx + idx / 64 * 8]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 72 + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u32 idx = (i * WG + lowMe); u[i].x = lds[idx + idx / 64 * 8]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].y; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * 72 + lowMe]; } - else for (u32 i = 0; i < n; ++i) { u32 idx = (i * WG + lowMe); u[i].y = lds[idx + idx / 64 * 8]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 72 + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u32 idx = (i * WG + lowMe); u[i].y = lds[idx + idx / 64 * 8]; } return; } @@ -202,15 +202,15 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 // Pad after every 16th value to eliminate bank conflicts. - if (!force_default && f == 1 && n == 4) { + if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].x; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].y; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } return; } @@ -218,15 +218,15 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 // Pad 4 values after every 16th value to eliminate bank conflicts. - if (!force_default && f == 4 && n == 4) { + if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].x; } bar(WG); - for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i].x = lds[idx + idx / 16 * 4]; } + for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i].x = lds[idx + idx / 16 * 4]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].y; } bar(WG); - for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i].y = lds[idx + idx / 16 * 4]; } + for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i].y = lds[idx + idx / 16 * 4]; } return; } #endif @@ -237,17 +237,17 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). - if (!force_default && f == 1 && n == 8) { + if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } return; } @@ -256,17 +256,17 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). - if (!force_default && f == 8 && n == 8) { + if (!force_default && f == 8 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } return; } @@ -275,15 +275,15 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). - if (!force_default && f == 1 && n == 4) { + if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } return; } @@ -292,28 +292,28 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). - if (!force_default && f == 4 && n == 4) { + if (!force_default && f == 4 && n == RADIX) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } return; } #endif // Execute the original shufl code bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + lowMe]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + lowMe]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } } // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. @@ -326,21 +326,21 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).x; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } bar(WG); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).y; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } bar(WG); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).z; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } bar(WG); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = as_int4(u[i]).w; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } bar(WG); - for (u32 i = 0; i < n; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } } } @@ -350,7 +350,7 @@ void OVERLOAD shufl64(u32 WG, local T2 *lds2, T2 *u, u32 n, u32 f, u32 numWG, u3 #if FFT_FP32 | NTT_GF31 // Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats. -void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u32 lowMe) { +void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { u32 mask = f - 1; assert((mask & (mask + 1)) == 0); @@ -375,11 +375,11 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 // Pad after every 16th value to eliminate bank conflicts. - if (!force_default && f == 1 && n == 8) { + if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } return; } @@ -387,12 +387,12 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 // Pad 8 values after every 64 values to eliminate bank conflicts. - if (!force_default && f == 8 && n == 8) { + if (!force_default && f == 8 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i]; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i] = lds[i * 72 + lowMe]; } - else for (u32 i = 0; i < n; ++i) { u32 idx = (i * WG + lowMe); u[i] = lds[idx + idx / 64 * 8]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 72 + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u32 idx = (i * WG + lowMe); u[i] = lds[idx + idx / 64 * 8]; } return; } @@ -400,11 +400,11 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 // Pad after every 16th value to eliminate bank conflicts. - if (!force_default && f == 1 && n == 4) { + if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } return; } @@ -412,11 +412,11 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 // Pad 4 values after every 16th value to eliminate bank conflicts. - if (!force_default && f == 4 && n == 4) { + if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } + for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } return; } #endif @@ -427,12 +427,12 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). - if (!force_default && f == 1 && n == 8) { + if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } return; } @@ -441,12 +441,12 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). - if (!force_default && f == 8 && n == 8) { + if (!force_default && f == 8 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } bar(WG); - if (WG == 64) for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } return; } @@ -455,11 +455,11 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). - if (!force_default && f == 1 && n == 4) { + if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } return; } @@ -468,20 +468,20 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). - if (!force_default && f == 4 && n == 4) { + if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < n; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } return; } #endif // Execute the original shufl code bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i] = lds[i * WG + lowMe]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } } // If SHUFL_BYTES is 4 we split the F2 values into 2 int values. These are written to LDS memory using two instructions. @@ -492,13 +492,13 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F); bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].x = lds[i * WG + lowMe]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } bar(WG); - for (u32 i = 0; i < n; ++i) { lds[i * f + (lowMe & ~mask) * n + (lowMe & mask)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } bar(WG); - for (u32 i = 0; i < n; ++i) { u[i].y = lds[i * WG + lowMe]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } } } @@ -507,6 +507,10 @@ void OVERLOAD shufl32(u32 WG, local F2 *lds2, F2 *u, u32 n, u32 f, u32 numWG, u3 #if FFT_FP64 +void OVERLOAD shufl(local T2 *lds, T2 *u, u32 f, u32 numWG, u32 lowMe) { + shufl64(lds, u, f, numWG, lowMe); +} + void OVERLOAD chainMul4(T2 *u, T2 w) { u[1] = cmul(u[1], w); @@ -521,12 +525,12 @@ void OVERLOAD chainMul4(T2 *u, T2 w) { #if 1 // This version of chainMul8 tries to minimize roundoff error even if more F64 ops are used. // Trial and error looking at Z values on a WIDTH=512 FFT was used to determine when to switch from fancy to non-fancy powers of w. -void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(T2 *u, T2 w) { u[1] = cmulFancy(u[1], w); T2 w2; - // Rocm optimizer behaves weirdly. Using multiple mul2s instead of one mul2 in csqTrigFancy makes double-wide single-kernel tailSquare inexplicably slower - if (!tailSquareBcast) { + // Rocm optimizer behaves weirdly. Using multiple mul2s instead of one mul2 in csqTrigFancy makes double-wide single-kernel variant 0 tailSquare inexplicably slower + if (DOING_WIDTH || VARIANT != 0) { w2 = csqTrigFancy(w); } else { w2 = U2(mulminus2(w.y) * w.y, mul2(fma(w.x, w.y, w.y))); @@ -536,7 +540,7 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { T2 w3; // Rocm optimizer behaves weirdly yet again. Using mul2 instead of 2.0* makes double-wide single-kernel tailSquare inexplicably slower // even though it is one fewer F64 op. - if (!tailSquareBcast) { + if (DOING_WIDTH || VARIANT != 0) { w3 = ccubeTrigFancy(w2, w); } else { double a = 2*w2.y; @@ -554,12 +558,12 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { #else // This version of chainMul8 minimizes F64 ops even if that increases roundoff error. -// This version is faster on a Radeon 7 with worse roundoff. However, new_FFT_width is even faster with better roundoff. +// This version is faster on a Radeon 7 with worse roundoff. However, FFT_width is even faster with better roundoff. // This version is the same speed on a TitanV probably due to its great F64 throughput. // This version is slower on R7Pro due to a rocm optimizer issue in double-wide single-kernel tailSquare using BCAST. I could not find a work-around. // Other GPUs??? This version might be useful. If we decide to make this available, it will need a new width and height fft spec number. // Consequently, an increase in the BPW table and increase work for -ztune and -tune. -void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(T2 *u, T2 w) { u[1] = cmulFancy(u[1], w); T2 w2 = csqTrigFancy(w); @@ -579,15 +583,15 @@ void OVERLOAD chainMul8(T2 *u, T2 w, u32 tailSquareBcast) { } #endif -void OVERLOAD chainMul(u32 len, T2 *u, T2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul(T2 *u, T2 w) { // Do a length 4 chain mul, w must not be in Fancy format - if (len == 4) chainMul4(u, w); + if (RADIX == 4) chainMul4(u, w); // Do a length 8 chain mul, w must be in Fancy format - if (len == 8) chainMul8(u, w, tailSquareBcast); + if (RADIX == 8) chainMul8(u, w); } -#if AMDGPU && (FFT_VARIANT_W == 0 || FFT_VARIANT_H == 0) +#if AMDGPU && VARIANT == 0 int bcast4(int x) { return __builtin_amdgcn_mov_dpp(x, 0, 0xf, 0xf, false); } int bcast8(int x) { return __builtin_amdgcn_ds_swizzle(x, 0x0018); } @@ -606,11 +610,19 @@ T2 bcast(T2 src, u32 span) { #endif -void OVERLOAD shufl(u32 WG, local T2 *lds, T2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl64(WG, lds, u, n, f, numWG, lowMe); +void OVERLOAD fft_RADIX(T2 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 5 + fft5(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif } -void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { +void OVERLOAD tabMul(Trig trig, T2 *u, u32 f, u32 me) { #if 0 u32 p = me / f * f; #else @@ -619,9 +631,9 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { // Compute trigs from scratch every time. This can't possibly be a good idea on any GPUs. #if 0 - T2 w = slowTrig_N(ND / n / WG * p, ND / n); + T2 w = slowTrig_N(ND / RADIX / WG * p, ND / RADIX); T2 base = w; - for (int i = 1; i < n; ++i) { + for (int i = 1; i < RADIX; ++i) { u[i] = cmul(u[i], w); w = cmul(w, base); } @@ -634,7 +646,7 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { if (TABMUL_CHAIN) { T2 w = TFLOAD(&trig[p]); - chainMul(n, u, w, 0); + chainMul(u, w); return; } @@ -644,20 +656,19 @@ void OVERLOAD tabMul(u32 WG, Trig trig, T2 *u, u32 n, u32 f, u32 me) { if (!TABMUL_CHAIN) { T2 w = TFLOAD(&trig[p]); - if (n >= 8) { + if (RADIX >= 8) { u[1] = cmulFancy(u[1], w); } else { u[1] = cmul(u[1], w); } - for (u32 i = 2; i < n; ++i) { + for (u32 i = 2; i < RADIX; ++i) { u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } } - //************************************************************************************ // New fft WIDTH and HEIGHT macros to support radix-4 FFTs with more FMA instructions //************************************************************************************ @@ -674,7 +685,7 @@ T2 partial_cmul(T2 u, T sine_over_cosine) { #define X2_via_FMA(a, c, b) { T2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul4_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { +void preload_tabMul4_trig(Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { TrigSingle trig1 = (TrigSingle) trig; // Read 3 lines of sine/cosine values for the first fft4. Read two of the lines as a pair as AMD likes T2 global memory reads @@ -687,7 +698,7 @@ void preload_tabMul4_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { +void partial_tabMul4(local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { local T *lds1 = (local T *) lds; TrigSingle trig1 = (TrigSingle) trig; trig1 += 4*WG; // Skip past sine_over_cosine values @@ -728,7 +739,7 @@ void partial_tabMul4(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f } // Finish off a partial tabMul while doing next fft4 making more use of FMA. -void finish_tabMul4_fft4(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { +void finish_tabMul4_fft4(Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingle trig1 = (TrigSingle) trig; // @@ -761,7 +772,7 @@ void finish_tabMul4_fft4(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG //************************************************************************************ // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul8_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { +void preload_tabMul8_trig(Trig trig, T *preloads, u32 f, u32 numWG, u32 me) { TrigSingle trig1 = (TrigSingle) trig; // Read 7 lines of sine/cosine values for the first fft8. Read six of the lines as pairs as AMD likes T2 global memory reads @@ -776,7 +787,7 @@ void preload_tabMul8_trig(u32 WG, Trig trig, T *preloads, u32 f, u32 numWG, u32 } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { +void partial_tabMul8(local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me) { local T *lds1 = (local T *) lds; TrigSingle trig1 = (TrigSingle) trig; trig1 += 8*WG; // Skip past sine_over_cosine values @@ -822,7 +833,7 @@ void partial_tabMul8(u32 WG, local T2 *lds, Trig trig, T *preloads, T2 *u, u32 f } // Finish off a partial tabMul while doing next fft8 making more use of FMA. -void finish_tabMul8_fft8(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { +void finish_tabMul8_fft8(Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingle trig1 = (TrigSingle) trig; // @@ -832,7 +843,7 @@ void finish_tabMul8_fft8(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG // Apply cosine0 to u[0] if (f < WG/8) u[0] = u[0] * preloads[0]; - if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in new_fft_WIDTH case. + if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in fft_WIDTH case. // Apply cosine4, cosine5/cosine1, cosine6/cosine2, cosine7/cosine3 to u[4] through u[7] using FMA X2_via_FMA(u[0], preloads[4], u[4]); @@ -895,6 +906,172 @@ void finish_tabMul8_fft8(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG SWAP(u[3], u[6]); } + +void OVERLOAD fft_common(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe, int callnum) { + + // This line mimics shufl -- partition lds for variant 2 + local T2* partitioned_lds = lds; + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); + +// Variant 0 uses broadcast instructions. Only available on AMD GPUs. + +#if VARIANT == 0 + +#if WG * RADIX > 1024 +#error VARIANT == 0 only supported for FFT size <= 1024 +#endif +#if !AMDGPU +#error VARIANT == 0 only supported by AMD GPUs +#endif + +// There is a slight difference between fft_WIDTH and fft_HEIGHT. Tail square computes the trig values +// to be broadcast, while fft_WIDTH does not. Compute the trig values now for fft_WIDTH, +#if DOING_WIDTH +#if RADIX == 8 + w = fancyTrig_N(ND / (WG * RADIX) * lowMe); +#else + w = slowTrig_N(ND / (WG * RADIX) * lowMe, ND / RADIX); +#endif +#endif + + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + w = bcast(w, s); + chainMul(u, w); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +// Variant 2 uses more FMA instructions than the original FFT code. +// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. +// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. +// The downside is sine/cosine cannot be computed with chained multiplies. + +// Variant 2 code for SIZE=256, RADIX=4 +#elif WG == 64 && RADIX == 4 && VARIANT == 2 + + T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish third tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + +// Variant 2 code for SIZE=512, RADIX=8 +#elif WG == 64 && RADIX == 8 && VARIANT == 2 + + T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8 + SAVE_ONE_MUL*2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, SAVE_ONE_MUL); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish second tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, SAVE_ONE_MUL); // We'd rather set save_one_more_mul to 1 + +// Variant 2 code for SIZE=1024, RADIX=4 +#elif WG == 256 && RADIX == 4 && VARIANT == 2 + + T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish fourth tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 64, numWG, lowMe, 1); + +// Custom code for SIZE=4K, RADIX=8 +#elif WG == 512 && RADIX == 8 && VARIANT == 2 + + T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish third tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + +#else + + // Old / original version + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +#endif +} + #endif @@ -904,6 +1081,20 @@ void finish_tabMul8_fft8(u32 WG, Trig trig, T *preloads, T2 *u, u32 f, u32 numWG #if FFT_FP32 +void OVERLOAD shufl(local F2 *lds, F2 *u, u32 f, u32 numWG, u32 lowMe) { + shufl32(lds, u, f, numWG, lowMe); +} + +void OVERLOAD fft_RADIX(F2 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif +} + void OVERLOAD chainMul4(F2 *u, F2 w) { u[1] = cmul(u[1], w); @@ -915,7 +1106,7 @@ void OVERLOAD chainMul4(F2 *u, F2 w) { u[3] = cmul(u[3], base); } -void OVERLOAD chainMul8(F2 *u, F2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(F2 *u, F2 w) { u[1] = cmulFancy(u[1], w); //GWBUG - see FP64 version for many possible optimizations F2 w2 = csqTrigFancy(w); @@ -932,36 +1123,32 @@ void OVERLOAD chainMul8(F2 *u, F2 w, u32 tailSquareBcast) { } } -void OVERLOAD chainMul(u32 len, F2 *u, F2 w, u32 tailSquareBcast) { +void OVERLOAD chainMul(F2 *u, F2 w) { // Do a length 4 chain mul - if (len == 4) chainMul4(u, w); + if (RADIX == 4) chainMul4(u, w); // Do a length 8 chain mul - if (len == 8) chainMul8(u, w, tailSquareBcast); + if (RADIX == 8) chainMul8(u, w); } -void OVERLOAD shufl(u32 WG, local F2 *lds, F2 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl32(WG, lds, u, n, f, numWG, lowMe); -} - -void OVERLOAD tabMul(u32 WG, TrigFP32 trig, F2 *u, u32 n, u32 f, u32 me) { +void OVERLOAD tabMul(TrigFP32 trig, F2 *u, u32 f, u32 me) { u32 p = me & ~(f - 1); // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN32) { - chainMul(n, u, TFLOAD(&trig[p]), 0); + chainMul(u, TFLOAD(&trig[p])); return; } // Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. if (!TABMUL_CHAIN32) { - if (n >= 8) { + if (RADIX >= 8) { u[1] = cmulFancy(u[1], TFLOAD(&trig[p])); } else { u[1] = cmul(u[1], TFLOAD(&trig[p])); } - for (u32 i = 2; i < n; ++i) { + for (u32 i = 2; i < RADIX; ++i) { u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; @@ -984,7 +1171,7 @@ F2 partial_cmul(F2 u, F sine_over_cosine) { #define X2_via_FMA(a, c, b) { F2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul4_trig(u32 WG, TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { +void preload_tabMul4_trig(TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { TrigSingleFP32 trig1 = (TrigSingleFP32) trig; // Read 3 lines of sine/cosine values for the first fft4. Read two of the lines as a pair as AMD likes T2 global memory reads @@ -997,7 +1184,7 @@ void preload_tabMul4_trig(u32 WG, TrigFP32 trig, F *preloads, u32 f, u32 numWG, } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul4(u32 WG, local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { +void partial_tabMul4(local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { local F *lds1 = (local F *) lds; TrigSingleFP32 trig1 = (TrigSingleFP32) trig; trig1 += 4*WG; // Skip past sine_over_cosine values @@ -1038,7 +1225,7 @@ void partial_tabMul4(u32 WG, local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u } // Finish off a partial tabMul while doing next fft4 making more use of FMA. -void finish_tabMul4_fft4(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { +void finish_tabMul4_fft4(TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingleFP32 trig1 = (TrigSingleFP32) trig; // @@ -1071,7 +1258,7 @@ void finish_tabMul4_fft4(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 n //************************************************************************************ // Preload trig values for the first partial tabMul. We load the sine/cosine values early so that F64 ops can hide the read latency. -void preload_tabMul8_trig(u32 WG, TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { +void preload_tabMul8_trig(TrigFP32 trig, F *preloads, u32 f, u32 numWG, u32 me) { TrigSingleFP32 trig1 = (TrigSingleFP32) trig; // Read 7 lines of sine/cosine values for the first fft8. Read six of the lines as pairs as AMD likes T2 global memory reads @@ -1086,7 +1273,7 @@ void preload_tabMul8_trig(u32 WG, TrigFP32 trig, F *preloads, u32 f, u32 numWG, } // Do a partial tabMul. Save the mul-by-cosine for later FMA instructions. -void partial_tabMul8(u32 WG, local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { +void partial_tabMul8(local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me) { local F *lds1 = (local F *) lds; TrigSingleFP32 trig1 = (TrigSingleFP32) trig; trig1 += 8*WG; // Skip past sine_over_cosine values @@ -1132,7 +1319,7 @@ void partial_tabMul8(u32 WG, local F2 *lds, TrigFP32 trig, F *preloads, F2 *u, u } // Finish off a partial tabMul while doing next fft8 making more use of FMA. -void finish_tabMul8_fft8(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { +void finish_tabMul8_fft8(TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u32 me, u32 save_one_more_mul) { TrigSingleFP32 trig1 = (TrigSingleFP32) trig; // @@ -1142,7 +1329,7 @@ void finish_tabMul8_fft8(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 n // Apply cosine0 to u[0] if (f < WG/8) u[0] = u[0] * preloads[0]; - if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in new_fft_WIDTH case. + if (save_one_more_mul) { // This should always be the best option. ROCm optimizer is doing something weird in fft_WIDTH case. // Apply cosine4, cosine5/cosine1, cosine6/cosine2, cosine7/cosine3 to u[4] through u[7] using FMA X2_via_FMA(u[0], preloads[4], u[4]); @@ -1205,6 +1392,142 @@ void finish_tabMul8_fft8(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 n SWAP(u[3], u[6]); } +// Variant 2 code uses more FMA instructions than the original fft version. +// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. +// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. +// The downside is sine/cosine cannot be computed with chained multiplies. + +void OVERLOAD fft_common(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe, int callnum) { + + // This line mimics shufl -- partition lds + local F2* partitioned_lds = lds; + if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); + +// Variant 2 code for SIZE=256, RADIX=4 +#if ENABLE_FP32_VARIANT_2 && WG == 64 && RADIX == 4 && VARIANT == 2 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish third tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + +// Variant 2 code for SIZE=512, RADIX=8 +#elif ENABLE_FP32_VARIANT_2 && WG == 64 && RADIX == 8 && VARIANT == 2 + + F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8 + SAVE_ONE_MUL*2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, SAVE_ONE_MUL); + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish second tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, SAVE_ONE_MUL); + +// Variant 2 code for SIZE=1024, RADIX=4 +#elif ENABLE_FP32_VARIANT_2 && WG == 256 && RADIX == 4 && VARIANT == 2 + + F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul4_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft4, partial tabMul, and shufl. + fft4(u); + partial_tabMul4(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 1, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 4, numWG, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 4, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 16, numWG, lowMe); + shufl(lds, u, 16, numWG, lowMe); + + // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. + finish_tabMul4_fft4(trig, preloads, u, 16, numWG, lowMe, 1); + partial_tabMul4(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish fourth tabMul and perform final fft4. + finish_tabMul4_fft4(trig, preloads, u, 64, numWG, lowMe, 1); + +// Variant 2 code for SIZE=4K, RADIX=8 +#elif ENABLE_FP32_VARIANT_2 && WG == 512 && RADIX == 8 && VARIANT == 2 + + F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. + trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values + + // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. + preload_tabMul8_trig(trig, preloads, 1, numWG, lowMe); + + // Do first fft8, partial tabMul, and shufl. + fft8(u); + partial_tabMul8(partitioned_lds, trig, preloads, u, 1, numWG, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 8, numWG, lowMe); + shufl(lds, u, 8, numWG, lowMe); + + // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. + finish_tabMul8_fft8(trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + partial_tabMul8(partitioned_lds, trig, preloads, u, 64, numWG, lowMe); + shufl(lds, u, 64, numWG, lowMe); + + // Finish third tabMul and perform final fft8. + finish_tabMul8_fft8(trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + +#else + + // Old / original version + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); + +#endif +} + #endif @@ -1214,6 +1537,20 @@ void finish_tabMul8_fft8(u32 WG, TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 n #if NTT_GF31 +void OVERLOAD shufl(local GF31 *lds, GF31 *u, u32 f, u32 numWG, u32 lowMe) { + shufl32((local F2 *) lds, (local F2 *) u, f, numWG, lowMe); +} + +void OVERLOAD fft_RADIX(GF31 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif +} + void OVERLOAD chainMul4(GF31 *u, GF31 w) { u[1] = cmul(u[1], w); @@ -1237,37 +1574,46 @@ void OVERLOAD chainMul8(GF31 *u, GF31 w) { } } -void OVERLOAD chainMul(u32 len, GF31 *u, GF31 w) { +void OVERLOAD chainMul(GF31 *u, GF31 w) { // Do a length 4 chain mul - if (len == 4) chainMul4(u, w); + if (RADIX == 4) chainMul4(u, w); // Do a length 8 chain mul - if (len == 8) chainMul8(u, w); -} - -void OVERLOAD shufl(u32 WG, local GF31 *lds, GF31 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl32(WG, (local F2 *) lds, (local F2 *) u, n, f, numWG, lowMe); + if (RADIX == 8) chainMul8(u, w); } -void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { +void OVERLOAD tabMul(TrigGF31 trig, GF31 *u, u32 f, u32 me) { u32 p = me & ~(f - 1); // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN31) { - chainMul(n, u, TFLOAD(&trig[p])); + chainMul(u, TFLOAD(&trig[p])); return; } // Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. if (!TABMUL_CHAIN31) { - for (u32 i = 1; i < n; ++i) { + for (u32 i = 1; i < RADIX; ++i) { u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } } +void OVERLOAD fft_common(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); +} + #endif @@ -1277,6 +1623,20 @@ void OVERLOAD tabMul(u32 WG, TrigGF31 trig, GF31 *u, u32 n, u32 f, u32 me) { #if NTT_GF61 +void OVERLOAD shufl(local GF61 *lds, GF61 *u, u32 f, u32 numWG, u32 lowMe) { + shufl64((local T2 *) lds, (T2 *) u, f, numWG, lowMe); +} + +void OVERLOAD fft_RADIX(GF61 *u) { +#if RADIX == 4 + fft4(u); +#elif RADIX == 8 + fft8(u); +#else +#error RADIX +#endif +} + void OVERLOAD chainMul4(GF61 *u, GF61 w) { u[1] = cmul(u[1], w); @@ -1287,7 +1647,7 @@ void OVERLOAD chainMul4(GF61 *u, GF61 w) { u[3] = cmul(u[3], base); } -void OVERLOAD chainMul8(GF61 *u, GF61 w, u32 tailSquareBcast) { +void OVERLOAD chainMul8(GF61 *u, GF61 w) { u[1] = cmul(u[1], w); GF61 w2 = csq(w); @@ -1300,35 +1660,44 @@ void OVERLOAD chainMul8(GF61 *u, GF61 w, u32 tailSquareBcast) { } } -void OVERLOAD chainMul(u32 len, GF61 *u, GF61 w, u32 tailSquareBcast) { +void OVERLOAD chainMul(GF61 *u, GF61 w) { // Do a length 4 chain mul - if (len == 4) chainMul4(u, w); + if (RADIX == 4) chainMul4(u, w); // Do a length 8 chain mul - if (len == 8) chainMul8(u, w, tailSquareBcast); -} - -void OVERLOAD shufl(u32 WG, local GF61 *lds, GF61 *u, u32 n, u32 f, u32 numWG, const u32 sb, u32 lowMe) { - shufl64(WG, (local T2 *) lds, (T2 *) u, n, f, numWG, lowMe); + if (RADIX == 8) chainMul8(u, w); } -void OVERLOAD tabMul(u32 WG, TrigGF61 trig, GF61 *u, u32 n, u32 f, u32 me) { +void OVERLOAD tabMul(TrigGF61 trig, GF61 *u, u32 f, u32 me) { u32 p = me & ~(f - 1); // This code uses chained complex multiplies which could be faster on GPUs with great mul throughput or poor memory bandwidth or caching. if (TABMUL_CHAIN61) { - chainMul(n, u, TFLOAD(&trig[p]), 0); + chainMul(u, TFLOAD(&trig[p])); return; } // Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. if (!TABMUL_CHAIN61) { - for (u32 i = 1; i < n; ++i) { + for (u32 i = 1; i < RADIX; ++i) { u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*WG + p])); } return; } } +void OVERLOAD fft_common(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { + +#if !UNROLL + __attribute__((opencl_unroll_hint(1))) +#endif + for (u32 s = 1; s < WG; s *= RADIX) { + fft_RADIX(u); + tabMul(trig, u, s, lowMe); + shufl(lds, u, s, numWG, lowMe); + } + fft_RADIX(u); +} + #endif diff --git a/src/cl/fftheight.cl b/src/cl/fftheight.cl index a528bf5e..7d835b41 100644 --- a/src/cl/fftheight.cl +++ b/src/cl/fftheight.cl @@ -1,19 +1,23 @@ // Copyright (C) Mihai Preda // #defines that allow fft_height and fft_width share common code in fftbase.cl +#define WG G_H +#define RADIX NH #define VARIANT FFT_VARIANT_H #define LDSPAD LDSPAD_H #define LDSSWIZ LDSSWIZ_H #define SHUFL_BYTES SHUFL_BYTES_H -#define WGSZ G_H // Change this to WG!!! -#define RADIX NH +#define UNROLL UNROLL_H +#define SAVE_ONE_MUL 1 // Radeon VII weirdness where saving one mul was slower (needs retesting!) +#define DOING_HEIGHT 1 // Flags to work around any optimizer weirdness where common code performs better in fft_WIDTH and worse in fft_HEIGHT or vice versa +#define DOING_WIDTH 0 #include "math.cl" #include "trig.cl" #include "fftbase.cl" -#if SMALL_HEIGHT != 256 && SMALL_HEIGHT != 512 && SMALL_HEIGHT != 1024 && SMALL_HEIGHT != 4096 -#error SMALL_HEIGHT must be one of: 256, 512, 1024, 4096 +#if SMALL_HEIGHT != 256 && SMALL_HEIGHT != 512 && SMALL_HEIGHT != 1024 +#error SMALL_HEIGHT must be one of: 256, 512, 1024 #endif #if !INPLACE @@ -24,159 +28,10 @@ u32 transPos(u32 k, u32 middle, u32 width) { return k; } #if FFT_FP64 -void OVERLOAD fft_NH(T2 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -#if FFT_VARIANT_H == 0 - -#if HEIGHT > 1024 -#error FFT_VARIANT_H == 0 only supports HEIGHT <= 1024 -#endif -#if !AMDGPU -#error FFT_VARIANT_H == 0 only supported by AMD GPUs -#endif - -void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = SMALL_HEIGHT / NH; - for (u32 s = 1; s < WG; s *= NH) { - fft_NH(u); - w = bcast(w, s); - - chainMul(NH, u, w, 1); - - shufl(WG, lds, u, NH, s, numWG, sb, lowMe); - } - fft_NH(u); -} - -#else - -void OVERLOAD fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NH) { - fft_NH(u); - tabMul(WG, trig, u, NH, s, lowMe); - shufl(WG, lds, u, NH, s, numWG, sb, lowMe); - } - fft_NH(u); -} - -#endif - -void OVERLOAD new_fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe, int callnum) { - u32 WG = SMALL_HEIGHT / NH; - - // This line mimics shufl -- partition lds - local T2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); - -// Custom code for various SMALL_HEIGHT values - -#if SMALL_HEIGHT == 256 && NH == 4 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=256, NH=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); - - // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - -#elif SMALL_HEIGHT == 512 && NH == 8 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=512, NH=8 - - T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8 + 2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); - shufl(WG, lds, u, NH, 8, numWG, sb, lowMe); - - // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 1); - -#elif SMALL_HEIGHT == 1024 && NH == 4 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=1024, NH=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); - - // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); - shufl(WG, lds, u, NH, 64, numWG, sb, lowMe); - - // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); - -#else - - // Old version - fft_HEIGHT(lds, u, trig, w, numWG, sb, lowMe); - -#endif -} - -void new_fft_HEIGHT1(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, w, numWG, sb, lowMe, 1); } -void new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, w, numWG, sb, lowMe, 2); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void fft_HEIGHT(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, w, numWG, lowMe, 0); } +void fft_HEIGHT1(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, w, numWG, lowMe, 1); } +void fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, w, numWG, lowMe, 2); } #endif @@ -187,133 +42,10 @@ void new_fft_HEIGHT2(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, const u32 #if FFT_FP32 -void OVERLOAD fft_NH(F2 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -void OVERLOAD fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NH) { - fft_NH(u); - tabMul(WG, trig, u, NH, s, lowMe); - shufl(WG, lds, u, NH, s, numWG, sb, lowMe); - } - fft_NH(u); -} - -void OVERLOAD new_fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe, int callnum) { - u32 WG = SMALL_HEIGHT / NH; - - // This line mimics shufl -- partition lds - local F2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); - -// Custom code for various SMALL_HEIGHT values - -#if ENABLE_FP32_VARIANT_2 && SMALL_HEIGHT == 256 && NH == 4 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=256, NH=4 - - F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); - - // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - -#elif ENABLE_FP32_VARIANT_2 && SMALL_HEIGHT == 512 && NH == 8 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=512, NH=8 - - F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8 + 2*WG*8; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); - shufl(WG, lds, u, NH, 8, numWG, sb, lowMe); - - // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 1); - -#elif ENABLE_FP32_VARIANT_2 && SMALL_HEIGHT == 1024 && NH == 4 && FFT_VARIANT_H == 2 - -// Custom code for SMALL_HEIGHT=1024, NH=4 - - F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NH, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NH, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NH, 16, numWG, sb, lowMe); - - // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); - shufl(WG, lds, u, NH, 64, numWG, sb, lowMe); - - // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); - -#else - - // Old version - fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); - -#endif -} - -void new_fft_HEIGHT1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, numWG, sb, lowMe, 1); } -void new_fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_HEIGHT(lds, u, trig, numWG, sb, lowMe, 2); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void fft_HEIGHT(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 0); } +void fft_HEIGHT1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 1); } +void fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 2); } #endif @@ -324,32 +56,10 @@ void new_fft_HEIGHT2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 s #if NTT_GF31 -void OVERLOAD fft_NH(GF31 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -void OVERLOAD fft_HEIGHT(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NH) { - fft_NH(u); - tabMul(WG, trig, u, NH, s, lowMe); - shufl(WG, lds, u, NH, s, numWG, sb, lowMe); - } - fft_NH(u); -} - -void OVERLOAD new_fft_HEIGHT1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } -void OVERLOAD new_fft_HEIGHT2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_HEIGHT(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif @@ -360,31 +70,9 @@ void OVERLOAD new_fft_HEIGHT2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG #if NTT_GF61 -void OVERLOAD fft_NH(GF61 *u) { -#if NH == 4 - fft4(u); -#elif NH == 8 - fft8(u); -#else -#error NH -#endif -} - -void OVERLOAD fft_HEIGHT(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = SMALL_HEIGHT / NH; - -#if !UNROLL_H - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NH) { - fft_NH(u); - tabMul(WG, trig, u, NH, s, lowMe); - shufl(WG, lds, u, NH, s, numWG, sb, lowMe); - } - fft_NH(u); -} - -void OVERLOAD new_fft_HEIGHT1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } -void OVERLOAD new_fft_HEIGHT2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_HEIGHT(lds, u, trig, numWG, sb, lowMe); } +// Three versions. fft_HEIGHT1 and fft_HEIGHT2 are for the two tailSquare calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_HEIGHT(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_HEIGHT2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index faea14dd..02d388d3 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -22,7 +22,7 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); #endif - fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig, w, 1, me); write(G_H, NH, u, out, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } @@ -56,7 +56,7 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { F2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); #endif - fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrigF2, 1, me); write(G_H, NH, u, outF2, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } @@ -84,7 +84,7 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in31, u, g, me); - fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig31, 1, me); write(G_H, NH, u, out31, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } @@ -112,7 +112,7 @@ KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in61, u, g, me); - fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT(lds, u, smallTrig61, 1, me); write(G_H, NH, u, out61, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); } diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index e7cdbd35..b263d8ab 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -26,7 +26,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) u[i] = U2(in[p].x * w1, in[p].y * w2); } - fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig, 1, me); writeCarryFusedLine(u, out, g, me); } @@ -57,7 +57,7 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ u[i] = U2(in[p].x * w1, in[p].y * w2); } - fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig, 1, me); writeCarryFusedLine(u, out, g, me); } @@ -112,7 +112,7 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig, 1, me); writeCarryFusedLine(u, out, g, me); } @@ -169,7 +169,7 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { if (weight_shift > 61) weight_shift -= 61; } - fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig, 1, me); writeCarryFusedLine(u, out, g, me); } @@ -235,10 +235,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig, 1, me); writeCarryFusedLine(u, out, g, me); - fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds31, u31, smallTrig31, 1, me); writeCarryFusedLine(u31, out31, g, me); } @@ -305,10 +305,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG if (weight_shift > 31) weight_shift -= 31; } - fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, SHUFL_BYTES_W, me); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, me); writeCarryFusedLine(uF2, outF2, g, me); - fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds31, u31, smallTrig31, 1, me); writeCarryFusedLine(u31, out31, g, me); } @@ -375,10 +375,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG if (weight_shift > 61) weight_shift -= 61; } - fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, SHUFL_BYTES_W, me); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, me); writeCarryFusedLine(uF2, outF2, g, me); - fft_WIDTH(lds61, u61, smallTrig61, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds61, u61, smallTrig61, 1, me); writeCarryFusedLine(u61, out61, g, me); } @@ -458,10 +458,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds31, u31, smallTrig31, 1, me); writeCarryFusedLine(u31, out31, g, me); - fft_WIDTH(lds61, u61, smallTrig61, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds61, u61, smallTrig61, 1, me); writeCarryFusedLine(u61, out61, g, me); } @@ -550,13 +550,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG m61_weight_shift = adjust_m61_weight_shift(m61_weight_shift); } - fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, SHUFL_BYTES_W, me); + fft_WIDTH(ldsF2, uF2, smallTrigF2, 1, me); writeCarryFusedLine(uF2, outF2, g, me); - fft_WIDTH(lds31, u31, smallTrig31, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds31, u31, smallTrig31, 1, me); writeCarryFusedLine(u31, out31, g, me); - fft_WIDTH(lds61, u61, smallTrig61, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds61, u61, smallTrig61, 1, me); writeCarryFusedLine(u61, out61, g, me); } diff --git a/src/cl/fftw.cl b/src/cl/fftw.cl index 6b54faf7..72fe1ea8 100644 --- a/src/cl/fftw.cl +++ b/src/cl/fftw.cl @@ -17,7 +17,7 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunchWait(); // Previous kernel was fftMiddleOut readCarryFusedLine(in, u, g, me); - fft_WIDTH(lds, u, smallTrig, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig, 1, me); out += WIDTH * g; write(G_W, NW, u, out, 0); } @@ -46,7 +46,7 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunchWait(); // Previous kernel was fftMiddleOut readCarryFusedLine(inF2, u, g, me); - fft_WIDTH(lds, u, smallTrigF2, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrigF2, 1, me); outF2 += WIDTH * g; write(G_W, NW, u, outF2, 0); } @@ -74,7 +74,7 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunchWait(); // Previous kernel was fftMiddleOut readCarryFusedLine(in31, u, g, me); - fft_WIDTH(lds, u, smallTrig31, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig31, 1, me); out31 += WIDTH * g; write(G_W, NW, u, out31, 0); } @@ -102,7 +102,7 @@ KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunchWait(); // Previous kernel was fftMiddleOut readCarryFusedLine(in61, u, g, me); - fft_WIDTH(lds, u, smallTrig61, 1, SHUFL_BYTES_W, me); + fft_WIDTH(lds, u, smallTrig61, 1, me); out61 += WIDTH * g; write(G_W, NW, u, out61, 0); } diff --git a/src/cl/fftwidth.cl b/src/cl/fftwidth.cl index e5ef26e0..f1c65c0e 100644 --- a/src/cl/fftwidth.cl +++ b/src/cl/fftwidth.cl @@ -1,12 +1,16 @@ // Copyright (C) Mihai Preda // #defines that allow fft_height and fft_width share common code in fftbase.cl +#define WG G_W +#define RADIX NW #define VARIANT FFT_VARIANT_W #define LDSPAD LDSPAD_W #define LDSSWIZ LDSSWIZ_W #define SHUFL_BYTES SHUFL_BYTES_W -#define WGSZ G_W // Change this to WG!!! -#define RADIX NW +#define UNROLL UNROLL_W +#define SAVE_ONE_MUL 0 // Radeon VII weirdness where saving one mul in width variant 2 was slower +#define DOING_HEIGHT 0 // Flags to work around any optimizer weirdness where common code performs better in fft_WIDTH and worse in fft_HEIGHT or vice versa +#define DOING_WIDTH 1 #include "math.cl" #include "trig.cl" @@ -18,203 +22,10 @@ #if FFT_FP64 -void OVERLOAD fft_NW(T2 *u) { -#if NW == 4 - fft4(u); -#elif NW == 5 - fft5(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -#if FFT_VARIANT_W == 0 - -#if WIDTH > 1024 -#error FFT_VARIANT_W == 0 only supports WIDTH <= 1024 -#endif -#if !AMDGPU -#error FFT_VARIANT_W == 0 only supported by AMD GPUs -#endif - -void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = WIDTH / NW; - -#if NW == 8 - T2 w = fancyTrig_N(ND / WIDTH * lowMe); -#else - T2 w = slowTrig_N(ND / WIDTH * lowMe, ND / NW); -#endif - - for (u32 s = 1; s < WG; s *= NW) { - fft_NW(u); - w = bcast(w, s); - - chainMul(NW, u, w, 0); - - shufl(WG, lds, u, NW, s, numWG, sb, lowMe); - } - fft_NW(u); -} - -#else - -void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = WIDTH / NW; - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NW) { - fft_NW(u); - tabMul(WG, trig, u, NW, s, lowMe); - shufl(WG, lds, u, NW, s, numWG, sb, lowMe); - } - fft_NW(u); -} - -#endif - - -// New fft_WIDTH that uses more FMA instructions than the old fft_WIDTH. -// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. -// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. -// The downside is sine/cosine cannot be computed with chained multiplies. - -void OVERLOAD new_fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe, int callnum) { - u32 WG = WIDTH / NW; - - // This line mimics shufl -- partition lds - local T2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); - -// Custom code for various WIDTH values - -#if WIDTH == 256 && NW == 4 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=256, NW=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); - - // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - -#elif WIDTH == 512 && NW == 8 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=512, NW=8 - - T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8; // Skip past old FFT_width trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); - shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); - - // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - -#elif WIDTH == 1024 && NW == 4 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=1024, NW=4 - - T preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); - - // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); - shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); - - // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); - -#elif WIDTH == 4096 && NW == 8 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=4K, NW=8 - - T preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); - shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); - shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); - - // Finish third tabMul and perform final fft8. - finish_tabMul8_fft8(WG, trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - -#else - - // Old version - fft_WIDTH(lds, u, trig, numWG, sb, lowMe); - -#endif -} - -// There are two version of new_fft_WIDTH in case we want to try saving some trig values from new_fft_WIDTH1 in LDS memory for later use in new_fft_WIDTH2. -void OVERLOAD new_fft_WIDTH1(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 1); } -void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 2); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe) { T2 dummy; fft_common(lds, u, trig, dummy, numWG, lowMe, 0); } +void OVERLOAD fft_WIDTH1(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe) { T2 dummy; fft_common(lds, u, trig, dummy, numWG, lowMe, 1); } +void OVERLOAD fft_WIDTH2(local T2 *lds, T2 *u, Trig trig, u32 numWG, u32 lowMe) { T2 dummy; fft_common(lds, u, trig, dummy, numWG, lowMe, 2); } #endif @@ -225,167 +36,10 @@ void OVERLOAD new_fft_WIDTH2(local T2 *lds, T2 *u, Trig trig, u32 numWG, const u #if FFT_FP32 -void OVERLOAD fft_NW(F2 *u) { -#if NW == 4 - fft4(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -void OVERLOAD fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = WIDTH / NW; - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NW) { - fft_NW(u); - tabMul(WG, trig, u, NW, s, lowMe); - shufl(WG, lds, u, NW, s, numWG, sb, lowMe); - } - fft_NW(u); -} - -// New fft_WIDTH that uses more FMA instructions than the old fft_WIDTH. -// The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. -// To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. -// The downside is sine/cosine cannot be computed with chained multiplies. - -void OVERLOAD new_fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe, int callnum) { - u32 WG = WIDTH / NW; - - // This line mimics shufl -- partition lds - local F2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); - -// Custom code for various WIDTH values - -#if ENABLE_FP32_VARIANT_2 && WIDTH == 256 && NW == 4 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=256, NW=4 - - F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); - - // Finish third tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - -#elif ENABLE_FP32_VARIANT_2 && WIDTH == 512 && NW == 8 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=512, NW=8 - - F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8; // Skip past old FFT_width trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); - shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); - - // Finish second tabMul and perform final fft8. - finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - -#elif ENABLE_FP32_VARIANT_2 && WIDTH == 1024 && NW == 4 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=1024, NW=4 - - F preloads[6]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*4 + 2*WG*4; // Skip past old FFT_width trig values. Also skip past !save_one_more_mul trig values. - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul4_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft4, partial tabMul, and shufl. - fft4(u); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft4. Do second partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 1, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 4, numWG, lowMe); - shufl(WG, lds, u, NW, 4, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft4. Do third partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 4, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 16, numWG, lowMe); - shufl(WG, lds, u, NW, 16, numWG, sb, lowMe); - - // Finish the third tabMul and perform fourth fft4. Do fourth partial tabMul and shufl. - finish_tabMul4_fft4(WG, trig, preloads, u, 16, numWG, lowMe, 1); - partial_tabMul4(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); - shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); - - // Finish fourth tabMul and perform final fft4. - finish_tabMul4_fft4(WG, trig, preloads, u, 64, numWG, lowMe, 1); - -#elif ENABLE_FP32_VARIANT_2 && WIDTH == 4096 && NW == 8 && FFT_VARIANT_W == 2 - -// Custom code for WIDTH=4K, NW=8 - - F preloads[10]; // Place to store preloaded trig values. We want F64 ops to hide load latencies without creating register pressure. - trig += WG*8; // Skip past old FFT_width trig values to the !save_one_more_mul trig values - - // Preload trig values to hide global memory latencies. As the preloads are used, the next set of trig values are preloaded. - preload_tabMul8_trig(WG, trig, preloads, 1, numWG, lowMe); - - // Do first fft8, partial tabMul, and shufl. - fft8(u); - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 1, numWG, lowMe); - shufl(WG, lds, u, NW, 1, numWG, sb, lowMe); - - // Finish the first tabMul and perform second fft8. Do second partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 1, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 8, numWG, lowMe); - shufl(WG, lds, u, NW, 8, numWG, sb, lowMe); - - // Finish the second tabMul and perform third fft8. Do third partial tabMul and shufl. - finish_tabMul8_fft8(WG, trig, preloads, u, 8, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - partial_tabMul8(WG, partitioned_lds, trig, preloads, u, 64, numWG, lowMe); - shufl(WG, lds, u, NW, 64, numWG, sb, lowMe); - - // Finish third tabMul and perform final fft8. - finish_tabMul8_fft8(WG, trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 - -#else - - // Old version - fft_WIDTH(lds, u, trig, numWG, sb, lowMe); - -#endif -} - -// There are two version of new_fft_WIDTH in case we want to try saving some trig values from new_fft_WIDTH1 in LDS memory for later use in new_fft_WIDTH2. -void OVERLOAD new_fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 1); } -void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, const u32 sb, u32 lowMe) { new_fft_WIDTH(lds, u, trig, numWG, sb, lowMe, 2); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds , u, trig, numWG, lowMe, 0); } +void OVERLOAD fft_WIDTH1(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 1); } +void OVERLOAD fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe, 2); } #endif @@ -396,32 +50,10 @@ void OVERLOAD new_fft_WIDTH2(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, con #if NTT_GF31 -void OVERLOAD fft_NW(GF31 *u) { -#if NW == 4 - fft4(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -void OVERLOAD fft_WIDTH(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = WIDTH / NW; - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NW) { - fft_NW(u); - tabMul(WG, trig, u, NW, s, lowMe); - shufl(WG, lds, u, NW, s, numWG, sb, lowMe); - } - fft_NW(u); -} - -void OVERLOAD new_fft_WIDTH1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } -void OVERLOAD new_fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH1(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif @@ -432,31 +64,9 @@ void OVERLOAD new_fft_WIDTH2(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, #if NTT_GF61 -void OVERLOAD fft_NW(GF61 *u) { -#if NW == 4 - fft4(u); -#elif NW == 8 - fft8(u); -#else -#error NW -#endif -} - -void OVERLOAD fft_WIDTH(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { - u32 WG = WIDTH / NW; - -#if !UNROLL_W - __attribute__((opencl_unroll_hint(1))) -#endif - for (u32 s = 1; s < WG; s *= NW) { - fft_NW(u); - tabMul(WG, trig, u, NW, s, lowMe); - shufl(WG, lds, u, NW, s, numWG, sb, lowMe); - } - fft_NW(u); -} - -void OVERLOAD new_fft_WIDTH1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } -void OVERLOAD new_fft_WIDTH2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, const u32 sb, u32 lowMe) { fft_WIDTH(lds, u, trig, numWG, sb, lowMe); } +// Three versions. fft_WIDTH1 and fft_WIDTH2 are for the two carryFused calls where a future version might save some data from call 1 for use in call 2. +void OVERLOAD fft_WIDTH(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH1(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } +void OVERLOAD fft_WIDTH2(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { fft_common(lds, u, trig, numWG, lowMe); } #endif diff --git a/src/cl/selftest.cl b/src/cl/selftest.cl index 3564f131..ecf5f063 100644 --- a/src/cl/selftest.cl +++ b/src/cl/selftest.cl @@ -1,6 +1,7 @@ // Copyright (C) Mihai Preda #include "base.cl" +#include "math.cl" #include "trig.cl" #include "fft3.cl" #include "fft4.cl" diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 9728edc7..0b308fa0 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -78,41 +78,41 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + fft_HEIGHT1(lds, v, smallTrig, w, 1, me); #else readTailFusedLine(a, p, line1, me); readTailFusedLine(a, q, line2, me); - fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, p, smallTrig, w, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, q, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + fft_HEIGHT1(lds, v, smallTrig, w, 1, me); + fft_HEIGHT1(lds, p, smallTrig, w, 1, me); + fft_HEIGHT1(lds, q, smallTrig, w, 1, me); #endif T2 trig = slowTrig_N(line1 + me * H, ND / NH); if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); T2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrig, w, 1, me); + fft_HEIGHT2(lds, u, smallTrig, w, 1, me); writeTailFusedLine(v, out, memline2, me); writeTailFusedLine(u, out, memline1, me); } @@ -190,41 +190,41 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, aF2, memline1 * SMALL_HEIGHT); read(G_H, NH, q, aF2, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + fft_HEIGHT1(lds, v, smallTrigF2, 1, me); #else readTailFusedLine(aF2, p, line1, me); readTailFusedLine(aF2, q, line2, me); - fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, p, smallTrigF2, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, q, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + fft_HEIGHT1(lds, v, smallTrigF2, 1, me); + fft_HEIGHT1(lds, p, smallTrigF2, 1, me); + fft_HEIGHT1(lds, q, smallTrigF2, 1, me); #endif F2 trig = slowTrig_N(line1 + me * H, ND / NH); if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); F2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrigF2, 1, me); + fft_HEIGHT2(lds, u, smallTrigF2, 1, me); writeTailFusedLine(v, outF2, memline2, me); writeTailFusedLine(u, outF2, memline1, me); } @@ -302,15 +302,15 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a31, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a31, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + fft_HEIGHT1(lds, v, smallTrig31, 1, me); #else readTailFusedLine(a31, p, line1, me); readTailFusedLine(a31, q, line2, me); - fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, p, smallTrig31, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, q, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + fft_HEIGHT1(lds, v, smallTrig31, 1, me); + fft_HEIGHT1(lds, p, smallTrig31, 1, me); + fft_HEIGHT1(lds, q, smallTrig31, 1, me); #endif // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) @@ -333,27 +333,27 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #endif if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); GF31 trig2 = cmul(trig, TAILTGF31); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrig31, 1, me); + fft_HEIGHT2(lds, u, smallTrig31, 1, me); writeTailFusedLine(v, out31, memline2, me); writeTailFusedLine(u, out31, memline1, me); } @@ -431,15 +431,15 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if MUL_LOW read(G_H, NH, p, a61, memline1 * SMALL_HEIGHT); read(G_H, NH, q, a61, memline2 * SMALL_HEIGHT); - fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + fft_HEIGHT1(lds, v, smallTrig61, 1, me); #else readTailFusedLine(a61, p, line1, me); readTailFusedLine(a61, q, line2, me); - fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, p, smallTrig61, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, q, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + fft_HEIGHT1(lds, v, smallTrig61, 1, me); + fft_HEIGHT1(lds, p, smallTrig61, 1, me); + fft_HEIGHT1(lds, q, smallTrig61, 1, me); #endif // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) @@ -462,27 +462,27 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #endif if (line1 == 0) { - reverse(G_H, lds, u + NH/2, true); - reverse(G_H, lds, p + NH/2, true); + reverse(lds, u + NH/2, true); + reverse(lds, p + NH/2, true); pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); GF61 trig2 = cmul(trig, TAILTGF61); - reverse(G_H, lds, v + NH/2, false); - reverse(G_H, lds, q + NH/2, false); + reverse(lds, v + NH/2, false); + reverse(lds, q + NH/2, false); pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { - reverseLine(G_H, lds, v); - reverseLine(G_H, lds, q); + reverseLine(lds, v); + reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data - fft_HEIGHT(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); - fft_HEIGHT(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrig61, 1, me); + fft_HEIGHT2(lds, u, smallTrig61, 1, me); writeTailFusedLine(v, out61, memline2, me); writeTailFusedLine(u, out61, memline1, me); } diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 970baa32..f4b91df3 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -81,12 +81,12 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { T2 trig = slowTrig_N(line + me * H, ND / NH); - new_fft_HEIGHT1(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); + reverse(lds, u + NH/2, !which); - new_fft_HEIGHT1(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), me); } @@ -125,8 +125,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #endif u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT1(lds + zerohack, v, smallTrig + zerohack, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrig + zerohack, w, 1, me); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS == 2 @@ -154,29 +154,29 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). T2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, v, smallTrig, w, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT2(lds, u, smallTrig, w, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrig, w, 1, me); + fft_HEIGHT2(lds, u, smallTrig, w, 1, me); writeTailFusedLine(v, out, memline2, me); writeTailFusedLine(u, out, memline1, me); @@ -242,7 +242,7 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #endif u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT1(lds + zerohack, u, smallTrig + zerohack, w, 2, lowMe); // Compute trig values from scratch. Good on GPUs with high DP throughput. #if TAIL_TRIGS == 2 @@ -285,7 +285,7 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, u, smallTrig, w, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT2(lds, u, smallTrig, w, 2, lowMe); // Write lines u and v writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), lowMe); @@ -365,12 +365,12 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { F2 trig = slowTrig_N(line + me * H, ND / NH); - new_fft_HEIGHT1(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); + reverse(lds, u + NH/2, !which); - new_fft_HEIGHT1(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), me); } @@ -405,8 +405,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(inF2, v, line2, me); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT1(lds + zerohack, v, smallTrigF2 + zerohack, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrigF2 + zerohack, 1, me); // Compute trig values from scratch. Good on GPUs with high FP throughput. #if TAIL_TRIGS32 == 2 @@ -434,29 +434,29 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). F2 trig2 = cmulFancy(trig, TAILT); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, v, smallTrigF2, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT2(lds, u, smallTrigF2, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrigF2, 1, me); + fft_HEIGHT2(lds, u, smallTrigF2, 1, me); writeTailFusedLine(v, outF2, memline2, me); writeTailFusedLine(u, outF2, memline1, me); @@ -518,7 +518,7 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(inF2, u, line, lowMe); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT1(lds + zerohack, u, smallTrigF2 + zerohack, 2, lowMe); // Compute trig values from scratch. Good on GPUs with high FP throughput. #if TAIL_TRIGS32 == 2 @@ -561,7 +561,7 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, u, smallTrigF2, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT2(lds, u, smallTrigF2, 2, lowMe); // Write lines u and v writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), lowMe); @@ -662,12 +662,12 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - new_fft_HEIGHT1(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); + reverse(lds, u + NH/2, !which); - new_fft_HEIGHT2(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, u, smallTrig31, 1, me); writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), me); } @@ -702,8 +702,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in31, v, line2, me); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT1(lds + zerohack, v, smallTrig31 + zerohack, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrig31 + zerohack, 1, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS31 >= 1 @@ -727,29 +727,29 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). GF31 trig2 = cmul(trig, TAILTGF31); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, v, smallTrig31, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT2(lds, u, smallTrig31, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrig31, 1, me); + fft_HEIGHT2(lds, u, smallTrig31, 1, me); writeTailFusedLine(v, out31, memline2, me); writeTailFusedLine(u, out31, memline1, me); @@ -810,7 +810,7 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in31, u, line, lowMe); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT1(lds + zerohack, u, smallTrig31 + zerohack, 2, lowMe); // Do a little bit of memory access and a little bit of math. Good on a Radeon VII. #if TAIL_TRIGS31 >= 1 @@ -849,7 +849,7 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, u, smallTrig31, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT2(lds, u, smallTrig31, 2, lowMe); // Write lines u and v writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), lowMe); @@ -967,12 +967,12 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #endif #endif - new_fft_HEIGHT1(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); - reverse(G_H, lds, u + NH/2, !which); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + reverse(lds, u + NH/2, !which); pairSq(NH/2, u, u + NH/2, trig, !which); - reverse(G_H, lds, u + NH/2, !which); + reverse(lds, u + NH/2, !which); - new_fft_HEIGHT2(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, u, smallTrig61, 1, me); writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), me); } @@ -1007,8 +1007,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in61, v, line2, me); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT1(lds + zerohack, v, smallTrig61 + zerohack, 1, SHUFL_BYTES_H, me); + fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 1, me); + fft_HEIGHT1(lds + zerohack, v, smallTrig61 + zerohack, 1, me); // Do a little bit of memory access and a little bit of math. #if TAIL_TRIGS61 >= 1 @@ -1032,29 +1032,29 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_KERNEL if (line1 == 0) { // Line 0 is special: it pairs with itself, offseted by 1. - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); pairSq(NH/2, u, u + NH/2, trig, true); - reverse(G_H, lds, u + NH/2, true); + reverse(lds, u + NH/2, true); // Line H/2 also pairs with itself (but without offset). GF61 trig2 = cmul(trig, TAILTGF61); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); pairSq(NH/2, v, v + NH/2, trig2, false); - reverse(G_H, lds, v + NH/2, false); + reverse(lds, v + NH/2, false); } else { #else if (1) { #endif - reverseLine(G_H, lds, v); + reverseLine(lds, v); pairSq(NH, u, v, trig, false); - reverseLine(G_H, lds, v); + reverseLine(lds, v); } dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, v, smallTrig61, 1, SHUFL_BYTES_H, me); - new_fft_HEIGHT2(lds, u, smallTrig61, 1, SHUFL_BYTES_H, me); + fft_HEIGHT2(lds, v, smallTrig61, 1, me); + fft_HEIGHT2(lds, u, smallTrig61, 1, me); writeTailFusedLine(v, out61, memline2, me); writeTailFusedLine(u, out61, memline1, me); @@ -1115,7 +1115,7 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { readTailFusedLine(in61, u, line, lowMe); u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; - new_fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT1(lds + zerohack, u, smallTrig61 + zerohack, 2, lowMe); // Do a little bit of memory access and a little bit of math. Good on a Radeon VII. #if TAIL_TRIGS61 >= 1 @@ -1154,7 +1154,7 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data - new_fft_HEIGHT2(lds, u, smallTrig61, 2, SHUFL_BYTES_H, lowMe); + fft_HEIGHT2(lds, u, smallTrig61, 2, lowMe); // Write lines u and v writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), lowMe); diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index 43604852..5fd12dbf 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -32,7 +32,7 @@ #if FFT_FP64 | NTT_GF61 -void OVERLOAD reverse(u32 WG, local T2 *lds2, T2 *u, bool bump) { +void OVERLOAD reverse(local T2 *lds2, T2 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; @@ -81,7 +81,7 @@ void OVERLOAD reverse(u32 WG, local T2 *lds2, T2 *u, bool bump) { } } -void OVERLOAD reverseLine(u32 WG, local T2 *lds, T2 *u) { +void OVERLOAD reverseLine(local T2 *lds, T2 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; @@ -135,76 +135,76 @@ void OVERLOAD reverseLine(u32 WG, local T2 *lds, T2 *u) { void OVERLOAD reverse2(local T2 *lds2, T2 *u) { u32 me = get_local_id(0); - u32 lowMe = me % G_H; + u32 lowMe = me % WG; if (SHUFL_BYTES_H >= 8) { local T2 *lds = lds2; - if (me >= G_H) lds += LDS_BYTES / sizeof(T2); + if (me >= WG) lds += LDS_BYTES / sizeof(T2); // For NH=8, u[0] to u[3] are left unchanged. Write to lds: // u[7]rev u[6]rev u[5]rev u[4]rev // v[7]rev v[6]rev v[5]rev v[4]rev - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i]; } + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i]; } // For NH=8, read from lds into u[i]: // u[4] = u[7]rev v[7]rev // u[5] = u[6]rev v[6]rev // u[6] = u[5]rev v[5]rev // u[7] = u[4]rev v[4]rev - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * G_H + lowMe]; } + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * WG + lowMe]; } } else if (SHUFL_BYTES_H == 4) { local T *lds = (local T *) lds2; - if (me >= G_H) lds += LDS_BYTES / sizeof(T); - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i].x; } - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].x = lds[i * G_H + lowMe]; } - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i].y; } - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].y = lds[i * G_H + lowMe]; } + if (me >= WG) lds += LDS_BYTES / sizeof(T); + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i].x; } + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].x = lds[i * WG + lowMe]; } + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i].y; } + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].y = lds[i * WG + lowMe]; } } } // This is used to reverse the second part of a line, and cross the reversed parts between the halves. void OVERLOAD revCrossLine(local T2* lds2, T2 *u) { u32 me = get_local_id(0); - u32 lowMe = me % G_H; - u32 revLowMe = G_H - 1 - lowMe; + u32 lowMe = me % WG; + u32 revLowMe = WG - 1 - lowMe; if (SHUFL_BYTES_H >= 8) { local T2 *ldsOut = lds2; local T2 *ldsIn = lds2; - if (me < G_H) ldsOut += LDS_BYTES / sizeof(T2); // Crossing LDS halves + if (me < WG) ldsOut += LDS_BYTES / sizeof(T2); // Crossing LDS halves else ldsIn += LDS_BYTES / sizeof(T2); // Staying within LDS halves (just like shufl) bar(); // we need a full bar because we're crossing halves - for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. - for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[G_H * i + lowMe]; } + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[WG * i + lowMe]; } } else if (SHUFL_BYTES_H == 4) { local T *ldsOut = (local T *) lds2; local T *ldsIn = (local T *) lds2; - if (me < G_H) ldsOut += LDS_BYTES / sizeof(T); + if (me < WG) ldsOut += LDS_BYTES / sizeof(T); else ldsIn += LDS_BYTES / sizeof(T); bar(); // we need a full bar because we're crossing halves - for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].x; } + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].x; } bar(); // we need a full bar because we just crossed halves - for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].x = ldsIn[G_H * i + lowMe]; } + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].x = ldsIn[WG * i + lowMe]; } bar(); // we need a full bar because we're crossing halves - for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].y; } + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].y; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. - for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].y = ldsIn[G_H * i + lowMe]; } + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].y = ldsIn[WG * i + lowMe]; } } } #if 0 // Unused // Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. +// The u values are in threads < WG, the v values to reverse in threads >= WG. // Whereas reverseLine leaves u values alone. This reverseLine moves u values around // so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: // u[0] u[1] // Returned in u[0] @@ -218,31 +218,31 @@ void OVERLOAD reverseLine2(local T2 *lds, T2 *u) { // unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT T2 values). // The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT T2 values). - if (G_H > WAVEFRONT) bar(); + if (WG > WAVEFRONT) bar(); // For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG +// That means saving to lds using index: me < WG ? me % WG + i * WG : 8*WG-1 - me % WG - i * WG #if 1 - local T2 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; + local T2 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } lds += me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*WG]; } #else - local T *ldsOut = (local T *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } + local T *ldsOut = (local T *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; + for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*WG] = u[i].y; } local T *ldsIn = (local T *) lds + me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*WG]; u[i].y = ldsIn[NH*2*WG + i * 2*WG]; } #endif } @@ -250,37 +250,37 @@ void OVERLOAD reverseLine2(local T2 *lds, T2 *u) { void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially +// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl. By initially // writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of +// from the lds locations that shufl will use (u values in the upper half of lds memory, v values in the lower half of // lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. #if 1 local T2 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i]; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + lds += (me < WG) ? me % WG : (NH*2)*WG-1 - me % WG; + i32 ldsInc = (me < WG) ? WG : -WG; bar(); for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } #else local T *ldsOut = (local T *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i].x; ldsOut[NH*2*WG + i * 2*WG] = u[i].y; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local T *ldsIn = (local T *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + local T *ldsIn = (local T *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsInc = (me < WG) ? WG : -WG; bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } + for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*WG]; } #endif } @@ -295,7 +295,7 @@ void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { #if FFT_FP32 | NTT_GF31 -void OVERLOAD reverse(u32 WG, local F2 *lds, F2 *u, bool bump) { +void OVERLOAD reverse(local F2 *lds, F2 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; @@ -315,7 +315,7 @@ void OVERLOAD reverse(u32 WG, local F2 *lds, F2 *u, bool bump) { } } -void OVERLOAD reverseLine(u32 WG, local F2 *lds, F2 *u) { +void OVERLOAD reverseLine(local F2 *lds, F2 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; @@ -348,47 +348,47 @@ void OVERLOAD reverseLine(u32 WG, local F2 *lds, F2 *u) { void OVERLOAD reverse2(local F2 *lds, F2 *u) { u32 me = get_local_id(0); - u32 lowMe = me % G_H; + u32 lowMe = me % WG; if (SHUFL_BYTES_H >= 4) { - if (me >= G_H) lds += LDS_BYTES / sizeof(F2); + if (me >= WG) lds += LDS_BYTES / sizeof(F2); // For NH=8, u[0] to u[3] are left unchanged. Write to lds: // u[7]rev u[6]rev u[5]rev u[4]rev // v[7]rev v[6]rev v[5]rev v[4]rev - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * G_H - (me >= G_H ? 1 : 0) - lowMe) % (NH/2 * G_H)] = u[NH/2 + i]; } + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i]; } // For NH=8, read from lds into u[i]: // u[4] = u[7]rev v[7]rev // u[5] = u[6]rev v[6]rev // u[6] = u[5]rev v[5]rev // u[7] = u[4]rev v[4]rev - bar(G_H); - for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * G_H + lowMe]; } + bar(WG); + for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * WG + lowMe]; } } } // This is used to reverse the second part of a line, and cross the reversed parts between the halves. void OVERLOAD revCrossLine(local F2* lds2, F2 *u) { u32 me = get_local_id(0); - u32 lowMe = me % G_H; - u32 revLowMe = G_H - 1 - lowMe; + u32 lowMe = me % WG; + u32 revLowMe = WG - 1 - lowMe; if (SHUFL_BYTES_H >= 4) { local F2 *ldsOut = lds2; local F2 *ldsIn = lds2; - if (me < G_H) ldsOut += LDS_BYTES / sizeof(F2); + if (me < WG) ldsOut += LDS_BYTES / sizeof(F2); else ldsIn += LDS_BYTES / sizeof(F2); bar(); // we need a full bar because we're crossing halves - for (u32 i = 0; i < NH/2; ++i) { ldsOut[G_H * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } + for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. - for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[G_H * i + lowMe]; } + for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[WG * i + lowMe]; } } } #if 0 // Unused // Somewhat similar to reverseLine. -// The u values are in threads < G_H, the v values to reverse in threads >= G_H. +// The u values are in threads < WG, the v values to reverse in threads >= WG. // Whereas reverseLine leaves u values alone. This reverseLine moves u values around // so that pairSq2 can easily operate on pairs. This means for NH = 4, web output: // u[0] u[1] // Returned in u[0] @@ -398,35 +398,35 @@ void OVERLOAD revCrossLine(local F2* lds2, F2 *u) { void OVERLOAD reverseLine2(local F2 *lds, F2 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with shufl2. Failure to do so would require an +// NOTE: It is important that this routine use lds memory in coordination with shufl. Failure to do so would require an // unqualified bar() call here. Specifically, the u values are stored in the upper half of lds memory (SMALL_HEIGHT F2 values). // The v values are stored in the lower half of lds memory (the next SMALL_HEIGHT F2 values). - if (G_H > WAVEFRONT) bar(); + if (WG > WAVEFRONT) bar(); // For NH=4, the lds indices (where to write each incoming u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H -// That means saving to lds using index: me < G_H ? me % G_H + i * G_H : 8*G_H-1 - me % G_H - i * G_H +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG +// That means saving to lds using index: me < WG ? me % WG + i * WG : 8*WG-1 - me % WG - i * WG #if 1 - local F2 *ldsOut = lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; + local F2 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } lds += me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*WG]; } #else - local F *ldsOut = (local F *) lds + (me < G_H ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsOutInc = (me < G_H) ? G_H : -G_H; - for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*G_H] = u[i].y; } + local F *ldsOut = (local F *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsOutInc = (me < WG) ? WG : -WG; + for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*WG] = u[i].y; } local F *ldsIn = (local F *) lds + me; bar(); - for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*G_H]; u[i].y = ldsIn[NH*2*G_H + i * 2*G_H]; } + for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*WG]; u[i].y = ldsIn[NH*2*WG + i * 2*WG]; } #endif } @@ -434,37 +434,37 @@ void OVERLOAD reverseLine2(local F2 *lds, F2 *u) { void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { u32 me = get_local_id(0); -// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl2. By initially +// NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl. By initially // writing to the lds locations that reverseLine2 read from we do not need an initial bar() call here. Also, by reading -// from the lds locations that shufl2 will use (u values in the upper half of lds memory, v values in the lower half of +// from the lds locations that shufl will use (u values in the upper half of lds memory, v values in the lower half of // lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. #if 1 local F2 *ldsOut = lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i]; } + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i]; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - lds += (me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H; - i32 ldsInc = (me < G_H) ? G_H : -G_H; +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + lds += (me < WG) ? me % WG : (NH*2)*WG-1 - me % WG; + i32 ldsInc = (me < WG) ? WG : -WG; bar(); for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } #else local F *ldsOut = (local F *) lds + me; - for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*G_H] = u[i].x; ldsOut[NH*2*G_H + i * 2*G_H] = u[i].y; } + for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i].x; ldsOut[NH*2*WG + i * 2*WG] = u[i].y; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: -// 0..GH-1 +0*G_H GH-1..0 +7*G_H -// 0..GH-1 +1*G_H GH-1..0 +6*G_H -// 0..GH-1 +2*G_H GH-1..0 +5*G_H -// 0..GH-1 +3*G_H GH-1..0 +4*G_H - local F *ldsIn = (local F *) lds + ((me < G_H) ? me % G_H : (NH*2)*G_H-1 - me % G_H); - i32 ldsInc = (me < G_H) ? G_H : -G_H; +// 0..GH-1 +0*WG GH-1..0 +7*WG +// 0..GH-1 +1*WG GH-1..0 +6*WG +// 0..GH-1 +2*WG GH-1..0 +5*WG +// 0..GH-1 +3*WG GH-1..0 +4*WG + local F *ldsIn = (local F *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); + i32 ldsInc = (me < WG) ? WG : -WG; bar(); - for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*G_H]; } + for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*WG]; } #endif } @@ -479,12 +479,12 @@ void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { #if NTT_GF31 -void OVERLOAD reverse(u32 WG, local GF31 *lds, GF31 *u, bool bump) { - reverse(WG, (local F2 *) lds, (F2 *) u, bump); +void OVERLOAD reverse(local GF31 *lds, GF31 *u, bool bump) { + reverse((local F2 *) lds, (F2 *) u, bump); } -void OVERLOAD reverseLine(u32 WG, local GF31 *lds, GF31 *u) { - reverseLine(WG, (local F2 *) lds, (F2 *) u); +void OVERLOAD reverseLine(local GF31 *lds, GF31 *u) { + reverseLine((local F2 *) lds, (F2 *) u); } void OVERLOAD reverse2(local GF31 *lds, GF31 *u) { @@ -516,12 +516,12 @@ void OVERLOAD unreverseLine2(local GF31 *lds, GF31 *u) { #if NTT_GF61 -void OVERLOAD reverse(u32 WG, local GF61 *lds, GF61 *u, bool bump) { - reverse(WG, (local T2 *) lds, (T2 *) u, bump); +void OVERLOAD reverse(local GF61 *lds, GF61 *u, bool bump) { + reverse((local T2 *) lds, (T2 *) u, bump); } -void OVERLOAD reverseLine(u32 WG, local GF61 *lds, GF61 *u) { - reverseLine(WG, (local T2 *) lds, (T2 *) u); +void OVERLOAD reverseLine(local GF61 *lds, GF61 *u) { + reverseLine((local T2 *) lds, (T2 *) u); } void OVERLOAD reverse2(local GF61 *lds, GF61 *u) { From 33c72324b39593553e067a46aa5302527584f54b Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Apr 2026 19:58:54 +0000 Subject: [PATCH 043/214] Micro-optimization. Use optional_add for make_Z61. Unlikely to be any faster. --- src/cl/math.cl | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index 8408f397..06881fac 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -145,7 +145,7 @@ i32 select32(i32 a, i32 b, i32 c) { #endif } -// Optionally add a value if first arg is negative. +// Optionally add a constant value if first arg is negative. i32 optional_add(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" @@ -158,7 +158,7 @@ i32 optional_add(i32 a, const i32 b) { return a; } -// Optionally subtract a value if first arg is negative. +// Optionally subtract a constant value if first arg is negative. i32 optional_sub(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" @@ -171,7 +171,7 @@ i32 optional_sub(i32 a, const i32 b) { return a; } -// Optionally subtract a value if first arg is greater than value. +// Optionally subtract a constant value if first arg is greater than value. i32 optional_mod(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (too small a gain to measure??) __asm("{.reg .pred %%p;\n\t" @@ -184,7 +184,33 @@ i32 optional_mod(i32 a, const i32 b) { return a; } -// Optionally subtract c from a if a >= b. +// Optionally add a constant value if first arg is negative. +i64 optional_addM61(i64 a) { +#if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.lt.s64 %%p, %0, 0;\n\t" // a < 0 + " @%%p add.s64 %0, %0, 2305843009213693951;}" // if (a < 0) a = a + M61 + : "+l"(a)); +#else + if (a < 0) a = a + 2305843009213693951; +#endif + return a; +} + +// Optionally add a constant value if first arg is negative. +i64 optional_add(i64 a, const i64 b) { +#if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher + __asm("{.reg .pred %%p;\n\t" + " setp.lt.s64 %%p, %0, 0;\n\t" // a < 0 + " @%%p add.s64 %0, %0, %1;}" // if (a < 0) a = a + b + : "+l"(a) : "l"(b)); // It would be nice if there was an asm constraint to indicate a 64-bit constant +#else + if (a < 0) a = a + b; +#endif + return a; +} + +// Optionally subtract constant c from a if a >= b. u64 OVERLOAD optional_sub(u64 a, const u64 b, const u64 c) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" @@ -208,7 +234,7 @@ i64 OVERLOAD optional_sub(i64 a, const i64 b, const i64 c) { return a; } -// Optionally subtract c from a if hi32(a) >= b. +// Optionally subtract constant c from a if hi32(a) >= b. u64 OVERLOAD optional_sub(u64 a, const u32 b, const u64 c) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" @@ -819,7 +845,7 @@ GF31 OVERLOAD foo(GF31 a) { return foo2(a, a); } #define M61 ((((Z61) 1) << 61) - 1) Z61 OVERLOAD make_Z61(i32 a) { return (Z61) (a < 0 ? (i64) a + M61 : (i64) a); } // Handles all values of a -Z61 OVERLOAD make_Z61(i64 a) { return (Z61) (a < 0 ? a + M61 : a); } // a must be in range of -M61 .. M61-1 +Z61 OVERLOAD make_Z61(i64 a) { return (Z61) optional_addM61(a); } // a must be in range of -M61 .. M61-1 Z61 OVERLOAD make_Z61(u32 a) { return (Z61) (a); } // Handles all values of a Z61 OVERLOAD make_Z61(u64 a) { return (Z61) (a); } // a must be in range of 0 .. M61-1 From fb454fbbdbbcfa25e4e221626359097419885fcf Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Apr 2026 21:36:58 +0000 Subject: [PATCH 044/214] Fixed bug in MODM31=2 compilation --- src/cl/math.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index 06881fac..c6a53e3c 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -683,7 +683,7 @@ Z31 OVERLOAD modM31(i32 a) { return (a & M31) + (a >> 31); } Z31 OVERLOAD modM31(Z31 a) { i32 alt = a + 0x80000001; return select32(a, a, alt); } // Assumes a is not 0xFFFFFFFF (which would return 0x80000000) Z31 OVERLOAD modM31(i32 a) { i32 alt = a - 0x80000001; return select32(a, a, alt); } // Assumes a is not 0x80000000 (which would return 0xFFFFFFFF) #else -Z31 OVERLOAD modM31(Z31 a) { return optional_add(a, 0x80000001); } // Assumes a is not 0xFFFFFFFF (which would return 0x80000000) +Z31 OVERLOAD modM31(Z31 a) { return optional_add((i32)a, 0x80000001); } // Assumes a is not 0xFFFFFFFF (which would return 0x80000000) Z31 OVERLOAD modM31(i32 a) { return optional_sub(a, 0x80000001); } // Assumes a is not 0x80000000 (which would return 0xFFFFFFFF) #endif From ac45f89ebb935df208e01a30374da8232527b402 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 24 Apr 2026 00:07:29 +0000 Subject: [PATCH 045/214] Don't compile the squareZero kernels when SINGLE_WIDE is set --- src/cl/tailsquare.cl | 8 ++++++++ src/cl/tailutil.cl | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index f4b91df3..10105194 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -52,6 +52,7 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { @@ -89,6 +90,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { fft_HEIGHT1(lds, u, smallTrig, w, 1, me); writeTailFusedLine(u, out, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE @@ -340,6 +342,7 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { @@ -373,6 +376,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { fft_HEIGHT1(lds, u, smallTrigF2, 1, me); writeTailFusedLine(u, outF2, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE @@ -619,6 +623,7 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { @@ -670,6 +675,7 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { fft_HEIGHT2(lds, u, smallTrig31, 1, me); writeTailFusedLine(u, out31, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE @@ -924,6 +930,7 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { } } +#if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { @@ -975,6 +982,7 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { fft_HEIGHT2(lds, u, smallTrig61, 1, me); writeTailFusedLine(u, out61, transPos(line, MIDDLE, WIDTH), me); } +#endif #if SINGLE_WIDE diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index 5fd12dbf..1af416e7 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -25,8 +25,8 @@ #if !defined(TAIL_KERNELS) #define TAIL_KERNELS 2 // Default is double-wide tailSquare with two kernels #endif -#define SINGLE_WIDE TAIL_KERNELS < 2 // Old single-wide tailSquare vs. new double-wide tailSquare -#define SINGLE_KERNEL (TAIL_KERNELS & 1) == 0 // TailSquare uses a single kernel vs. two kernels +#define SINGLE_WIDE (TAIL_KERNELS < 2) // Old single-wide tailSquare vs. new double-wide tailSquare +#define SINGLE_KERNEL ((TAIL_KERNELS & 1) == 0) // TailSquare uses a single kernel vs. two kernels // 64-bit implementations of reverse routines From cf9a6cb978fe37b85f507fb50d9db29ab851e37d Mon Sep 17 00:00:00 2001 From: george Date: Fri, 24 Apr 2026 00:29:13 +0000 Subject: [PATCH 046/214] Added OVERLOAD keyword to optional_add for the AMD compiler. --- src/cl/math.cl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index c6a53e3c..3c0c0dfd 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -146,7 +146,7 @@ i32 select32(i32 a, i32 b, i32 c) { } // Optionally add a constant value if first arg is negative. -i32 optional_add(i32 a, const i32 b) { +i32 OVERLOAD optional_add(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" " setp.lt.s32 %%p, %0, 0;\n\t" // a < 0 @@ -159,7 +159,7 @@ i32 optional_add(i32 a, const i32 b) { } // Optionally subtract a constant value if first arg is negative. -i32 optional_sub(i32 a, const i32 b) { +i32 OVERLOAD optional_sub(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" " setp.lt.s32 %%p, %0, 0;\n\t" // a < 0 @@ -172,7 +172,7 @@ i32 optional_sub(i32 a, const i32 b) { } // Optionally subtract a constant value if first arg is greater than value. -i32 optional_mod(i32 a, const i32 b) { +i32 OVERLOAD optional_mod(i32 a, const i32 b) { #if HAS_PTX >= 100 // setp/sub instruction requires sm_10 support or higher // Not faster on 5xxx GPUs (too small a gain to measure??) __asm("{.reg .pred %%p;\n\t" " setp.ge.s32 %%p, %0, %1;\n\t" // a > b @@ -185,7 +185,7 @@ i32 optional_mod(i32 a, const i32 b) { } // Optionally add a constant value if first arg is negative. -i64 optional_addM61(i64 a) { +i64 OVERLOAD optional_addM61(i64 a) { #if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" " setp.lt.s64 %%p, %0, 0;\n\t" // a < 0 @@ -198,7 +198,7 @@ i64 optional_addM61(i64 a) { } // Optionally add a constant value if first arg is negative. -i64 optional_add(i64 a, const i64 b) { +i64 OVERLOAD optional_add(i64 a, const i64 b) { #if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher __asm("{.reg .pred %%p;\n\t" " setp.lt.s64 %%p, %0, 0;\n\t" // a < 0 From 4c7d159e8140e8314a1cb5ac55e4ddb7e82c30ed Mon Sep 17 00:00:00 2001 From: george Date: Fri, 24 Apr 2026 19:36:36 +0000 Subject: [PATCH 047/214] Replaced LDS padding code with better code. All address calculations allow calculating a fixed base before the loop and simple base+offset accesses within the loop. Less generated code, fewer registers used. --- src/cl/fftbase.cl | 156 +++++++++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 63 deletions(-) diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 498d18f0..daed1895 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -5,13 +5,13 @@ // Calculate the LDS bytes used by shufl #if LDSPAD && SHUFL_BYTES == 16 && RADIX == 8 -#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 72 / 64) +#define LDS_BYTES ((WG * RADIX + 7) * SHUFL_BYTES) #elif LDSPAD && SHUFL_BYTES == 16 && RADIX == 4 -#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 20 / 16) +#define LDS_BYTES ((WG * RADIX + 12) * SHUFL_BYTES) #elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 8 -#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 72 / 64) +#define LDS_BYTES ((WG * RADIX + 56) * SHUFL_BYTES) #elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 4 -#define LDS_BYTES ((WG * RADIX * SHUFL_BYTES) * 20 / 16) +#define LDS_BYTES ((WG * RADIX + 12) * SHUFL_BYTES) #else #define LDS_BYTES (WG * RADIX * SHUFL_BYTES) #endif @@ -50,50 +50,56 @@ void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { #if LDSPAD // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Pad after every 8th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 8, 72..., 16... lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad 1 value every row to eliminate bank conflicts. if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe * 9 + i] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe & 7) * (WG + 1) + (lowMe / 8) * 8 + i] = u[i]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 32 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } return; } // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // No padding of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and - // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. - // We can however save a bar() by writing to same locations that previous shufl wrote to. - if (!force_default && f == 8 && RADIX == 8) { - for (u32 i = 0; i < RADIX; ++i) { lds[(i * WG + lowMe) * 9 / 8] = u[i]; } + // No padding of LDS blocks is needed to eliminate bank conflicts! Groups of 8 threads are already in separate LDS banks. + // We could however save a bar() by writing to same locations that the previous shufl wrote to. + if (0 && f == 8 && RADIX == 8) { + // for (u32 i = 0; i < RADIX; ++i) { lds[something] = u[i]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[((lowMe & ~7) + i) * 9 + (lowMe & 7)]; } + //for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[something]; } return; } // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Pad after every 8th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1, 65..., 16... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad 1 value every row to eliminate bank conflicts. if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 9 / 8] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + (lowMe & 1) * 4 + i] = u[i]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 9 / 8]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 4 + (lowMe / 32) * 8 + ((lowMe / 8) & 3) * (WG + 1) + (lowMe & 7)]; } return; } // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 - // Pad 4 values after every 16th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16... 4.. lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 16... 1.. output[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * (WG + 4) + i * 4 + (lowMe & 3)] = u[i]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } return; } #endif @@ -166,67 +172,81 @@ void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { #if LDSPAD // Special case first n == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Pad after every 16th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].x; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].y; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } return; } // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Pad 8 values after every 64 values to eliminate bank conflicts. + // Pad 8 values after every row to eliminate bank conflicts. if (!force_default && f == 8 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].x; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 72 + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u32 idx = (i * WG + lowMe); u[i].x = lds[idx + idx / 64 * 8]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i].y; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 72 + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u32 idx = (i * WG + lowMe); u[i].y = lds[idx + idx / 64 * 8]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } return; } // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Pad after every 16th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].x; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) * 17 / 16]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].y; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) * 17 / 16]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } return; } // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 - // Pad 4 values after every 16th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].x; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i].x = lds[idx + idx / 16 * 4]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 20 + i * 4 + (lowMe & 3))] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i].y = lds[idx + idx / 16 * 4]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } return; } #endif @@ -373,13 +393,16 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { #if LDSPAD // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Pad after every 16th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. if (!force_default && f == 1 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) * 17 / 16] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } return; } @@ -389,34 +412,41 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { // Pad 8 values after every 64 values to eliminate bank conflicts. if (!force_default && f == 8 && RADIX == 8) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 8 * 72 + i * 8 + (lowMe & 7)] = u[i]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i]; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i]; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 72 + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u32 idx = (i * WG + lowMe); u[i] = lds[idx + idx / 64 * 8]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } return; } // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Pad after every 16th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. if (!force_default && f == 1 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) * 17 / 16] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) * 17 / 16]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } return; } // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 - // Pad 4 values after every 16th value to eliminate bank conflicts. + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. if (!force_default && f == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * 20 + i * 4 + (lowMe & 3)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u32 idx = i * WG + lowMe; u[i] = lds[idx + idx / 16 * 4]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } return; } #endif From df67d21f5edceb33f6dd5682a38475752f56130f Mon Sep 17 00:00:00 2001 From: george Date: Sun, 26 Apr 2026 04:42:04 +0000 Subject: [PATCH 048/214] Cleaner way to specify an explicit CUDA launch_bounds rather than a register count --- src/Gpu.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 446b9638..28f568de 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -671,12 +671,13 @@ string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { use_override = "REGMO61"; break; } + // Get the optional override register count int override_regs = args.value(use_override, 0); - // Allow command line to set CUDA launch_bounds rather than explicit maximum register count - if (override_regs && args.value("REGLB", 0)) return string("-DCUDA_MIN_BLOCKS=") + to_string(override_regs) + " "; - // Return the maximum register count + // If a specified override is small, use the count as a CUDA launch_bounds rather than a maximum register count + if (override_regs && (override_regs > 0 && override_regs <= 16)) return string("-DCUDA_MIN_BLOCKS=") + to_string(override_regs) + " "; + // If specified, override the default maximum register count if (override_regs) regs = override_regs; - // Sometimes the results using default launch_bounds without setting an explicit maxrrregcount can't be beat + // Sometimes the results using CUDA compiler's default launch_bounds without setting an explicit launch bounds or maxrrregcount can't be beat if (regs == -1) return string(""); // Format an explicit register count setting return string("--maxrregcount=") + to_string(regs) + " "; From 321c9c65606aca340e600be875f15055c914892f Mon Sep 17 00:00:00 2001 From: george Date: Tue, 28 Apr 2026 01:18:46 +0000 Subject: [PATCH 049/214] CUDA compiler generates slow mul.lo.s64 instructions for u64 * u64. PTX macros added to eliminate this. --- src/cl/carry.cl | 18 +++++++++--------- src/cl/carryfused.cl | 22 +++++++++++----------- src/cl/fftp.cl | 18 +++++++++--------- src/cl/math.cl | 37 ++++++++++++++++++++++++++++++++++--- 4 files changed, 63 insertions(+), 32 deletions(-) diff --git a/src/cl/carry.cl b/src/cl/carry.cl index 88a0b2b2..e8dc75a0 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -129,7 +129,7 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF31) in, u32 posROE, P(CarryABM) carryOut, P #define combo_counter combo.b const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -203,7 +203,7 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF61) in, u32 posROE, P(CarryABM) carryOut, P #define combo_counter combo.b const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -278,7 +278,7 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define combo_counter combo.b const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -357,7 +357,7 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define combo_counter combo.b const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -436,7 +436,7 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define combo_counter combo.b const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -519,9 +519,9 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, P(u #define m61_combo_counter m61_combo.b const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -613,9 +613,9 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define m61_combo_counter m61_combo.b const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 5d169d41..db304570 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -162,7 +162,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. u32 word_index = (lowMe * H + line) * 2; - u32 frac_bits = word_index * FRAC_BPW_HI + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 frac_bits = mul3264(word_index, FRAC_BPW_HI) + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); // Apply the inverse weights and carry propagate pairs to generate the output carries @@ -352,7 +352,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. u32 word_index = (lowMe * H + line) * 2; - u32 frac_bits = word_index * FRAC_BPW_HI + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 frac_bits = mul3264(word_index, FRAC_BPW_HI) + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); // Apply the inverse weights and carry propagate pairs to generate the output carries @@ -549,7 +549,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -766,7 +766,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 61; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -996,7 +996,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1244,7 +1244,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1492,7 +1492,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 61; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1736,12 +1736,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m31_combo_bigstep = ((G_W * H * 2 - 1) * m31_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m31_weight_shift = m31_weight_shift % 31; u64 m31_starting_combo_counter = m31_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m61_combo_bigstep = ((G_W * H * 2 - 1) * m61_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m61_weight_shift = m61_weight_shift % 61; u64 m61_starting_combo_counter = m61_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -2005,12 +2005,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m31_combo_bigstep = ((G_W * H * 2 - 1) * m31_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m31_weight_shift = m31_weight_shift % 31; u64 m31_starting_combo_counter = m31_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m61_combo_bigstep = ((G_W * H * 2 - 1) * m61_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m61_weight_shift = m61_weight_shift % 61; u64 m61_starting_combo_counter = m61_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index b263d8ab..43ffef8e 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -95,7 +95,7 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { @@ -152,7 +152,7 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 61; for (u32 i = 0; i < NW; ++i) { @@ -215,7 +215,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { @@ -285,7 +285,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { @@ -355,7 +355,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = word_index * combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; weight_shift = weight_shift % 61; for (u32 i = 0; i < NW; ++i) { @@ -429,11 +429,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m31_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m31_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m31_weight_shift = m31_weight_shift % 31; const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m61_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m61_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m61_weight_shift = m61_weight_shift % 61; for (u32 i = 0; i < NW; ++i) { @@ -518,11 +518,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m31_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m31_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = word_index * m31_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m31_weight_shift = m31_weight_shift % 31; const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; const u64 m61_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m61_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = word_index * m61_combo_step + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; m61_weight_shift = m61_weight_shift % 61; for (u32 i = 0; i < NW; ++i) { diff --git a/src/cl/math.cl b/src/cl/math.cl index 3c0c0dfd..87766ac6 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -112,9 +112,8 @@ i128 OVERLOAD sub(i128 a, i64 b) { i128 val; val.x = a.x - (__int128)b; return v u128 OVERLOAD make_u128(u64 hi, u64 lo) { u128 val; val.x = ((unsigned __int128)hi << 64) | lo; return val; } u64 u128_lo64(u128 val) { return val.x; } u64 u128_hi64(u128 val) { return val.x >> 64; } -u128 mul64(u64 a, u64 b) { u128 val; val.x = (unsigned __int128)a * (unsigned __int128)b; return val; } u128 OVERLOAD add(u128 a, u128 b) { u128 val; val.x = a.x + b.x; return val; } -#else // UNTESTED! The mul64 macro causes clang to hang! +#else // UNTESTED! typedef struct { i64 hi64; u64 lo64; } i128; typedef struct { u64 hi64; u64 lo64; } u128; i128 OVERLOAD make_i128(i64 hi, u64 lo) { i128 val; val.hi64 = hi; val.lo64 = lo; return val; } @@ -130,7 +129,6 @@ i128 OVERLOAD sub(i128 a, i64 b) { i128 val; val.lo64 = a.lo64 - (u64)b; val.hi6 u128 OVERLOAD make_u128(u64 hi, u64 lo) { u128 val; val.hi64 = hi; val.lo64 = lo; return val; } u64 u128_lo64(u128 val) { return val.lo64; } u64 u128_hi64(u128 val) { return val.hi64; } -u128 mul64(u64 a, u64 b) { u128 val; val.lo64 = a * b; val.hi64 = mul_hi(a, b); return val; } u128 OVERLOAD add(u128 a, u128 b) { u128 val; val.lo64 = a.lo64 + b.lo64; val.hi64 = a.hi64 + b.hi64 + (val.lo64 < a.lo64); return val; } #endif @@ -260,6 +258,12 @@ i64 OVERLOAD optional_sub(i64 a, const i32 b, const i64 c) { // Multiply and add primitives +u64 OVERLOAD mul3264(u32 a, u64 b) { // Work around nVidia generating a slow mul.lo.s64 instruction for a * b + u32 blo = lo32(b); + u32 bhi = hi32(b); + return make_u64(mul_hi(a, blo) + a * bhi, a * blo); +} + u64 OVERLOAD mad32(u32 a, u32 b, u32 c) { #if HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Same speed on TitanV, any gain may be too small to measure u32 reslo, reshi; @@ -282,6 +286,33 @@ u64 OVERLOAD mad32(u32 a, u32 b, u64 c) { #endif } +u128 OVERLOAD mul64(u64 a, u64 b) { +#if HAS_PTX >= 100 // mul instruction requires sm_10 support or higher + u64 reslo, reshi; + __asm("mul.lo.u64 %0, %2, %3;\n\t" + "mul.hi.u64 %1, %2, %3;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b)); + return make_u128(reshi, reslo); +#elif ENABLE_ALT_MUL64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher. This is slower than the above. + uint2 a2 = as_uint2(a); + uint2 b2 = as_uint2(b); + uint2 rlo2, rhi2; + __asm("mul.lo.u32 %0, %4, %6;\n\t" + "mul.hi.u32 %1, %4, %6;\n\t" + "mul.lo.u32 %2, %5, %7;\n\t" + "mad.lo.cc.u32 %1, %5, %6, %1;\n\t" + "madc.hi.cc.u32 %2, %5, %6, %2;\n\t" + "madc.hi.u32 %3, %5, %7, 0;\n\t" + "mad.lo.cc.u32 %1, %4, %7, %1;\n\t" + "madc.hi.cc.u32 %2, %4, %7, %2;\n\t" + "addc.u32 %3, %3, 0;" + : "=r"(rlo2.x), "=r"(rlo2.y), "=r"(rhi2.x), "=r"(rhi2.y) + : "r"(a2.x), "r"(a2.y), "r"(b2.x), "r"(b2.y)); + return make_u128((u64)as_ulong(rhi2), (u64)as_ulong(rlo2)); +#else // May cause clang to hang! + return make_u128(mul_hi(a, b), a * b); // This generates a mul.lo.s64 PTX instruction which is much slower! +#endif +} + u128 OVERLOAD mad64(u64 a, u64 b, u64 c) { #if ENABLE_MAD64 && HAS_PTX >= 200 // mad instruction requires sm_20 support or higher // Slower on TitanV and mobile 4070, don't understand why u64 reslo, reshi; From f9e9f04986c81c5ca68d8a40b3dd63e89fc78de6 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 28 Apr 2026 21:29:20 +0000 Subject: [PATCH 050/214] Enforce 32KB shared memory maximum. Micro GF31 math optimization. --- src/Gpu.cpp | 5 +++-- src/cl/math.cl | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 28f568de..78fdbd7e 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -310,11 +310,12 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< if (k == "PAD") pad_size = atoi(v.c_str()); } - // Maximum WMUL is 32KB / (WIDTH * SHUFL_BYTES_W) + // Maximum WMUL is 32KB / (WIDTH * SHUFL_BYTES_W). If using the 32KB maximum, LDS padding must be disabled. { u32 shufl_bytes_w = args.value("SHUFL_BYTES_W", 8); u32 max_wmul = 32768 / (fft.shape.width * shufl_bytes_w); - if (wmul > max_wmul) wmul = max_wmul; + if (wmul > max_wmul) { wmul = max_wmul; config["WMUL"] = to_string(wmul); } + if (fft.shape.width * shufl_bytes_w * wmul >= 32768) { config["LDSPAD_W"] = to_string(0); } } string defines = toDefine(config); diff --git a/src/cl/math.cl b/src/cl/math.cl index 87766ac6..a0278128 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -737,7 +737,7 @@ GF31 OVERLOAD neg(GF31 a) { return U2(neg(a.x), neg(a.y)); } Z31 OVERLOAD add(Z31 a, Z31 b) { return modM31(a + b); } GF31 OVERLOAD add(GF31 a, GF31 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } -Z31 OVERLOAD sub(Z31 a, Z31 b) { i32 t = a - b; return (t & M31) + (t >> 31); } +Z31 OVERLOAD sub(Z31 a, Z31 b) { return modM31((i32)(a - b)); } GF31 OVERLOAD sub(GF31 a, GF31 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } Z31 OVERLOAD make_Z31(i32 a) { return (Z31) (a < 0 ? a + M31 : a); } // Handles signed values of a From 1fd99292d5b30c1e8230d822d84441d1edcb685e Mon Sep 17 00:00:00 2001 From: george Date: Wed, 29 Apr 2026 04:09:00 +0000 Subject: [PATCH 051/214] Added log messages when changing settings to accommodate 32KB LDS limit. --- src/Gpu.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 78fdbd7e..693d6cf4 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -314,8 +314,15 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< { u32 shufl_bytes_w = args.value("SHUFL_BYTES_W", 8); u32 max_wmul = 32768 / (fft.shape.width * shufl_bytes_w); - if (wmul > max_wmul) { wmul = max_wmul; config["WMUL"] = to_string(wmul); } - if (fft.shape.width * shufl_bytes_w * wmul >= 32768) { config["LDSPAD_W"] = to_string(0); } + if (wmul > max_wmul) { + wmul = max_wmul; + config["WMUL"] = to_string(wmul); + log("Local shared memory limit of 32KB exceeded. Changing to WMUL=%d\n", wmul); + } + if (fft.shape.width * shufl_bytes_w * wmul >= 32768) { + log("Local shared memory limit of 32KB exceeded. Changing to LDSPAD_W=0\n"); + config["LDSPAD_W"] = to_string(0); + } } string defines = toDefine(config); From d075f476a73ccaef1965d4daeccd952cc584bbd8 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 29 Apr 2026 19:23:12 +0000 Subject: [PATCH 052/214] Changed comments on why MUL64 inline is needed --- src/cl/math.cl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index a0278128..ca51597a 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -258,7 +258,7 @@ i64 OVERLOAD optional_sub(i64 a, const i32 b, const i64 c) { // Multiply and add primitives -u64 OVERLOAD mul3264(u32 a, u64 b) { // Work around nVidia generating a slow mul.lo.s64 instruction for a * b +u64 OVERLOAD mul3264(u32 a, u64 b) { // 3 32-bit multiplies (instead of 4 for a u64 * u64 multiply) u32 blo = lo32(b); u32 bhi = hi32(b); return make_u64(mul_hi(a, blo) + a * bhi, a * blo); @@ -286,8 +286,11 @@ u64 OVERLOAD mad32(u32 a, u32 b, u64 c) { #endif } +// Multiply to u64s creating a u128. This inline mysteriously speeds up PRPLL by 2%. Verified on RTX 3xxx through RTX 5xxx GPUs, both CUDA 12 and CUDA 13. +// The generated PTX code is nearly identical with DISABLE_MUL64. It must somehow trigger different/weird PTXAS optimization decisions. +// Future CUDA releases may make this inline unnecessary. u128 OVERLOAD mul64(u64 a, u64 b) { -#if HAS_PTX >= 100 // mul instruction requires sm_10 support or higher +#if !DISABLE_MUL64 && HAS_PTX >= 100 // mul instruction requires sm_10 support or higher u64 reslo, reshi; __asm("mul.lo.u64 %0, %2, %3;\n\t" "mul.hi.u64 %1, %2, %3;" : "=l"(reslo), "=l"(reshi) : "l"(a), "l"(b)); @@ -309,7 +312,7 @@ u128 OVERLOAD mul64(u64 a, u64 b) { : "r"(a2.x), "r"(a2.y), "r"(b2.x), "r"(b2.y)); return make_u128((u64)as_ulong(rhi2), (u64)as_ulong(rlo2)); #else // May cause clang to hang! - return make_u128(mul_hi(a, b), a * b); // This generates a mul.lo.s64 PTX instruction which is much slower! + return make_u128(mul_hi(a, b), a * b); #endif } From b9663e6f1ed71bee3cefd440cfa2c3a8839c77c3 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 30 Apr 2026 01:17:02 +0000 Subject: [PATCH 053/214] Minor change that should speed up kernel compilations --- src/Gpu.cpp | 120 +++++++++++++++++++++++++++++------------------ src/Gpu.h | 2 + src/cl/middle.cl | 8 ++-- 3 files changed, 81 insertions(+), 49 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 693d6cf4..d31ba518 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -370,10 +370,6 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Send the FFT/NTT type and booleans that enable/disable code for each possible FP and NTT defines += toDefine("FFT_TYPE", (int) fft.shape.fft_type); - defines += toDefine("FFT_FP64", (int) fft.FFT_FP64); - defines += toDefine("FFT_FP32", (int) fft.FFT_FP32); - defines += toDefine("NTT_GF31", (int) fft.NTT_GF31); - defines += toDefine("NTT_GF61", (int) fft.NTT_GF61); defines += toDefine("WordSize", fft.WordSize); // When using multiple NTT primes or hybrid FFT/NTT, each FFT/NTT prime's data buffer and trig values are combined into one buffer. @@ -694,6 +690,40 @@ string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { #endif } +// Kernels are compiled one at a time, but OpenCL source files contain multiple kernels. This routine set the #defines necessary so that only one kernel is compiled. +// While not strictly necessary, startup speed will be a bit faster if we do less compilations. +string Gpu::kernelDefines(enum WHICH_KERNEL_TYPE which_kernel) { + string defines; + // Determine the kernel specific #defines + switch (which_kernel) { + case KFP: // FP64 or FP32 kernel + defines += toDefine("FFT_FP64", (int) fft.FFT_FP64); + defines += toDefine("FFT_FP32", (int) fft.FFT_FP32); + defines += toDefine("NTT_GF31", 0); + defines += toDefine("NTT_GF61", 0); + break; + case K31: // GF1 kernel + defines += toDefine("FFT_FP64", 0); + defines += toDefine("FFT_FP32", 0); + defines += toDefine("NTT_GF31", (int) fft.NTT_GF31); + defines += toDefine("NTT_GF61", 0); + break; + case K61: // GF61 kernel + defines += toDefine("FFT_FP64", 0); + defines += toDefine("FFT_FP32", 0); + defines += toDefine("NTT_GF31", 0); + defines += toDefine("NTT_GF61", (int) fft.NTT_GF61); + break; + case KALL: // Kernels, like carryFused, that need all defines set properly + defines += toDefine("FFT_FP64", (int) fft.FFT_FP64); + defines += toDefine("FFT_FP32", (int) fft.FFT_FP32); + defines += toDefine("NTT_GF31", (int) fft.NTT_GF31); + defines += toDefine("NTT_GF61", (int) fft.NTT_GF61); + break; + } + return defines + " "; +} + #define ROE_SIZE 100000 #define CARRY_SIZE 100000 @@ -715,58 +745,58 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& #define K(name, ...) name(#name, &compiler, profile.make(#name), queue, __VA_ARGS__) - K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDIN).c_str()), - K(kfftHin, "ffthin.cl", "fftHin", hN / nH), - K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2), + K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), (kernelDefines(KFP) + numCudaRegisters(MIDIN)).c_str()), + K(kfftHin, "ffthin.cl", "fftHin", hN / nH, kernelDefines(KFP).c_str()), + K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2, kernelDefines(KFP).c_str()), K(ktailSquare, "tailsquare.cl", "tailSquare", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, numCudaRegisters(TAIL).c_str()), // Single-wide tailSquare with one kernel - K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2), - K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDOUT).c_str()), - K(kfftW, "fftw.cl", "fftW", hN / nW), - - K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDIN31).c_str()), - K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH), - K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2), + hN / nH / 2, (kernelDefines(KFP) + numCudaRegisters(TAIL)).c_str()), // Single-wide tailSquare with one kernel + K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2, kernelDefines(KFP).c_str()), + K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, (kernelDefines(KFP) + "-DMUL_LOW=1").c_str()), + K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), (kernelDefines(KFP) + numCudaRegisters(MIDOUT)).c_str()), + K(kfftW, "fftw.cl", "fftW", hN / nW, kernelDefines(KFP).c_str()), + + K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), (kernelDefines(K31) + numCudaRegisters(MIDIN31)).c_str()), + K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH, kernelDefines(K31).c_str()), + K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2, kernelDefines(K31).c_str()), K(ktailSquareGF31, "tailsquare.cl", "tailSquareGF31", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, numCudaRegisters(TAIL31).c_str()), // Single-wide tailSquare with one kernel - K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2), - K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDOUT31).c_str()), - K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW), - - K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDIN61).c_str()), - K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH), - K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2), + hN / nH / 2, (kernelDefines(K31) + numCudaRegisters(TAIL31)).c_str()), // Single-wide tailSquare with one kernel + K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, kernelDefines(K31).c_str()), + K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, (kernelDefines(K31) + "-DMUL_LOW=1").c_str()), + K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), (kernelDefines(K31) + numCudaRegisters(MIDOUT31)).c_str()), + K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW, kernelDefines(K31).c_str()), + + K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), (kernelDefines(K61) + numCudaRegisters(MIDIN61)).c_str()), + K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH, kernelDefines(K61).c_str()), + K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2, kernelDefines(K61).c_str()), K(ktailSquareGF61, "tailsquare.cl", "tailSquareGF61", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, numCudaRegisters(TAIL61).c_str()), // Single-wide tailSquare with one kernel - K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2), - K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, "-DMUL_LOW=1"), - K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), numCudaRegisters(MIDOUT61).c_str()), - K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW), - - K(kfftP, "fftp.cl", "fftP", hN / nW), - K(kCarryA, "carry.cl", "carry", hN / CARRY_LEN), - K(kCarryAROE, "carry.cl", "carry", hN / CARRY_LEN, "-DROE=1"), - K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1"), - K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, "-DMUL3=1 -DROE=1"), - K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, "-DLL=1"), - K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, numCudaRegisters(CARRYFUSED).c_str()), - K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DROE=1").c_str()), - K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DMUL3=1").c_str()), - K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1").c_str()), - K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (numCudaRegisters(CARRYFUSED) + "-DLL=1").c_str()), - - K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN), + hN / nH / 2, (kernelDefines(K61) + numCudaRegisters(TAIL61)).c_str()), // Single-wide tailSquare with one kernel + K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, kernelDefines(K61).c_str()), + K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, (kernelDefines(K61) + "-DMUL_LOW=1").c_str()), + K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), (kernelDefines(K61) + numCudaRegisters(MIDOUT61)).c_str()), + K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW, kernelDefines(K61).c_str()), + + K(kfftP, "fftp.cl", "fftP", hN / nW, kernelDefines(KALL).c_str()), + K(kCarryA, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL).c_str()), + K(kCarryAROE, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DROE=1").c_str()), + K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DMUL3=1").c_str()), + K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DMUL3=1 -DROE=1").c_str()), + K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DLL=1").c_str()), + K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED)).c_str()), + K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DROE=1").c_str()), + K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1").c_str()), + K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1").c_str()), + K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DLL=1").c_str()), + + K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN, kernelDefines(KALL).c_str()), // 64 K(transpIn, "transpose.cl", "transposeIn", hN / 64), @@ -778,7 +808,7 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& K(kernIsEqual, "etc.cl", "isEqual", 256 * 256, "-DISEQUAL=1"), K(sum64, "etc.cl", "sum64", 256 * 256, "-DSUM64=1"), - K(testTrig, "selftest.cl", "testTrig", 256 * 256), + K(testTrig, "selftest.cl", "testTrig", 256 * 256), K(testFFT4, "selftest.cl", "testFFT4", 256), K(testFFT14, "selftest.cl", "testFFT14", 256), K(testFFT15, "selftest.cl", "testFFT15", 256), diff --git a/src/Gpu.h b/src/Gpu.h index 2c0554a0..a9ee95de 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -339,6 +339,8 @@ class Gpu { void doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors); enum WHICH_KERNEL {CARRYFUSED=0, MIDIN=1, MIDIN31=2, MIDIN61=3, TAIL=4, TAIL31=5, TAIL61=6, MIDOUT=7, MIDOUT31=8, MIDOUT61=9}; string numCudaRegisters(enum WHICH_KERNEL which_kernel); + enum WHICH_KERNEL_TYPE {KFP=0, K31=1, K61=2, KALL=3}; + string kernelDefines(enum WHICH_KERNEL_TYPE which_kernel); }; // Compute the size of an FFT/NTT data buffer depending on the FFT/NTT float/prime. Size is returned in units of sizeof(double). diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 41809b48..0236141f 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -875,8 +875,8 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) //#define SIZEM32 (MIDDLE * SIZEB + (1 - (MIDDLE & 1)) * 16) // Pad 128 bytes if MIDDLE is even // Place middle rows after all width rows #define SIZEBLK32 (SMALL_HEIGHT + 0) // No pad needed when swizzling -#define SIZEW32 (16 * SIZEBLK + 16) // Pad 128 bytes -#define SIZEM32 (WIDTH / 16 * SIZEW + 16) // Pad 128 bytes +#define SIZEW32 (16 * SIZEBLK32 + 16) // Pad 128 bytes +#define SIZEM32 (WIDTH / 16 * SIZEW32 + 16) // Pad 128 bytes #define SWIZ32(a,m) ((m) ^ (a)) // Swizzle 16 rows (remove "^ (a)" to turn swizzling off) #else // AMD friendly padding // Place middle rows after first 16 rows @@ -885,8 +885,8 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) //#define SIZEW32 (MIDDLE * SIZEM + (1 - (MIDDLE & 1)) * 16) // Pad 128 bytes if MIDDLE is even // Place middle rows after all width rows #define SIZEBLK32 (SMALL_HEIGHT + 0) // No pad needed when swizzling -#define SIZEW32 (16 * SIZEBLK + 16) // Pad 128 bytes -#define SIZEM32 (WIDTH / 16 * SIZEW + 0) // Pad 0 bytes +#define SIZEW32 (16 * SIZEBLK32 + 16) // Pad 128 bytes +#define SIZEM32 (WIDTH / 16 * SIZEW32 + 0) // Pad 0 bytes #define SWIZ32(a,m) ((m) ^ (a)) // Swizzle 16 rows (remove "^ (a)" to turn swizzling off) #endif From 644fde44c4f8a1e0a8266daa13d2294aec2e1d34 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 30 Apr 2026 17:53:17 +0000 Subject: [PATCH 054/214] Enhanced csqa (complex-square-and-add) to use mad64 via weakMulAdd. Tailsquare now computes 2ab as (a+b)^2 - a^2 - b^2. Since a^2 and b^2 are already computed, this replaces a cmul with a csq which saves a wideMul. --- src/cl/math.cl | 194 +++++++++++-------------------------------- src/cl/tailsquare.cl | 43 ++++++---- 2 files changed, 75 insertions(+), 162 deletions(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index ca51597a..6e260c6b 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -875,7 +875,6 @@ GF31 OVERLOAD foo(GF31 a) { return foo2(a, a); } #if NTT_GF61 -// bits in reduced mod M. #define M61 ((((Z61) 1) << 61) - 1) Z61 OVERLOAD make_Z61(i32 a) { return (Z61) (a < 0 ? (i64) a + M61 : (i64) a); } // Handles all values of a @@ -883,120 +882,10 @@ Z61 OVERLOAD make_Z61(i64 a) { return (Z61) optional_addM61(a); } Z61 OVERLOAD make_Z61(u32 a) { return (Z61) (a); } // Handles all values of a Z61 OVERLOAD make_Z61(u64 a) { return (Z61) (a); } // a must be in range of 0 .. M61-1 -#if 0 // Slower version that keeps results strictly in the range 0 .. M61-1 - -u64 OVERLOAD get_Z61(Z61 a) { return a; } // Get value in range 0 to M61-1 -i64 OVERLOAD get_balanced_Z61(Z61 a) { return (hi32(a) & 0xF0000000) ? (i64) a - (i64) M61 : (i64) a; } // Get balanced value in range -M61/2 to M61/2 - -Z61 OVERLOAD neg(Z61 a) { return a == 0 ? 0 : M61 - a; } // GWBUG: Examine all callers to see if neg call can be avoided -GF61 OVERLOAD neg(GF61 a) { return U2(neg(a.x), neg(a.y)); } - -Z61 OVERLOAD add(Z61 a, Z61 b) { Z61 t = a + b; Z61 m = t - M61; return (m & 0x8000000000000000ULL) ? t : m; } -//Z61 OVERLOAD add(Z61 a, Z61 b) { Z61 t = a + b; Z61 m = t - M61; return t < m ? t : m; } // Slower on TitanV -//Z61 OVERLOAD add(Z61 a, Z61 b) { Z61 t = a + b; return t - (t >= M61 ? M61 : 0); } // Slower on TitanV -GF61 OVERLOAD add(GF61 a, GF61 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } - -Z61 OVERLOAD sub(Z61 a, Z61 b) { Z61 t = a - b; return t + (((i64) t >> 63) & 0x1FFFFFFFFFFFFFFFULL); } -//Z61 OVERLOAD sub(Z61 a, Z61 b) { Z61 t = a - b; Z61 p = t + M61; return (t & 0x8000000000000000ULL) ? p : t; } // Better??? -//Z61 OVERLOAD sub(Z61 a, Z61 b) { Z61 t = a - b; return t + (t >= M61 ? M61 : 0); } // Slower on TitanV -// BETTER???: t = a - b; carry_mask = sbb x, x; (generates 32 bits of 0 or 1; return t + make_carry_mask_64_bits -GF61 OVERLOAD sub(GF61 a, GF61 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } - -// Assumes k reduced mod 61. -Z61 OVERLOAD shl(Z61 a, u32 k) { return ((a << k) + (a >> (61 - k))) & M61; } //GWBUG: Make sure & M61 operates on just one u32 -GF61 OVERLOAD shl(GF61 a, u32 k) { return U2(shl(a.x, k), shl(a.y, k)); } -Z61 OVERLOAD shr(Z61 a, u32 k) { return ((a >> k) + (a << (61 - k))) & M61; } //GWBUG: Make sure & M61 operates on just one u32. & M61 not needed? -GF61 OVERLOAD shr(GF61 a, u32 k) { return U2(shr(a.x, k), shr(a.y, k)); } - -ulong2 wideMul(u64 ab, u64 cd) { - u128 r = mul64(ab, cd); - return U2(u128_lo64(r), u128_hi64(r)); -} - -Z61 OVERLOAD mul(Z61 a, Z61 b) { - ulong2 ab = wideMul(a, b); - u64 lo = ab.x, hi = ab.y; - u64 lo61 = lo & M61, hi61 = (hi << 3) + (lo >> 61); - return add(lo61, hi61); -} - -Z61 OVERLOAD fma(Z61 a, Z61 b, Z61 c) { return add(mul(a, b), c); } // GWBUG: Can we do better? - -// Multiply by 2 -Z61 OVERLOAD mul2(Z61 a) { return ((a + a) + (a >> 60)) & M61; } // GWBUG: Make sure "+ a>>60" does an add to lower u32 without a followup adc. -GF61 OVERLOAD mul2(GF61 a) { return U2(mul2(a.x), mul2(a.y)); } - -// Return conjugate of a -GF61 OVERLOAD conjugate(GF61 a) { return U2(a.x, neg(a.y)); } - -// Complex square. input, output 61 bits. Uses (a + i*b)^2 == ((a+b)*(a-b) + i*2*a*b). -GF61 OVERLOAD csq(GF61 a) { return U2(mul(add(a.x, a.y), sub(a.x, a.y)), mul2(mul(a.x, a.y))); } //GWBUG: Probably faster to double a.y and have a mul that takes non-normalized inputs - -// a^2 + c -GF61 OVERLOAD csqa(GF61 a, GF61 c) { return add(csq(a), c); } // GWBUG: inline csq so we only "mod" after adding c?? Find a way to use fma instructions - -// Complex mul -//GF61 OVERLOAD cmul(GF61 a, GF61 b) { return U2(sub(mul(a.x, b.x), mul(a.y, b.y)), add(mul(a.x, b.y), mul(a.y, b.x)));} // GWBUG: Is a 3 multiply complex mul faster? See above -GF61 OVERLOAD cmul(GF61 a, GF61 b) { - Z61 k1 = mul(b.x, add(a.x, a.y)); - Z61 k2 = mul(a.x, sub(b.y, b.x)); - Z61 k3 = mul(a.y, add(b.y, b.x)); - return U2(sub(k1, k3), add(k1, k2)); -} - -// mul with (0, 1). (twiddle of tau/4, sqrt(-1) aka "i"). -GF61 OVERLOAD mul_t4(GF61 a) { return U2(neg(a.y), a.x); } // GWBUG: Can caller use a version that does not negate real? - -// mul with (-2^30, -2^30). (twiddle of tau/8 aka sqrt(i)). Note: 2 * (+/-2^30)^2 == 1 (mod M61). -GF61 OVERLOAD mul_t8(GF61 a) { return shl(U2(sub(a.y, a.x), neg(add(a.x, a.y))), 30); } // GWBUG: Can caller use a version that does not negate real? - -// mul with (2^30, -2^30). (twiddle of 3*tau/8). -GF61 OVERLOAD mul_3t8(GF61 a) { return shl(U2(add(a.x, a.y), sub(a.y, a.x)), 30); } - -// Return a+b and a-b -void OVERLOAD X2_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); *b = sub(t, *b); } - -// Same as X2(a, conjugate(b)) -void OVERLOAD X2conjb_internal(GF61 *a, GF61 *b) { GF61 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } - -// Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } - -// Same as X2(a, b), b = mul_t8(b) -void OVERLOAD X2_mul_t8_internal(GF61 *a, GF61 *b) { X2(*a, *b); *b = mul_t8(*b); } - -// Same as X2(a, b), b = mul_3t8(b) -void OVERLOAD X2_mul_3t8_internal(GF61 *a, GF61 *b) { X2(*a, *b); *b = mul_3t8(*b); } - -// Same as X2(a, b), b = conjugate(b) -void OVERLOAD X2_conjb_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); b->x = sub(t.x, b->x); b->y = sub(b->y, t.y); } - -void OVERLOAD SWAP_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = *b; *b = t; } - -GF61 OVERLOAD addsub(GF61 a) { return U2(add(a.x, a.y), sub(a.x, a.y)); } -GF61 OVERLOAD foo2(GF61 a, GF61 b) { a = addsub(a); b = addsub(b); return addsub(U2(mul(RE(a), RE(b)), mul(IM(a), IM(b)))); } -GF61 OVERLOAD foo(GF61 a) { return foo2(a, a); } - -// The following routines can be used to reduce mod M61 operations (in the other Z61 implementations). -// Caller must track how many M61s need to be added to make positive values for subtractions. -// In function names, "q" stands for quick, "s" stands for slow (i.e. does mod). -// These functions are untested with this strict Z61 implementation. Callers need to eliminate all uses of + or - operators. - -Z61 OVERLOAD modM61(Z61 a) { return a; } -GF61 OVERLOAD modM61(GF61 a) { return a; } -Z61 OVERLOAD neg(Z61 a, u32 m61_count) { return neg(a); } -GF61 OVERLOAD neg(GF61 a, u32 m61_count) { return neg(a); } - - // Philosophy: This Z61/GF61 implementation uses faster, sloppier mod M61 reduction where the end result is in the range 0..M61+epsilon. -// This implementation also handles subtractions by adding enough M61s to make a value positive. This allows us to always deal with positive -// intermediate results. An alternative implementation is to have Z61 be an i64 (costs us a precious bit of precision) and is surprisingly slower (at least on TitanV) because -// mod(a - b), where the mod routinue uses a signed right shift is slower than -// mod(a + (M61*2 - b)) where the mod routine uses an unsigned shift right. +// This implementation also handles subtractions by adding enough M61s to make a value positive. This allows us to always deal with positive intermediate results. // However, a long string of subtracts (example, fft8 does 3 subtracts before mod M61 will be better off using the quick routines and negative intermediate results). -// The mul routine (and obviously csq and cmul) must use only positive values as __int128 multiply is very slow. - -#elif 1 // Faster version that keeps results in the range 0 .. M61+epsilon +// The mul routine (and csq and cmul) must use only positive values as __int128 multiply is very slow. u64 OVERLOAD get_Z61(Z61 a) { Z61 m = a - M61; return (m & 0x8000000000000000ULL) ? a : m; } // Get value in range 0 to M61-1 i64 OVERLOAD get_balanced_Z61(Z61 a) { return (hi32(a) >= 0x10000000) ? (i64)(a - M61) : (i64)a; } // Get balanced value in range -M61/2 to M61/2 @@ -1038,7 +927,7 @@ Z61 OVERLOAD weakMul(Z61 a, Z61 b, const u32 a_m61_count, const u32 b_m61_count) ulong2 ab = wideMul(a, b); u64 lo = ab.x, hi = ab.y; u64 lo61 = lo & M61; // Max value is M61 - if ((a_m61_count - 1) * (b_m61_count - 1) <= 4) { + if ((a_m61_count - 1) * (b_m61_count - 1) <= 6) { hi = (hi << 3) + (lo >> 61); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61 + epsilon return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon } else { @@ -1046,10 +935,34 @@ Z61 OVERLOAD weakMul(Z61 a, Z61 b, const u32 a_m61_count, const u32 b_m61_count) return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon } } +Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u64 c, const u32 a_m61_count, const u32 b_m61_count) { + u128 ab = mad64(a, b, c); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61^2 + epsilon + u64 lo = u128_lo64(ab), hi = u128_hi64(ab); + u64 lo61 = lo & M61; // Max value is M61 + if ((a_m61_count - 1) * (b_m61_count - 1) <= 6) { + hi = (hi << 3) + (lo >> 61); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61 + epsilon + return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon + } else { + u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 + return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + } +} +Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u128 c, const u32 a_m61_count, const u32 b_m61_count) { // Max c value assumed to be 2*M61^2+epsilon + u128 ab = mad64(a, b, c); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61^2 + epsilon + u64 lo = u128_lo64(ab), hi = u128_hi64(ab); + u64 lo61 = lo & M61; // Max value is M61 + if ((a_m61_count - 1) * (b_m61_count - 1) + 2 <= 6) { + hi = (hi << 3) + (lo >> 61); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61 + epsilon + return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 3) * M61 + epsilon + } else { + u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 + return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + } +} Z61 OVERLOAD mul(Z61 a, Z61 b) { return modM61(weakMul(a, b, 2, 2)); } -Z61 OVERLOAD fma(Z61 a, Z61 b, Z61 c) { return modM61(weakMul(a, b, 2, 2) + c); } // GWBUG: Can we do better? +Z61 OVERLOAD fma(Z61 a, Z61 b, Z61 c) { return modM61(weakMulAdd(a, b, c, 2, 2)); } // Multiply by 2 Z61 OVERLOAD mul2(Z61 a) { return add(a, a); } @@ -1059,46 +972,38 @@ GF61 OVERLOAD mul2(GF61 a) { return U2(mul2(a.x), mul2(a.y)); } GF61 OVERLOAD conjugate(GF61 a) { return U2(a.x, neg(a.y)); } // Complex square. Uses (a + i*b)^2 == ((a+b)*(a-b) + i*2*a*b). -GF61 OVERLOAD csqq(GF61 a, const u32 m61_count) { - if (m61_count > 4) return csqq(modM61(a), 2); - Z61 re = weakMul(a.x + a.y, a.x + neg(a.y, m61_count), 2 * m61_count - 1, 2 * m61_count); - Z61 im = weakMul(a.x + a.x, a.y, 2 * m61_count - 1, m61_count); +GF61 OVERLOAD csqq(GF61 a, const u32 x_m61_count, const u32 y_m61_count) { + if (x_m61_count + y_m61_count >= 9) return csqq(modM61(a), 2, 2); + Z61 re = weakMul(a.x + a.y, a.x + neg(a.y, y_m61_count), x_m61_count + y_m61_count - 1, x_m61_count + y_m61_count); + Z61 im = (x_m61_count <= y_m61_count) ? weakMul(a.x + a.x, a.y, x_m61_count + x_m61_count - 1, y_m61_count) : + weakMul(a.x, a.y + a.y, x_m61_count, y_m61_count + y_m61_count - 1); return U2(re, im); } -GF61 OVERLOAD csqs(GF61 a, const u32 m61_count) { return modM61(csqq(a, m61_count)); } -GF61 OVERLOAD csq(GF61 a) { return csqs(a, 2); } +GF61 OVERLOAD csqq(GF61 a, const u32 m61_count) { return csqq(a, m61_count, m61_count); } +GF61 OVERLOAD csq(GF61 a, const u32 x_m61_count, const u32 y_m61_count) { return modM61(csqq(a, x_m61_count, y_m61_count)); } +GF61 OVERLOAD csq(GF61 a, const u32 m61_count) { return csq(a, m61_count, m61_count); } +GF61 OVERLOAD csq(GF61 a) { return csq(a, 2); } // a^2 + c -GF61 OVERLOAD csqa(GF61 a, GF61 c) { return U2(modM61(weakMul(a.x + a.y, a.x + neg(a.y, 2), 3, 4) + c.x), modM61(weakMul(a.x + a.x, a.y, 3, 2) + c.y)); } +GF61 OVERLOAD csqaq(GF61 a, GF61 c, const u32 x_m61_count, const u32 y_m61_count) { + if (x_m61_count + y_m61_count >= 9) return csqaq(modM61(a), c, 2, 2); + Z61 re = weakMulAdd(a.x + a.y, a.x + neg(a.y, y_m61_count), c.x, x_m61_count + y_m61_count - 1, x_m61_count + y_m61_count); + Z61 im = (x_m61_count <= y_m61_count) ? weakMulAdd(a.x + a.x, a.y, c.y, x_m61_count + x_m61_count - 1, y_m61_count) : + weakMulAdd(a.x, a.y + a.y, c.y, x_m61_count, y_m61_count + y_m61_count - 1); + return U2(re, im); +} +GF61 OVERLOAD csqaq(GF61 a, GF61 c, const u32 m61_count) { return csqaq(a, c, m61_count, m61_count); } +GF61 OVERLOAD csqa(GF61 a, GF61 c, const u32 x_m61_count, const u32 y_m61_count) { return modM61(csqaq(a, c, x_m61_count, y_m61_count)); } +GF61 OVERLOAD csqa(GF61 a, GF61 c, const u32 m61_count) { return csqa(a, c, m61_count, m61_count); } +GF61 OVERLOAD csqa(GF61 a, GF61 c) { return csqa(a, c, 2); } // Complex mul -#if 0 -GF61 OVERLOAD cmul(GF61 a, GF61 b) { // Use 3-epsilon extra bits in u64 - Z61 k1 = weakMul(b.x, a.x + a.y, 2, 3); // max value is 3*M61+epsilon - Z61 k2 = weakMul(a.x, b.y + neg(b.x, 2), 2, 3); // max value is 3*M61+epsilon - Z61 k3 = weakMul(a.y, b.y + b.x, 2, 3); // max value is 3*M61+epsilon - return U2(modM61(k1 + neg(k3, 4)), modM61(k1 + k2)); -} -#else -Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u128 c, const u32 a_m61_count, const u32 b_m61_count) { // Max c value assumed to be 2*M61^2+epsilon - u128 ab = mad64(a, b, c); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61^2 + epsilon - u64 lo = u128_lo64(ab), hi = u128_hi64(ab); - u64 lo61 = lo & M61; // Max value is M61 - if ((a_m61_count - 1) * (b_m61_count - 1) + 2 <= 6) { - hi = (hi << 3) + (lo >> 61); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61 + epsilon - return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 3) * M61 + epsilon - } else { - u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 - return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon - } -} GF61 OVERLOAD cmul(GF61 a, GF61 b) { u128 k1 = mul64(b.x, a.x + a.y); // max value is 2*M61^2+epsilon Z61 k1k2 = weakMulAdd(a.x, b.y + neg(b.x, 2), k1, 2, 4); // max value is 6*M61+epsilon Z61 k1k3 = weakMulAdd(a.y, neg(b.y + b.x, 3), k1, 2, 4); // max value is 6*M61+epsilon return U2(modM61(k1k3), modM61(k1k2)); } -#endif // Square a root of unity complex number (the second version may be faster if the compiler optimizes the u128 squaring). //GF61 OVERLOAD csqTrig(GF61 a) { Z61 two_ay = a.y + a.y; return U2(modM61(1 + weakMul(two_ay, neg(a.y, 2))), mul(a.x, two_ay)); } @@ -1158,6 +1063,7 @@ GF61 OVERLOAD subiq(GF61 a, GF61 b) { return U2(a.x + b.y, a.y - b.x); } void OVERLOAD X2q(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; *b = t - *b; } void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; t.x = t.x - b->x; b->x = b->y - t.y; b->y = t.x; } +void OVERLOAD X2qconjb(GF61 *a, GF61 *b) { GF61 t = *a; a->x += b->x; a->y -= b->y; b->x = t.x - b->x; b->y = t.y + b->y; } void OVERLOAD X2q_conjb(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } GF61 OVERLOAD mul_t8q(GF61 a, const u32 m61_count) { return shl(U2(m61_count * M61 + (a.y - a.x), m61_count * M61 - (a.x + a.y)), 30); } @@ -1170,5 +1076,3 @@ GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count) { if (m61_count) { a.x += m61 GF61 OVERLOAD modM61q(GF61 a, const u32 m61_count_x, const u32 m61_count_y) { if (m61_count_x) a.x += m61_count_x * M61; if (m61_count_y) a.y += m61_count_y * M61; return modM61(a); } #endif - -#endif diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 10105194..e707d138 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -874,35 +874,44 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_type) { GF61 a = *pa, b = *pb; - GF61 a2, b2t2, c, d; + GF61 a2, b2, b2t2, ab, addin, c, d; + + X2qconjb(&a, &b); // X2(a, conjugate(b)). a.x range is 0..2+, a.y range is -1-..1+, b.x range is -1-..1+, b.y range is 0..2+ + a.y += 2*M61; // a range is 0..2+ / 1-..3+ + b.x += 2*M61; // b range is 1-..3+ / 0..2+ + + a2 = csqq(a, 3, 4); // a2 = a^2, a2 range is 0..2+ + b2 = csq(b, 4, 3); // b2 = b^2, b2 range is 0..1+ + + ab = addq(a, b); // Compute 2ab as (a + b)^2 - a^2 - b^2. ab range is 1-..5+ + addin = neg(addq(a2, b2), 4); // add this into the csq of a+b, addin in range 0..4 + d = csqa(ab, addin, 6); // d = 2ab, range is 0..1+ + + b2t2 = cmul(b2, t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ - X2conjb(a, b); // X2(a, conjugate(b)) - a2 = csqq(a, 2); // a2 = a^2, a2.x range is 0..2+, a2.y range is 0..3+ - b2t2 = cmul(csq(b), t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ - d = cmul(a, b); d = d + d; // d = 2ab, d range is 0..2+ if (t_squared_type == 0) { // mul t_squared by 1 - c = subq(a2, b2t2); // c.x range is -1..2+, c.y range is -1-..3+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is -1-..5+, d.x range is -3-..2+, d.y range is -3-..3+ + c = subq(a2, b2t2); // c range is -1-..2+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c range is -1-..3+, d.x range is -1-..2+, d.y range is -2-..1+ c = modM61q(c, 2); - d = modM61q(d, 4); + d = modM61q(d, 3); } if (t_squared_type == 1) { // mul t_squared by i - c = subiq(a2, b2t2); // c.x range is 0..3+, c.y range is -1-..3+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is -1..5+, d.x range is -2-..3+, d.y range is -3-..3+ + c = subiq(a2, b2t2); // c.x range is 0..3+, c.y range is -1-..2+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..4+, c.y range is -1..3+, d.x range is -1-..3+, d.y range is -2-..1+ c = modM61q(c, 0, 2); - d = modM61q(d, 4); + d = modM61q(d, 2, 3); } if (t_squared_type == 2) { // mul t_squared by -1 - c = addq(a2, b2t2); // c.x range is 0..3+, c.y range is 0..4+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is 0..6+, d.x range is -2-..3+, d.y range is -4-..2+ + c = addq(a2, b2t2); // c range is 0..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c range is 0..4+, d.x range is -1-..3+, d.y range is -3-..1+ c = modM61q(c, 0); - d = modM61q(d, 3, 5); + d = modM61q(d, 2, 4); } if (t_squared_type == 3) { // mul t_squared by -i - c = addiq(a2, b2t2); // c.x range is -1-..2+, c.y range is 0..4+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is 0..6+, d.x range is -3-..2+, d.y range is -4-..2+ + c = addiq(a2, b2t2); // c.x range is -1-..2+, c.y range is 0..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..3+, c.y range is 0..4+, d.x range is -1-..2+, d.y range is -3-..1+ c = modM61q(c, 2, 0); - d = modM61q(d, 4, 5); + d = modM61q(d, 2, 4); } *pa = SWAP_XY(c), *pb = SWAP_XY(d); } From 590c4d17d4e1f5bd47c686b5ca8e9cc5add5714a Mon Sep 17 00:00:00 2001 From: george Date: Thu, 30 Apr 2026 18:21:37 +0000 Subject: [PATCH 055/214] Eliminate a cmul in the little-used GF61 onePairMul routine. --- src/cl/tailmul.cl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 0b308fa0..bcce141d 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -372,11 +372,13 @@ void OVERLOAD onePairMul(GF61* pa, GF61* pb, GF61* pc, GF61* pd, GF61 t_squared) X2conjb(a, b); X2conjb(c, d); - GF61 e = subq(cmul(a, c), cmul(cmul(b, d), t_squared)); // Range is -1-..1+ - GF61 f = addq(cmul(b, c), cmul(a, d)); // Range is 0..2+ - X2q_conjb(&e, &f); // e range is -1..3+, f.x range is -3-..1+, f.y range is -1-..3+ - e = modM61q(e, 2); - f = modM61q(f, 4, 2); + GF61 ac = cmul(a, c); + GF61 bd = cmul(b, d); + GF61 e = subq(ac, cmul(bd, t_squared)); // Range is -1-..1+ + GF61 f = subq(subq(cmul(add(a, b), add(c, d)), ac), bd); // Compute bc + ad. Range is -2-..1+ + X2q_conjb(&e, &f); // e range is -3..2+, f.x range is -2-..3+, f.y range is -3-..2+ + e = modM61q(e, 4); + f = modM61q(f, 3, 4); *pa = SWAP_XY(e), *pb = SWAP_XY(f); } From a40510cdf487b49541e93e6813354f2cfbf21c56 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 30 Apr 2026 18:45:13 +0000 Subject: [PATCH 056/214] Save one cmul in GF31 onePairMul --- src/cl/tailmul.cl | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index bcce141d..decc0e61 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -240,13 +240,12 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { void OVERLOAD onePairMul(GF31* pa, GF31* pb, GF31* pc, GF31* pd, GF31 t_squared) { GF31 a = *pa, b = *pb, c = *pc, d = *pd; - X2conjb(a, b); X2conjb(c, d); - - *pa = sub(cmul(a, c), cmul(cmul(b, d), t_squared)); - *pb = add(cmul(b, c), cmul(a, d)); - + GF31 ac = cmul(a, c); + GF31 bd = cmul(b, d); + *pa = sub(ac, cmul(bd, t_squared)); + *pb = sub(sub(cmul(add(a, b), add(c, d)), ac), bd); X2_conjb(*pa, *pb); *pa = SWAP_XY(*pa), *pb = SWAP_XY(*pb); } @@ -369,14 +368,13 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { void OVERLOAD onePairMul(GF61* pa, GF61* pb, GF61* pc, GF61* pd, GF61 t_squared) { GF61 a = *pa, b = *pb, c = *pc, d = *pd; - X2conjb(a, b); X2conjb(c, d); GF61 ac = cmul(a, c); GF61 bd = cmul(b, d); GF61 e = subq(ac, cmul(bd, t_squared)); // Range is -1-..1+ GF61 f = subq(subq(cmul(add(a, b), add(c, d)), ac), bd); // Compute bc + ad. Range is -2-..1+ - X2q_conjb(&e, &f); // e range is -3..2+, f.x range is -2-..3+, f.y range is -3-..2+ + X2q_conjb(&e, &f); // e range is -3-..2+, f.x range is -2-..3+, f.y range is -3-..2+ e = modM61q(e, 4); f = modM61q(f, 3, 4); *pa = SWAP_XY(e), *pb = SWAP_XY(f); From 0d6809e078573c7548a25d887b883342974b6ba7 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 30 Apr 2026 19:04:55 +0000 Subject: [PATCH 057/214] Neatened GF31 onePairSquare --- src/cl/tailsquare.cl | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index e707d138..a18a42d4 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -584,17 +584,18 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { void OVERLOAD onePairSq(GF31* pa, GF31* pb, GF31 t_squared, const u32 t_squared_type) { GF31 a = *pa, b = *pb; - GF31 c, d; + GF31 b2t2, c, d; X2conjb(a, b); - if (t_squared_type == 0) // mul t_squared by 1 - c = csq_sub(a, cmul(csq(b), t_squared)); // a^2 - (b^2 * t_squared) - if (t_squared_type == 1) // mul t_squared by i - c = csq_subi(a, cmul(csq(b), t_squared)); // a^2 - i*(b^2 * t_squared) - if (t_squared_type == 2) // mul t_squared by -1 - c = csq_add(a, cmul(csq(b), t_squared)); // a^2 - -1*(b^2 * t_squared) - if (t_squared_type == 3) // mul t_squared by -i - c = csq_addi(a, cmul(csq(b), t_squared)); // a^2 - -i*(b^2 * t_squared) + b2t2 = cmul(csq(b), t_squared); // b2t2 = b^2 * t_squared + if (t_squared_type == 0) // mul t_squared by 1 + c = csq_sub(a, b2t2); // a^2 - (b^2 * t_squared) + if (t_squared_type == 1) // mul t_squared by i + c = csq_subi(a, b2t2); // a^2 - i*(b^2 * t_squared) + if (t_squared_type == 2) // mul t_squared by -1 + c = csq_add(a, b2t2); // a^2 - -1*(b^2 * t_squared) + if (t_squared_type == 3) // mul t_squared by -i + c = csq_addi(a, b2t2); // a^2 - -i*(b^2 * t_squared) d = mul2(cmul(a, b)); X2_conjb(c, d); *pa = SWAP_XY(c), *pb = SWAP_XY(d); @@ -884,7 +885,7 @@ void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_ b2 = csq(b, 4, 3); // b2 = b^2, b2 range is 0..1+ ab = addq(a, b); // Compute 2ab as (a + b)^2 - a^2 - b^2. ab range is 1-..5+ - addin = neg(addq(a2, b2), 4); // add this into the csq of a+b, addin in range 0..4 + addin = neg(addq(a2, b2), 4); // add this into the csq of a+b, addin range is 0..4 d = csqa(ab, addin, 6); // d = 2ab, range is 0..1+ b2t2 = cmul(b2, t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ From 3135c791a8991209e9e22533398d2afb2d2bdc16 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 1 May 2026 02:31:54 +0000 Subject: [PATCH 058/214] Ifdefed recent GF61 onePairSquare improvement. CUDA compiler couldn't handle it -- caused local memory usage. --- src/cl/tailsquare.cl | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index a18a42d4..55247cf0 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -877,14 +877,16 @@ void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_ GF61 a = *pa, b = *pb; GF61 a2, b2, b2t2, ab, addin, c, d; +// This code should be faster (saves at least one wide mul) but the CUDA compiler makes poorer decisions regarding register usage resulting in local memory usage +#if ENABLE_BETTER_ONEPAIRSQ X2qconjb(&a, &b); // X2(a, conjugate(b)). a.x range is 0..2+, a.y range is -1-..1+, b.x range is -1-..1+, b.y range is 0..2+ a.y += 2*M61; // a range is 0..2+ / 1-..3+ b.x += 2*M61; // b range is 1-..3+ / 0..2+ + ab = addq(a, b); // Compute 2ab as (a + b)^2 - a^2 - b^2. ab range is 1-..5+ a2 = csqq(a, 3, 4); // a2 = a^2, a2 range is 0..2+ b2 = csq(b, 4, 3); // b2 = b^2, b2 range is 0..1+ - ab = addq(a, b); // Compute 2ab as (a + b)^2 - a^2 - b^2. ab range is 1-..5+ addin = neg(addq(a2, b2), 4); // add this into the csq of a+b, addin range is 0..4 d = csqa(ab, addin, 6); // d = 2ab, range is 0..1+ @@ -914,6 +916,36 @@ void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_ c = modM61q(c, 2, 0); d = modM61q(d, 2, 4); } +#else + X2conjb(a, b); // X2(a, conjugate(b)) + a2 = csqq(a, 2); // a2 = a^2, a2.x range is 0..2+, a2.y range is 0..3+ + b2t2 = cmul(csq(b), t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ + d = cmul(a, b); d = d + d; // d = 2ab, d range is 0..2+ + if (t_squared_type == 0) { // mul t_squared by 1 + c = subq(a2, b2t2); // c.x range is -1..2+, c.y range is -1-..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is -1-..5+, d.x range is -3-.> + c = modM61q(c, 2); + d = modM61q(d, 4); + } + if (t_squared_type == 1) { // mul t_squared by i + c = subiq(a2, b2t2); // c.x range is 0..3+, c.y range is -1-..3+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is -1..5+, d.x range is -2-..3+> + c = modM61q(c, 0, 2); + d = modM61q(d, 4); + } + if (t_squared_type == 2) { // mul t_squared by -1 + c = addq(a2, b2t2); // c.x range is 0..3+, c.y range is 0..4+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is 0..6+, d.x range is -2-..3+,> + c = modM61q(c, 0); + d = modM61q(d, 3, 5); + } + if (t_squared_type == 3) { // mul t_squared by -i + c = addiq(a2, b2t2); // c.x range is -1-..2+, c.y range is 0..4+ + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is 0..6+, d.x range is -3-..2> + c = modM61q(c, 2, 0); + d = modM61q(d, 4, 5); + } +#endif *pa = SWAP_XY(c), *pb = SWAP_XY(d); } From 3791d56db37b90384ab09d2d83514b7b8ca25a41 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 3 May 2026 04:20:16 +0000 Subject: [PATCH 059/214] 23 bits of float precision is not enough to optionaDouble and optionalHalve the way doubles does. FracBits overhauled so FP32 hybrid FFTs can use fracBits for optional doubling and halving. --- src/Gpu.cpp | 11 +-- src/cl/base.cl | 1 + src/cl/carry.cl | 94 ++++++++++++-------- src/cl/carryb.cl | 5 +- src/cl/carryfused.cl | 203 ++++++++++++++++++++++++------------------- src/cl/fftp.cl | 98 ++++++++++++--------- src/cl/weight.cl | 61 ++++++++----- 7 files changed, 277 insertions(+), 196 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index d31ba518..4e7dafc7 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -86,8 +86,6 @@ float invWeightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { return expm1(M_LN2 * - (double)(extra(N, E, kAt(H, line, col) + rep)) / N); } -float boundUnderOne(float x) { return std::min(x, nexttowardf(1, 0)); } - Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { u32 N = 2u * W * H; u32 groupWidth = W / nW; @@ -131,9 +129,6 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { for (u32 thread = 0; thread < groupWidth; ++thread) { auto iw = invWeight32(N, E, H, 0, thread, 0) ; auto w = weight32(N, E, H, 0, thread, 0) ; - // Play with the weight so that optionalDouble and optionalHalve work - iw = 2.0f * boundUnderOne(iw); - w = 2.0f * w; // Weights are scaled by 2^-24 and 2^48 so that multiplicaton by 1/epsilon does not generate infinty results (width and height variant 2). iw = iw * 281474976710656.0f; w = w * 0.000000059604644775390625f; @@ -2131,7 +2126,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { } doBigLog(k, res, ok, secsPerIt, kEndEnd, nErrors); - + if (k >= kEndEnd) { fs::path proofFile = saveProof(args, proofSet); return {isPrime, finalRes64, nErrors, proofFile.string(), toHex(res2048)}; @@ -2150,9 +2145,9 @@ PRPResult Gpu::isPrimePRP(const Task& task) { lastFailedRes64 = res; if (!doStop) { goto reload; } } - + logTimeKernels(); - + if (doStop) { queue->finish(); throw "stop requested"; diff --git a/src/cl/base.cl b/src/cl/base.cl index e0168f8d..7654be7a 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -206,6 +206,7 @@ G_H "group height" == SMALL_HEIGHT / NH #define BIG_HEIGHT (SMALL_HEIGHT * MIDDLE) #define ND (WIDTH * BIG_HEIGHT) #define NWORDS (ND * 2u) +#define NWORDS_IS_POWER_OF_TWO !(NWORDS & (NWORDS - 1)) #if (NW != 4 && NW != 8) || (NH != 4 && NH != 8) #error NW and NH must be passed in, expected value 4 or 8. diff --git a/src/cl/carry.cl b/src/cl/carry.cl index e8dc75a0..e55311e1 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -25,8 +25,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. Also add FRAC_BPW_HI to test first biglit flag. u32 line = gy * CARRY_LEN; - u32 fft_word_index = (gx * G_W * H + me * H + line) * 2; - u32 frac_bits = fft_word_index * FRAC_BPW_HI + mad_hi (fft_word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 word_index = (gx * G_W * H + me * H + line) * 2; + u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; T base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); @@ -70,15 +70,21 @@ KERNEL(G_W) carry(P(Word2) out, CP(F2) in, u32 posROE, P(CarryABM) carryOut, Big // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. Also add FRAC_BPW_HI to test first biglit flag. u32 line = gy * CARRY_LEN; - u32 fft_word_index = (gx * G_W * H + me * H + line) * 2; - u32 frac_bits = fft_word_index * FRAC_BPW_HI + mad_hi (fft_word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 word_index = (gx * G_W * H + me * H + line) * 2; + u32 frac_bits = fracBits(word_index); - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); for (i32 i = 0; i < CARRY_LEN; ++i) { - u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); + u32 p = G_W * gx + WIDTH * (line + i) + me; + u32 line_frac_bits = fracBits((line + i) * 2); + u32 w1_frac_bits = base_frac_bits + line_frac_bits; + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); bool biglit0 = frac_bits + (2*i) * FRAC_BPW_HI <= FRAC_BPW_HI; bool biglit1 = frac_bits + (2*i) * FRAC_BPW_HI >= -FRAC_BPW_HI; // Same as frac_bits + (2*i) * FRAC_BPW_HI + FRAC_BPW_HI <= FRAC_BPW_HI; out[p] = weightAndCarryPair(SWAP_XY(in[p]), U2(w1, w2), carry, biglit0, biglit1, &carry, &roundMax, &carryMax); @@ -128,8 +134,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF31) in, u32 posROE, P(CarryABM) carryOut, P #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -202,8 +208,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF61) in, u32 posROE, P(CarryABM) carryOut, P #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -277,8 +283,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -342,7 +348,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 word_index = (gx * G_W * H + me * H + line) * 2; - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. const u32 log2_root_two = (u32) (((1ULL << 30) / NWORDS) % 31); @@ -356,8 +366,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -366,11 +376,13 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big weight_shift = (weight_shift + log2_NWORDS + 1) % 31; for (i32 i = 0; i < CARRY_LEN; ++i) { - u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; + u32 p = G_W * gx + WIDTH * (line + i) + me; // Generate the FP32 and second GF31 weight shift - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); + u32 line_frac_bits = fracBits((line + i) * 2); + u32 w1_frac_bits = base_frac_bits + line_frac_bits; + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 31) weight_shift -= 31; @@ -421,7 +433,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 word_index = (gx * G_W * H + me * H + line) * 2; - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. const u32 log2_root_two = (u32) (((1ULL << 60) / NWORDS) % 61); @@ -435,8 +451,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -445,11 +461,13 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big weight_shift = (weight_shift + log2_NWORDS + 1) % 61; for (i32 i = 0; i < CARRY_LEN; ++i) { - u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; + u32 p = G_W * gx + WIDTH * (line + i) + me; // Generate the FP32 and second GF61 weight shift - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); + u32 line_frac_bits = fracBits((line + i) * 2); + u32 w1_frac_bits = base_frac_bits + line_frac_bits; + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 61) weight_shift -= 61; @@ -518,10 +536,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, P(u #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -593,7 +611,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 word_index = (gx * G_W * H + me * H + line) * 2; - F base = optionalDouble(fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx))); + F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); + u32 me_frac_bits = fracBits(me * H * 2); + u32 step_frac_bits = weightStepFracBits(gx); + u32 base_frac_bits = me_frac_bits + step_frac_bits; + base = optionalDouble(base, base_frac_bits > step_frac_bits); // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. const u32 m31_log2_root_two = (u32) (((1ULL << 30) / NWORDS) % 31); @@ -613,9 +635,9 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #define m61_combo_counter m61_combo.b const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); // We also adjust shift amount for the fact that NTT returns results multiplied by 2*NWORDS. const u32 log2_NWORDS = (WIDTH == 256 ? 8 : WIDTH == 512 ? 9 : WIDTH == 1024 ? 10 : 12) + @@ -628,8 +650,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; // Generate the FP32 and second GF31 and GF61 weight shift - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + gy * CARRY_LEN + i].x)); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP)); + u32 line_frac_bits = fracBits((line + i) * 2); + u32 w1_frac_bits = base_frac_bits + line_frac_bits; + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; m31_weight_shift = adjust_m31_weight_shift(m31_weight_shift); diff --git a/src/cl/carryb.cl b/src/cl/carryb.cl index c08004cd..ae9da09c 100644 --- a/src/cl/carryb.cl +++ b/src/cl/carryb.cl @@ -3,6 +3,7 @@ #include "base.cl" #include "math.cl" #include "carryutil.cl" +#include "weight.cl" KERNEL(G_W) carryB(P(Word2) io, CP(CarryABM) carryIn) { u32 g = get_group_id(0); @@ -14,8 +15,8 @@ KERNEL(G_W) carryB(P(Word2) io, CP(CarryABM) carryIn) { // Derive the big vs. little flags from the fractional number of bits in each FFT word rather read the flags from memory. // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. Also add FRAC_BPW_HI to test first biglit flag. u32 line = gy * CARRY_LEN; - u32 fft_word_index = (gx * G_W * H + me * H + line) * 2; - u32 frac_bits = fft_word_index * FRAC_BPW_HI + mad_hi (fft_word_index, FRAC_BPW_LO, FRAC_BPW_HI); + u32 word_index = (gx * G_W * H + me * H + line) * 2; + u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; io += G_W * gx + WIDTH * CARRY_LEN * gy; diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index db304570..c4727115 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -162,8 +162,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. u32 word_index = (lowMe * H + line) * 2; - u32 frac_bits = mul3264(word_index, FRAC_BPW_HI) + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); - const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); + u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; + const u32 frac_bits_bigstep = fracBits(G_W * H * 2); // Apply the inverse weights and carry propagate pairs to generate the output carries @@ -335,13 +335,24 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut fft_WIDTH1(lds + zerohack, u, smallTrig + zerohack, WMUL, lowMe); Word2 wu[NW]; + u32 me_frac_bits = fracBits(lowMe * H * 2); #if !NVIDIAGPU || CUDA_BACKEND F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); - weights.x = optionalDouble(weights.x); - weights.y = optionalHalve(weights.y); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif P(CFcarry) carryShuttlePtr = (P(CFcarry)) carryShuttle; @@ -352,16 +363,15 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. u32 word_index = (lowMe * H + line) * 2; - u32 frac_bits = mul3264(word_index, FRAC_BPW_HI) + mad_hi (word_index, FRAC_BPW_LO, FRAC_BPW_HI); - const u32 frac_bits_bigstep = ((G_W * H * 2) * FRAC_BPW_HI + (u32)(((u64)(G_W * H * 2) * FRAC_BPW_LO) >> 32)); + u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; + const u32 frac_bits_bigstep = fracBits(G_W * H * 2); // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); - + F invBase = weights.x; for (u32 i = 0; i < NW; ++i) { - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); // Generate big-word/little-word flags bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; @@ -411,14 +421,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - u[i] = U2(weight1, weight2); - } - // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); @@ -470,10 +472,14 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut } // Apply each 32 or 64 bit carry to the 2 words + F base = weights.y; for (i32 i = 0; i < NW; ++i) { + // Calculate inverse weights + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); + u[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); } dependentLaunch(); // Next kernel will be fftMiddleInFP32 @@ -547,9 +553,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -764,9 +770,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -994,9 +1000,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1212,13 +1218,24 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut fft_WIDTH1(lds31 + zerohack, u31, smallTrig31 + zerohack, WMUL, lowMe); Word2 wu[NW]; + u32 me_frac_bits = fracBits(lowMe * H * 2); #if !NVIDIAGPU || CUDA_BACKEND F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); - weights.x = optionalDouble(weights.x); - weights.y = optionalHalve(weights.y); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif P(i32) carryShuttlePtr = (P(i32)) carryShuttle; @@ -1242,9 +1259,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1257,11 +1274,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); + F invBase = weights.x; for (u32 i = 0; i < NW; ++i) { // Generate the FP32 weights and second GF31 weight shift - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 31) weight_shift -= 31; @@ -1320,14 +1337,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - uF2[i] = U2(weight1, weight2); - } - // Shuffle carries up shufl_carries_up(ldsF2, carry, me, lowMe); @@ -1379,7 +1388,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } // Apply each 32 or 64 bit carry to the 2 words. Apply weights. + F base = weights.y; for (i32 i = 0; i < NW; ++i) { + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); // Generate the second weight shift u32 weight_shift0 = weight_shift; combo_counter += combo_step; @@ -1388,7 +1400,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Generate big-word/little-word flag, propagate final carry bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - uF2[i] = U2(uF2[i].x * wu[i].x, uF2[i].y * wu[i].y); + uF2[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); u31[i] = U2(shl(make_Z31(wu[i].x), weight_shift0), shl(make_Z31(wu[i].y), weight_shift1)); // Generate weight shifts and frac_bits for next pair @@ -1460,13 +1472,24 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; + u32 me_frac_bits = fracBits(lowMe * H * 2); #if !NVIDIAGPU || CUDA_BACKEND F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); - weights.x = optionalDouble(weights.x); - weights.y = optionalHalve(weights.y); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif P(i64) carryShuttlePtr = (P(i64)) carryShuttle; @@ -1490,9 +1513,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * H * 2 - 1) * combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; u64 starting_combo_counter = combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1505,11 +1528,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); + F invBase = weights.x; for (u32 i = 0; i < NW; ++i) { // Generate the FP32 weights and second GF61 weight shift - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 61) weight_shift -= 61; @@ -1568,14 +1592,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - uF2[i] = U2(weight1, weight2); - } - // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); @@ -1627,7 +1643,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } // Apply each 32 or 64 bit carry to the 2 words. Apply weights. + F base = weights.y; for (i32 i = 0; i < NW; ++i) { + // Calculate inverse weights + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); // Generate the second weight shift u32 weight_shift0 = weight_shift; combo_counter += combo_step; @@ -1636,7 +1656,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Generate big-word/little-word flag, propagate final carry bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - uF2[i] = U2(uF2[i].x * wu[i].x, uF2[i].y * wu[i].y); + uF2[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); u61[i] = U2(shl(make_Z61(wu[i].x), weight_shift0), shl(make_Z61(wu[i].y), weight_shift1)); // Generate weight shifts and frac_bits for next pair @@ -1734,14 +1754,14 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * H * 2 - 1) * m31_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; u64 m31_starting_combo_counter = m31_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * H * 2 - 1) * m61_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; u64 m61_starting_combo_counter = m61_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -1968,13 +1988,24 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut fft_WIDTH1(lds61 + zerohack, u61, smallTrig61 + zerohack, WMUL, lowMe); Word2 wu[NW]; + u32 me_frac_bits = fracBits(lowMe * H * 2); #if !NVIDIAGPU || CUDA_BACKEND F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), TSLOAD(&THREAD_WEIGHTS[G_W + line])); + u32 line_frac_bits = fracBits(line * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > line_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > line_frac_bits); #else F2 weights = fancyMul(TFLOAD(&THREAD_WEIGHTS[lowMe]), CONST_THREAD_WEIGHTS[line % 64]); - weights.x = optionalDouble(weights.x); - weights.y = optionalHalve(weights.y); + u32 partialLine_frac_bits = fracBits((line % 64) * 2); + u32 base_frac_bits = me_frac_bits + partialLine_frac_bits; + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); + partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); + base_frac_bits = base_frac_bits + partialLine_frac_bits + weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); + weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif P(i64) carryShuttlePtr = (P(i64)) carryShuttle; @@ -2003,14 +2034,14 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * H * 2 - 1) * m31_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; u64 m31_starting_combo_counter = m31_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * H * 2 - 1) * m61_combo_step + (((u64) (G_W * H * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * H * 2 - 1) + make_u64((G_W * H * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; u64 m61_starting_combo_counter = m61_combo_counter; // Save starting counter before adding log2_NWORDS+1 for applying weights after carry propagation @@ -2023,11 +2054,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Apply the inverse weights and carry propagate pairs to generate the output carries - F invBase = optionalDouble(weights.x); + F invBase = weights.x; for (u32 i = 0; i < NW; ++i) { // Generate the FP32 weights and second GF31 and GF61 weight shift - F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); - F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); + F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); + F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; m31_weight_shift = adjust_m31_weight_shift(m31_weight_shift); @@ -2093,14 +2124,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif - // Calculate inverse weights - F base = optionalHalve(weights.y); - for (u32 i = 0; i < NW; ++i) { - F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP)); - uF2[i] = U2(weight1, weight2); - } - // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); @@ -2152,7 +2175,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } // Apply each 32 or 64 bit carry to the 2 words. Apply weights. + F base = weights.y; for (i32 i = 0; i < NW; ++i) { + // Calculate inverse weights + F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); // Generate the second weight shifts u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; @@ -2165,7 +2192,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Generate big-word/little-word flag, propagate final carry bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); - uF2[i] = U2(uF2[i].x * wu[i].x, uF2[i].y * wu[i].y); + uF2[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); u31[i] = U2(shl(make_Z31(wu[i].x), m31_weight_shift0), shl(make_Z31(wu[i].y), m31_weight_shift1)); u61[i] = U2(shl(make_Z61(wu[i].x), m61_weight_shift0), shl(make_Z61(wu[i].y), m61_weight_shift1)); diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index 43ffef8e..57e2af82 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -48,11 +48,16 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); for (u32 i = 0; i < NW; ++i) { - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + u32 step_frac_bits = weightStepFracBits(i); + u32 w1_frac_bits = base_frac_bits + step_frac_bits; + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), base_frac_bits > step_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 p = G_W * i + me; u[i] = U2(in[p].x * w1, in[p].y * w2); } @@ -93,9 +98,9 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { @@ -150,9 +155,9 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; for (u32 i = 0; i < NW; ++i) { @@ -213,9 +218,9 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { @@ -266,10 +271,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); - u32 word_index = (me * BIG_HEIGHT + g) * 2; + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 31. const u32 log2_root_two = (u32) (((1ULL << 30) / NWORDS) % 31); @@ -283,16 +291,16 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (31ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 31; for (u32 i = 0; i < NW; ++i) { u32 p = G_W * i + me; // Generate the FP32 weights and the second GF31 weight shift - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 31) weight_shift -= 31; @@ -336,10 +344,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); - u32 word_index = (me * BIG_HEIGHT + g) * 2; + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 61. const u32 log2_root_two = (u32) (((1ULL << 60) / NWORDS) % 61); @@ -353,16 +364,16 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG #define weight_shift combo.a[1] #define combo_counter combo.b - const u64 combo_step = ((u64) bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - combo_counter = mul3264(word_index, combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 combo_step = make_u64(bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * bigword_weight_shift_minus1, 0)) % (61ULL << 32); + combo_counter = comboFracBits(word_index) + make_u64(word_index * bigword_weight_shift_minus1, 0xFFFFFFFF); weight_shift = weight_shift % 61; for (u32 i = 0; i < NW; ++i) { u32 p = G_W * i + me; // Generate the FP32 weights and the second GF61 weight shift - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 61) weight_shift -= 61; @@ -427,13 +438,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m31_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m61_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; for (u32 i = 0; i < NW; ++i) { @@ -493,10 +504,13 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG in += g * WIDTH; - F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y)); - u32 word_index = (me * BIG_HEIGHT + g) * 2; + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); + u32 line_frac_bits = fracBits(g * 2); + u32 base_frac_bits = me_frac_bits + line_frac_bits; + F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + // Weight is 2^[ceil(qj / n) - qj/n] where j is the word index, q is the Mersenne exponent, and n is the number of words. // Weights can be applied with shifts because 2 is the 60th root GF61. // Let s be the shift amount for word 1. The shift amount for word x is ceil(x * (s - 1) + num_big_words_less_than_x) % 61. @@ -516,20 +530,20 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG #define m61_weight_shift m61_combo.a[1] #define m61_combo_counter m61_combo.b - const u64 m31_combo_step = ((u64) m31_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m31_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m31_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (31ULL << 32); - m31_combo_counter = mul3264(word_index, m31_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m31_combo_step = make_u64(m31_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m31_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m31_bigword_weight_shift_minus1, 0)) % (31ULL << 32); + m31_combo_counter = comboFracBits(word_index) + make_u64(word_index * m31_bigword_weight_shift_minus1, 0xFFFFFFFF); m31_weight_shift = m31_weight_shift % 31; - const u64 m61_combo_step = ((u64) m61_bigword_weight_shift_minus1 << 32) + FRAC_BPW_HI; - const u64 m61_combo_bigstep = ((G_W * BIG_HEIGHT * 2 - 1) * m61_combo_step + (((u64) (G_W * BIG_HEIGHT * 2 - 1) * FRAC_BPW_LO) >> 32)) % (61ULL << 32); - m61_combo_counter = mul3264(word_index, m61_combo_step) + mul_hi(word_index, FRAC_BPW_LO) + 0xFFFFFFFFULL; + const u64 m61_combo_step = make_u64(m61_bigword_weight_shift_minus1, FRAC_BPW_HI); + const u64 m61_combo_bigstep = (comboFracBits(G_W * BIG_HEIGHT * 2 - 1) + make_u64((G_W * BIG_HEIGHT * 2 - 1) * m61_bigword_weight_shift_minus1, 0)) % (61ULL << 32); + m61_combo_counter = comboFracBits(word_index) + make_u64(word_index * m61_bigword_weight_shift_minus1, 0xFFFFFFFF); m61_weight_shift = m61_weight_shift % 61; for (u32 i = 0; i < NW; ++i) { u32 p = G_W * i + me; // Generate the FP32 weights and the second GF31 and GF61 weight shift - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i))); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP)); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; m31_weight_shift = adjust_m31_weight_shift(m31_weight_shift); diff --git a/src/cl/weight.cl b/src/cl/weight.cl index 66075bb7..ccad666a 100644 --- a/src/cl/weight.cl +++ b/src/cl/weight.cl @@ -3,6 +3,33 @@ #define STEP (NWORDS - (EXP % NWORDS)) // bool isBigWord(u32 extra) { return extra < NWORDS - STEP; } +// Determine the fractional-bits-per-word for a given FFT word. The fracbits value is (word * STEP % NWORDS) / NWORDS. +// The fracbits value is multiplied by 2^32 and truncated to make an integer. Fracbits can be used to determine weights and big-vs-little-word flags. +// Weight is 2^(1 - fracbits), but if fracbits is zero it is 2^0. +// Big word is true if fracbits + FRAC_BPW_HI does not overflow, but also true if fracbits is zero. +// To eliminate special logic for the FFT word 0, we subtract one from fracbits. +u32 fracBits(u32 i) { +#if NWORDS_IS_POWER_OF_TWO + return i * (FRAC_BPW_HI + 1) - 1; // We know FRAC_BPW_LO is -1 +#else + return i * FRAC_BPW_HI + mul_hi(i, FRAC_BPW_LO) - 1; +#endif +} + +// Somewhat similar to the above. Also returns the number of big words that occurred in getting to a given word. +u64 comboFracBits(u32 i) { +#if NWORDS_IS_POWER_OF_TWO + return (u64)i * (u64)(FRAC_BPW_HI + 1) - 1; // We know FRAC_BPW_LO is -1 +#else + return (u64)i * (u64)FRAC_BPW_HI + mul_hi(i, FRAC_BPW_LO) - 1; +#endif +} + +// Routines to acces the 8 precomputed step weights +u32 weightStepIndex(u32 i) { return i * STEP % NW * (8 / NW); } +u32 weightStepFracBits(u32 i) { return 0xFFFFFFFF - (weightStepIndex(i) << 29); } + + #if FFT_FP64 T fweightStep(u32 i) { @@ -17,7 +44,7 @@ T fweightStep(u32 i) { 0.68179283050742912, 0.83400808640934243, }; - return TWO_TO_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_NTH[weightStepIndex(i)]; } T iweightStep(u32 i) { @@ -32,7 +59,7 @@ T iweightStep(u32 i) { -0.40539644249863949, -0.45474613366737116, }; - return TWO_TO_MINUS_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_MINUS_NTH[weightStepIndex(i)]; } // This routine is not used. It forces "-use NO_ASM" in Windows. bfi should be replaced by a builtin if ever needed. @@ -93,7 +120,7 @@ F fweightStep(u32 i) { 0.68179283050742912, 0.83400808640934243, }; - return TWO_TO_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_NTH[weightStepIndex(i)]; } F iweightStep(u32 i) { @@ -108,28 +135,20 @@ F iweightStep(u32 i) { -0.40539644249863949, -0.45474613366737116, }; - return TWO_TO_MINUS_NTH[i * STEP % NW * (8 / NW)]; + return TWO_TO_MINUS_NTH[weightStepIndex(i)]; } -F optionalDouble(F iw) { - // In a straightforward implementation, inverse weights are between 0.5 and 1.0. We use inverse weights between 1.0 and 2.0 - // because it allows us to implement this routine with a single OR instruction on the exponent. The original implementation - // where this routine took as input values from 0.25 to 1.0 required both an AND and an OR instruction on the exponent. - // return iw <= 1.0 ? iw * 2 : iw; - assert(iw > 0.5 && iw < 2); - uint u = as_uint(iw); - u |= 0x00800000; - return as_float(u); +F optionalDouble(F iw, int flag) { + // The 23 bits of precision in a float is not enough to handle doubling and halving the same way FP64 does. + // A straightforward implementation. Inverse weights are between > 0.5 and <= 1.0. + F doubled_iw = iw + iw; + return flag ? doubled_iw : iw; } -F optionalHalve(F w) { // return w >= 4 ? w / 2 : w; - // In a straightforward implementation, weights are between 1.0 and 2.0. We use weights between 2.0 and 4.0 because - // it allows us to implement this routine with a single AND instruction on the exponent. The original implementation - // where this routine took as input values from 1.0 to 4.0 required both an AND and an OR instruction on the exponent. - assert(w >= 2 && w < 8); - uint u = as_uint(w); - u &= 0xFF7FFFFF; - return as_float(u); +F optionalHalve(F w, int flag) { + // A straightforward implementation. Weights are between >= 1.0 and < 2.0. + F halved_w = w * 0.5f; + return flag ? halved_w : w; } #endif From 1d5df448cbc238adaa88db0d831a4ee3ef64e315 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 3 May 2026 04:56:27 +0000 Subject: [PATCH 060/214] Fixed typo in openCL compile of previous commit --- src/cl/carryfused.cl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index c4727115..72e2b3cf 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -350,7 +350,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); - base_frac_bits = base_frac_bits + partialLine_frac_bits + base_frac_bits = base_frac_bits + partialLine_frac_bits; weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif @@ -1233,7 +1233,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); - base_frac_bits = base_frac_bits + partialLine_frac_bits + base_frac_bits = base_frac_bits + partialLine_frac_bits; weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif @@ -1487,7 +1487,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); - base_frac_bits = base_frac_bits + partialLine_frac_bits + base_frac_bits = base_frac_bits + partialLine_frac_bits; weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif @@ -2003,7 +2003,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); weights = fancyMul(weights, CONST_THREAD_WEIGHTS[64 + line / 64]); partialLine_frac_bits = fracBits(((line / 64) * 64) * 2); - base_frac_bits = base_frac_bits + partialLine_frac_bits + base_frac_bits = base_frac_bits + partialLine_frac_bits; weights.x = optionalDouble(weights.x, base_frac_bits > partialLine_frac_bits); weights.y = optionalHalve(weights.y, base_frac_bits > partialLine_frac_bits); #endif From 85c61cb1e6744563c0f5681d27e6486fccd498df Mon Sep 17 00:00:00 2001 From: george Date: Sun, 3 May 2026 14:22:19 +0000 Subject: [PATCH 061/214] Fixed crash bug looking up BPW on the experimental FFT types 52 and 53 --- src/FFTConfig.cpp | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index 2e45afd0..dd0568a7 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -140,24 +140,31 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : if (height > width) { bpw = FFTShape{t, h, m, w}.bpw; } else { - // Make up some defaults - - //double d = 0.275 * (log2(size()) - log2(256 * 13 * 1024 * 2)); - //bpw = {18.1-d, 18.2-d, 18.2-d, 18.3-d}; - //log("BPW info for %s not found, defaults={%.2f, %.2f, %.2f, %.2f}\n", s.c_str(), bpw[0], bpw[1], bpw[2], bpw[3]); - // Manipulate the shape into something that was likely pre-computed + u32 orig_w = w; + u32 orig_m = m; + u32 orig_h = h; while (m < 9) { m *= 2; w /= 2; } while (w >= 4*h) { w /= 2; h *= 2; } while (w < h || w < 256 || w == 2048) { w *= 2; h /= 2; } while (h < 256) { h *= 2; m /= 2; } if (m == 1) m = 2; - bpw = FFTShape{t, w, m, h}.bpw; - for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) bpw[j] -= 0.05f; // Assume this fft spec is worse than measured fft specs - if (this->isFavoredShape()) { // Don't output this warning message for non-favored shapes (we expect the BPW info to be missing) - printf("BPW info for %s not found, defaults={", s.c_str()); - for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) printf("%s%.2f", j ? ", " : "", (double) bpw[j]); - printf("}\n"); + + // Make up some defaults (should only happen for experimental FFT types (t >= 52) + if (w == orig_w && m == orig_m && h == orig_h) { + bpw = {18.1f, 18.1f, 18.1f, 18.1f, 18.1f, 18.1f}; + log("ERROR: BPW info for %s not found, using default of 18.1.\n", s.c_str()); + } + + // Try the modified shape + else { + bpw = FFTShape{t, w, m, h}.bpw; + for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) bpw[j] -= 0.05f; // Assume this fft spec is worse than measured fft specs + if (this->isFavoredShape()) { // Don't output this warning message for non-favored shapes (we expect the BPW info to be missing) + printf("BPW info for %s not found, defaults={", s.c_str()); + for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) printf("%s%.2f", j ? ", " : "", (double) bpw[j]); + printf("}\n"); + } } } } From 5e7f189bb00be729a22d9adacd6185e8a4805dc3 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 3 May 2026 18:57:51 +0000 Subject: [PATCH 062/214] Fix !ENABLE_BETTER_ONEPAIRSQ. All GF61 NTTs have been broken for a week. --- src/cl/math.cl | 4 ++-- src/cl/tailsquare.cl | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index 6e260c6b..0cd73719 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -921,8 +921,8 @@ ulong2 wideMul(u64 ab, u64 cd) { // Returns a * b not modded by M61. Max value of result depends on the m61_counts of the inputs. // Let n = (a_m61_count - 1) * (b_m61_count - 1). This is the maximum value in the highest 6 bits of a * b. -// If n <= 4 result will be at most (n+1)*M61+epsilon. -// If n > 4 result will be at most 2*M61+epsilon. +// If n <= 6 result will be at most (n+1)*M61+epsilon. +// If n > 6 result will be at most 2*M61+epsilon. Z61 OVERLOAD weakMul(Z61 a, Z61 b, const u32 a_m61_count, const u32 b_m61_count) { ulong2 ab = wideMul(a, b); u64 lo = ab.x, hi = ab.y; diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 55247cf0..80dc0260 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -918,30 +918,31 @@ void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_ } #else X2conjb(a, b); // X2(a, conjugate(b)) - a2 = csqq(a, 2); // a2 = a^2, a2.x range is 0..2+, a2.y range is 0..3+ + a2 = csqq(a, 2); // a2 = a^2, a2.x range is 0..7+, a2.y range is 0..2+ + a2.x = modM61(a2.x); // a2.x range is 0..1+, a2.y range is 0..2+ b2t2 = cmul(csq(b), t_squared); // b2t2 = b^2 * t_squared, b2t2 range is 0..1+ d = cmul(a, b); d = d + d; // d = 2ab, d range is 0..2+ if (t_squared_type == 0) { // mul t_squared by 1 c = subq(a2, b2t2); // c.x range is -1..2+, c.y range is -1-..3+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is -1-..5+, d.x range is -3-.> + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is -1-..5+, d.x range is -3-..2+, d.y range is -3-..3+ c = modM61q(c, 2); d = modM61q(d, 4); } if (t_squared_type == 1) { // mul t_squared by i c = subiq(a2, b2t2); // c.x range is 0..3+, c.y range is -1-..3+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is -1..5+, d.x range is -2-..3+> + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is -1..5+, d.x range is -2-..3+, d.y range is -3-..3+ c = modM61q(c, 0, 2); d = modM61q(d, 4); } if (t_squared_type == 2) { // mul t_squared by -1 c = addq(a2, b2t2); // c.x range is 0..3+, c.y range is 0..4+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is 0..6+, d.x range is -2-..3+,> + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is 0..5+, c.y range is 0..6+, d.x range is -2-..3+, d.y range is -4-..2+ c = modM61q(c, 0); d = modM61q(d, 3, 5); } if (t_squared_type == 3) { // mul t_squared by -i c = addiq(a2, b2t2); // c.x range is -1-..2+, c.y range is 0..4+ - X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is 0..6+, d.x range is -3-..2> + X2q_conjb(&c, &d); // X2(c, d); d = conjugate(d); c.x range is -1-..4+, c.y range is 0..6+, d.x range is -3-..2+, d.y range is -4-..2+ c = modM61q(c, 2, 0); d = modM61q(d, 4, 5); } From 04e6753dd48d95dfb66368c0189474ef7e871e48 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 3 May 2026 21:29:09 +0000 Subject: [PATCH 063/214] Finished frac_bits makeover. --- src/cl/carry.cl | 36 ++++++++++++++++-------------------- src/cl/carryfused.cl | 37 +++++++++++++++++++++++++++---------- src/cl/fftp.cl | 13 +++++++++---- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/src/cl/carry.cl b/src/cl/carry.cl index e55311e1..243d1b8c 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -68,10 +68,9 @@ KERNEL(G_W) carry(P(Word2) out, CP(F2) in, u32 posROE, P(CarryABM) carryOut, Big float roundMax = 0; float carryMax = 0; - // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. Also add FRAC_BPW_HI to test first biglit flag. + // Calculate the most significant 32-bits of FRAC_BPW * the index of the FFT word. u32 line = gy * CARRY_LEN; u32 word_index = (gx * G_W * H + me * H + line) * 2; - u32 frac_bits = fracBits(word_index); F base = fancyMul(THREAD_WEIGHTS[me].x, iweightStep(gx)); u32 me_frac_bits = fracBits(me * H * 2); @@ -79,15 +78,18 @@ KERNEL(G_W) carry(P(Word2) out, CP(F2) in, u32 posROE, P(CarryABM) carryOut, Big u32 base_frac_bits = me_frac_bits + step_frac_bits; base = optionalDouble(base, base_frac_bits > step_frac_bits); + u32 frac_bits = fracBits(word_index); + for (i32 i = 0; i < CARRY_LEN; ++i) { u32 p = G_W * gx + WIDTH * (line + i) + me; - u32 line_frac_bits = fracBits((line + i) * 2); - u32 w1_frac_bits = base_frac_bits + line_frac_bits; - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); - bool biglit0 = frac_bits + (2*i) * FRAC_BPW_HI <= FRAC_BPW_HI; - bool biglit1 = frac_bits + (2*i) * FRAC_BPW_HI >= -FRAC_BPW_HI; // Same as frac_bits + (2*i) * FRAC_BPW_HI + FRAC_BPW_HI <= FRAC_BPW_HI; + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + frac_bits += FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; + bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits <= FRAC_BPW_HI; out[p] = weightAndCarryPair(SWAP_XY(in[p]), U2(w1, w2), carry, biglit0, biglit1, &carry, &roundMax, &carryMax); + // Generate frac_bits for next pair + frac_bits += FRAC_BPW_HI; } carryOut[G_W * g + me] = carry; @@ -379,10 +381,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 p = G_W * gx + WIDTH * (line + i) + me; // Generate the FP32 and second GF31 weight shift - u32 line_frac_bits = fracBits((line + i) * 2); - u32 w1_frac_bits = base_frac_bits + line_frac_bits; - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 31) weight_shift -= 31; @@ -464,10 +464,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 p = G_W * gx + WIDTH * (line + i) + me; // Generate the FP32 and second GF61 weight shift - u32 line_frac_bits = fracBits((line + i) * 2); - u32 w1_frac_bits = base_frac_bits + line_frac_bits; - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 weight_shift0 = weight_shift; combo_counter += combo_step; if (weight_shift > 61) weight_shift -= 61; @@ -650,10 +648,8 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big u32 p = G_W * gx + WIDTH * (CARRY_LEN * gy + i) + me; // Generate the FP32 and second GF31 and GF61 weight shift - u32 line_frac_bits = fracBits((line + i) * 2); - u32 w1_frac_bits = base_frac_bits + line_frac_bits; - F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), w1_frac_bits > line_frac_bits); - F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); + F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 m31_weight_shift0 = m31_weight_shift; m31_combo_counter += m31_combo_step; m31_weight_shift = adjust_m31_weight_shift(m31_weight_shift); diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 72e2b3cf..5d683d02 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -164,18 +164,18 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut u32 word_index = (lowMe * H + line) * 2; u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; const u32 frac_bits_bigstep = fracBits(G_W * H * 2); + u32 starting_frac_bits = frac_bits; // Apply the inverse weights and carry propagate pairs to generate the output carries T invBase = optionalDouble(weights.x); - for (u32 i = 0; i < NW; ++i) { T invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i))); T invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP)); // Generate big-word/little-word flags - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; - bool biglit1 = frac_bits + i * frac_bits_bigstep >= -FRAC_BPW_HI; // Same as frac_bits + i * frac_bits_bigstep + FRAC_BPW_HI <= FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; + bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits + FRAC_BPW_HI <= FRAC_BPW_HI; // Apply the inverse weights, optionally compute roundoff error, and convert to integer. Also apply MUL3 here. // Then propagate carries through two words (the first carry does not have to be accurately calculated because it will @@ -184,7 +184,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // For an LL test, add -2 as the very initial "carry in" // We'd normally use logical &&, but the compiler whines with warning and bitwise fixes it (LL & (i == 0) & (line==0) & (me == 0)) ? -2 : 0, biglit0, biglit1, &carry[i], &roundMax, &carryMax); + + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; } + frac_bits = starting_frac_bits; // Restore starting frac_bits for applying weights after carry propagation #if ROE updateStats(bufROE, posROE, roundMax); @@ -281,9 +285,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Apply each 32 or 64 bit carry to the 2 words for (i32 i = 0; i < NW; ++i) { - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); u[i] = U2(u[i].x * wu[i].x, u[i].y * wu[i].y); + + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; } dependentLaunch(); // Next kernel will be fftMiddleInFP64 @@ -361,10 +368,11 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut float roundMax = 0; float carryMax = 0; - // Calculate the most significant 32-bits of FRAC_BPW * the word index. Also add FRAC_BPW_HI to test first biglit flag. + // Calculate the most significant 32-bits of FRAC_BPW * the word index. u32 word_index = (lowMe * H + line) * 2; - u32 frac_bits = fracBits(word_index) + FRAC_BPW_HI; - const u32 frac_bits_bigstep = fracBits(G_W * H * 2); + u32 frac_bits = fracBits(word_index); + const u32 frac_bits_bigstep = fracBits(G_W * H * 2 - 1); + u32 starting_frac_bits = frac_bits; // Apply the inverse weights and carry propagate pairs to generate the output carries @@ -372,10 +380,11 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut for (u32 i = 0; i < NW; ++i) { F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + frac_bits += FRAC_BPW_HI; // Generate big-word/little-word flags - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; - bool biglit1 = frac_bits + i * frac_bits_bigstep >= -FRAC_BPW_HI; // Same as frac_bits + i * frac_bits_bigstep + FRAC_BPW_HI <= FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; + bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits + FRAC_BPW_HI <= FRAC_BPW_HI; // Apply the inverse weights, optionally compute roundoff error, and convert to integer. Also apply MUL3 here. // Then propagate carries through two words (the first carry does not have to be accurately calculated because it will @@ -384,7 +393,11 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // For an LL test, add -2 as the very initial "carry in" // We'd normally use logical &&, but the compiler whines with warning and bitwise fixes it (LL & (i == 0) & (line==0) & (me == 0)) ? -2 : 0, biglit0, biglit1, &carry[i], &roundMax, &carryMax); + + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; } + frac_bits = starting_frac_bits; // Restore starting frac_bits for applying weights after carry propagation #if ROE updateStats(bufROE, posROE, roundMax); @@ -477,9 +490,13 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Calculate inverse weights F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); F weight2 = optionalHalve(fancyMul(weight1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); - bool biglit0 = frac_bits + i * frac_bits_bigstep <= FRAC_BPW_HI; + frac_bits += FRAC_BPW_HI; + bool biglit0 = frac_bits <= FRAC_BPW_HI; wu[i] = carryFinal(wu[i], carry[i], biglit0); u[i] = U2(weight1 * wu[i].x, weight2 * wu[i].y); + + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; } dependentLaunch(); // Next kernel will be fftMiddleInFP32 diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index 57e2af82..e3c4fecf 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -48,18 +48,23 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ in += g * WIDTH; + u32 word_index = (me * BIG_HEIGHT + g) * 2; + u32 me_frac_bits = fracBits(me * BIG_HEIGHT * 2); u32 line_frac_bits = fracBits(g * 2); u32 base_frac_bits = me_frac_bits + line_frac_bits; F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); + u32 frac_bits = fracBits(word_index); + const u32 frac_bits_bigstep = fracBits(G_W * BIG_HEIGHT * 2); + for (u32 i = 0; i < NW; ++i) { - u32 step_frac_bits = weightStepFracBits(i); - u32 w1_frac_bits = base_frac_bits + step_frac_bits; - F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), base_frac_bits > step_frac_bits); - F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), w1_frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); + F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); + F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); u32 p = G_W * i + me; u[i] = U2(in[p].x * w1, in[p].y * w2); + // Generate frac_bits for next pair + frac_bits += frac_bits_bigstep; } fft_WIDTH(lds, u, smallTrig, 1, me); From ea7e1b4844862206057e8f69d0a49a66d57a2fc0 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 4 May 2026 04:34:18 +0000 Subject: [PATCH 064/214] Fixed the constant multiplier in fftMiddleOut because FP32 weights are no longer doubled for optionalDouble and optionalHalve. --- src/cl/carry.cl | 6 +++++- src/cl/carryfused.cl | 7 +++---- src/cl/fftmiddleout.cl | 14 ++++---------- src/cl/fftp.cl | 3 ++- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/cl/carry.cl b/src/cl/carry.cl index 243d1b8c..fe2ce3d3 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -80,13 +80,17 @@ KERNEL(G_W) carry(P(Word2) out, CP(F2) in, u32 posROE, P(CarryABM) carryOut, Big u32 frac_bits = fracBits(word_index); + // Base_frac_bits and frac_bits are inexact values. We only want to trigger an optional double when it is clear to do so. + // Fudge base_frac_bits to make it harder to trigger a double when the two inexact values are equal. + base_frac_bits++; + for (i32 i = 0; i < CARRY_LEN; ++i) { u32 p = G_W * gx + WIDTH * (line + i) + me; F w1 = optionalDouble(fancyMul(base, THREAD_WEIGHTS[G_W + line + i].x), frac_bits > base_frac_bits); F w2 = optionalDouble(fancyMul(w1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); frac_bits += FRAC_BPW_HI; bool biglit0 = frac_bits <= FRAC_BPW_HI; - bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits <= FRAC_BPW_HI; + bool biglit1 = frac_bits >= -FRAC_BPW_HI; // Same as frac_bits + FRAC_BPW_HI <= FRAC_BPW_HI; out[p] = weightAndCarryPair(SWAP_XY(in[p]), U2(w1, w2), carry, biglit0, biglit1, &carry, &roundMax, &carryMax); // Generate frac_bits for next pair frac_bits += FRAC_BPW_HI; diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 5d683d02..d5d52610 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -368,15 +368,14 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut float roundMax = 0; float carryMax = 0; - // Calculate the most significant 32-bits of FRAC_BPW * the word index. + // Calculate the most significant 32-bits of FRAC_BPW * the word index (it's the same as base_frac_bits). u32 word_index = (lowMe * H + line) * 2; - u32 frac_bits = fracBits(word_index); const u32 frac_bits_bigstep = fracBits(G_W * H * 2 - 1); - u32 starting_frac_bits = frac_bits; // Apply the inverse weights and carry propagate pairs to generate the output carries F invBase = weights.x; + u32 frac_bits = base_frac_bits; for (u32 i = 0; i < NW; ++i) { F invWeight1 = i == 0 ? invBase : optionalDouble(fancyMul(invBase, iweightStep(i)), frac_bits > base_frac_bits); F invWeight2 = optionalDouble(fancyMul(invWeight1, IWEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); @@ -397,7 +396,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Generate frac_bits for next pair frac_bits += frac_bits_bigstep; } - frac_bits = starting_frac_bits; // Restore starting frac_bits for applying weights after carry propagation #if ROE updateStats(bufROE, posROE, roundMax); @@ -486,6 +484,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Apply each 32 or 64 bit carry to the 2 words F base = weights.y; + frac_bits = base_frac_bits; for (i32 i = 0; i < NW; ++i) { // Calculate inverse weights F weight1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); diff --git a/src/cl/fftmiddleout.cl b/src/cl/fftmiddleout.cl index 6e05b712..b7c486cb 100644 --- a/src/cl/fftmiddleout.cl +++ b/src/cl/fftmiddleout.cl @@ -110,11 +110,8 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { fft_MIDDLE(u); - // FFT results come out multiplied by the FFT length (NWORDS). Also, for performance reasons - // weights and invweights are doubled meaning we need to divide by another 2^2 and 2^2. - // Finally, roundoff errors are sometimes improved if we use the next lower double precision - // number. This may be due to roundoff errors introduced by applying inexact TWO_TO_N_8TH weights. - double factor = 1.0 / (4 * 4 * NWORDS); + // FFT results come out multiplied by the FFT length (NWORDS * 2). + const float factor = 1.0f / (NWORDS * 2); middleMul2(u, y, x, factor, trigF2); @@ -351,11 +348,8 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { fft_MIDDLE(u); - // FFT results come out multiplied by the FFT length (NWORDS). Also, for performance reasons - // weights and invweights are doubled meaning we need to divide by another 2^2 and 2^2. - // Finally, roundoff errors are sometimes improved if we use the next lower double precision - // number. This may be due to roundoff errors introduced by applying inexact TWO_TO_N_8TH weights. - double factor = 1.0 / (4 * 4 * NWORDS); + // FFT results come out multiplied by the FFT length (NWORDS * 2). + const float factor = 1.0f / (NWORDS * 2); middleMul2(u, y, x, factor, trigF2); diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index e3c4fecf..f91cd99a 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -55,9 +55,9 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ u32 base_frac_bits = me_frac_bits + line_frac_bits; F base = optionalHalve(fancyMul(THREAD_WEIGHTS[me].y, THREAD_WEIGHTS[G_W + g].y), base_frac_bits > line_frac_bits); - u32 frac_bits = fracBits(word_index); const u32 frac_bits_bigstep = fracBits(G_W * BIG_HEIGHT * 2); + u32 frac_bits = base_frac_bits; for (u32 i = 0; i < NW; ++i) { F w1 = i == 0 ? base : optionalHalve(fancyMul(base, fweightStep(i)), frac_bits > base_frac_bits); F w2 = optionalHalve(fancyMul(w1, WEIGHT_STEP), frac_bits + FRAC_BPW_HI > FRAC_BPW_HI); @@ -313,6 +313,7 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // Convert and weight input uF2[i] = U2(in[p].x * w1, in[p].y * w2); u31[i] = U2(shl(make_Z31(in[p].x), weight_shift0), shl(make_Z31(in[p].y), weight_shift1)); // Form a GF31 from each pair of input words + // Generate weight shifts and frac_bits for next pair combo_counter += combo_bigstep; if (weight_shift > 31) weight_shift -= 31; From e53a41b449fc0a67dcb30a9d67a42473e727bbe2 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 4 May 2026 06:48:08 +0000 Subject: [PATCH 065/214] With FP32 bug fixed, the BPW is updated with higher allowed values for the larger FP32 hybrid FFTs. --- src/fftbpw.h | 64 ++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/fftbpw.h b/src/fftbpw.h index fdd490fc..5bab17d0 100644 --- a/src/fftbpw.h +++ b/src/fftbpw.h @@ -112,24 +112,24 @@ { "1:4K:16:512", {38.37f, 38.37f, 38.37f, 38.37f, 38.37f, 38.37f}}, { "1:4K:16:1K", {38.12f, 38.12f, 38.12f, 38.12f, 38.12f, 38.12f}}, // Estimated // FFT3261 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "2:256:2:256", {34.57f, 34.57f, 34.57f, 34.57f, 34.57f, 34.57f}}, +{ "2:256:2:256", {34.53f, 34.53f, 34.53f, 34.53f, 34.53f, 34.53f}}, { "2:256:4:256", {34.24f, 34.24f, 34.24f, 34.24f, 34.24f, 34.24f}}, -{ "2:256:8:256", {34.06f, 34.06f, 34.06f, 34.06f, 34.06f, 34.06f}}, -{ "2:512:4:256", {34.06f, 34.06f, 34.06f, 34.06f, 34.06f, 34.06f}}, -{"2:256:16:256", {32.07f, 32.07f, 32.07f, 32.07f, 32.07f, 32.07f}}, -{ "2:512:8:256", {32.07f, 32.07f, 32.07f, 32.07f, 32.07f, 32.07f}}, -{ "2:512:4:512", {32.07f, 32.07f, 32.07f, 32.07f, 32.07f, 32.07f}}, -{ "2:1K:8:256", {31.81f, 31.81f, 31.81f, 31.81f, 31.81f, 31.81f}}, -{"2:512:16:256", {31.81f, 31.81f, 31.81f, 31.81f, 31.81f, 31.81f}}, -{ "2:512:8:512", {31.81f, 31.81f, 31.81f, 31.81f, 31.81f, 31.81f}}, -{ "2:1K:16:256", {31.50f, 31.50f, 31.50f, 31.50f, 31.50f, 31.50f}}, -{ "2:1K:8:512", {31.50f, 31.50f, 31.50f, 31.50f, 31.50f, 31.50f}}, -{"2:512:16:512", {31.50f, 31.50f, 31.50f, 31.50f, 31.50f, 31.50f}}, -{ "2:1K:16:512", {28.68f, 28.68f, 28.68f, 28.68f, 28.68f, 28.68f}}, // Very strange. 481421001 has a maxROE of 0.180, 481422001 has a maxROE of 0.5 -{ "2:1K:8:1K", {28.68f, 28.68f, 28.68f, 28.68f, 28.68f, 28.68f}}, -{ "2:1K:16:1K", {25.37f, 25.37f, 25.37f, 25.37f, 25.37f, 25.37f}}, // Also strange. 851422001 ROEmax=0.273, ROEavg=0.003 -{ "2:4K:16:512", {23.27f, 23.27f, 23.27f, 23.27f, 23.27f, 23.27f}}, -{ "2:4K:16:1K", {21.15f, 21.15f, 21.15f, 21.15f, 21.15f, 21.15f}}, // Estimated +{ "2:256:8:256", {33.89f, 33.89f, 33.89f, 33.89f, 33.89f, 33.89f}}, +{ "2:512:4:256", {33.89f, 33.89f, 33.89f, 33.89f, 33.89f, 33.89f}}, +{"2:256:16:256", {31.95f, 31.95f, 31.95f, 31.95f, 31.95f, 31.95f}}, +{ "2:512:8:256", {31.95f, 31.95f, 31.95f, 31.95f, 31.95f, 31.95f}}, +{ "2:512:4:512", {31.95f, 31.95f, 31.95f, 31.95f, 31.95f, 31.95f}}, +{ "2:1K:8:256", {31.66f, 31.66f, 31.66f, 31.66f, 31.66f, 31.66f}}, +{"2:512:16:256", {31.66f, 31.66f, 31.66f, 31.66f, 31.66f, 31.66f}}, +{ "2:512:8:512", {31.66f, 31.66f, 31.66f, 31.66f, 31.66f, 31.66f}}, +{ "2:1K:16:256", {31.35f, 31.35f, 31.35f, 31.35f, 31.35f, 31.35f}}, +{ "2:1K:8:512", {31.35f, 31.35f, 31.35f, 31.35f, 31.35f, 31.35f}}, +{"2:512:16:512", {31.35f, 31.35f, 31.35f, 31.35f, 31.35f, 31.35f}}, +{ "2:1K:16:512", {29.48f, 29.48f, 29.48f, 29.48f, 29.48f, 29.48f}}, +{ "2:1K:8:1K", {29.53f, 29.53f, 29.53f, 29.53f, 29.53f, 29.53f}}, +{ "2:1K:16:1K", {29.21f, 29.21f, 29.21f, 29.21f, 29.21f, 29.21f}}, +{ "2:4K:16:512", {28.94f, 28.94f, 28.94f, 28.94f, 28.94f, 28.94f}}, +{ "2:4K:16:1K", {28.64f, 28.64f, 28.64f, 28.64f, 28.64f, 28.64f}}, // Estimated // FFT61 - Computed by targeting maxROE of ~0.35 over 1000 iterations, probably could go higher { "3:256:2:256", {25.02f, 25.02f, 25.02f, 25.02f, 25.02f, 25.02f}}, { "3:256:4:256", {24.72f, 24.10f, 24.10f, 24.10f, 24.10f, 24.10f}}, @@ -150,24 +150,24 @@ { "3:4K:16:512", {22.92f, 22.92f, 22.92f, 22.92f, 22.92f, 22.92f}}, { "3:4K:16:1K", {22.72f, 22.72f, 22.72f, 22.72f, 22.72f, 22.72f}}, // Estimated // FFT323161 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher -{ "4:256:2:256", {50.05f, 50.05f, 50.05f, 50.05f, 50.05f, 50.05f}}, -{ "4:256:4:256", {49.76f, 49.76f, 49.76f, 49.76f, 49.76f, 49.76f}}, -{ "4:256:8:256", {49.59f, 49.59f, 49.59f, 49.59f, 49.59f, 49.59f}}, -{ "4:512:4:256", {49.59f, 49.59f, 49.59f, 49.59f, 49.59f, 49.59f}}, -{"4:256:16:256", {47.59f, 47.59f, 47.59f, 47.59f, 47.59f, 47.59f}}, -{ "4:512:8:256", {47.59f, 47.59f, 47.59f, 47.59f, 47.59f, 47.59f}}, -{ "4:512:4:512", {47.59f, 47.59f, 47.59f, 47.59f, 47.59f, 47.59f}}, -{ "4:1K:8:256", {47.33f, 47.33f, 47.33f, 47.33f, 47.33f, 47.33f}}, -{"4:512:16:256", {47.33f, 47.33f, 47.33f, 47.33f, 47.33f, 47.33f}}, -{ "4:512:8:512", {47.33f, 47.33f, 47.33f, 47.33f, 47.33f, 47.33f}}, +{ "4:256:2:256", {50.01f, 50.01f, 50.01f, 50.01f, 50.01f, 50.01f}}, +{ "4:256:4:256", {49.71f, 49.71f, 49.71f, 49.71f, 49.71f, 49.71f}}, +{ "4:256:8:256", {49.52f, 49.52f, 49.52f, 49.52f, 49.52f, 49.52f}}, +{ "4:512:4:256", {49.52f, 49.52f, 49.52f, 49.52f, 49.52f, 49.52f}}, +{"4:256:16:256", {47.55f, 47.55f, 47.55f, 47.55f, 47.55f, 47.55f}}, +{ "4:512:8:256", {47.55f, 47.55f, 47.55f, 47.55f, 47.55f, 47.55f}}, +{ "4:512:4:512", {47.55f, 47.55f, 47.55f, 47.55f, 47.55f, 47.55f}}, +{ "4:1K:8:256", {47.29f, 47.29f, 47.29f, 47.29f, 47.29f, 47.29f}}, +{"4:512:16:256", {47.29f, 47.29f, 47.29f, 47.29f, 47.29f, 47.29f}}, +{ "4:512:8:512", {47.29f, 47.29f, 47.29f, 47.29f, 47.29f, 47.29f}}, { "4:1K:16:256", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, { "4:1K:8:512", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, {"4:512:16:512", {47.00f, 47.00f, 47.00f, 47.00f, 47.00f, 47.00f}}, -{ "4:1K:16:512", {44.52f, 44.52f, 44.52f, 44.52f, 44.52f, 44.52f}}, -{ "4:1K:8:1K", {44.52f, 44.52f, 44.52f, 44.52f, 44.52f, 44.52f}}, -{ "4:1K:16:1K", {41.72f, 41.72f, 41.72f, 41.72f, 41.72f, 41.72f}}, // Strange 41.72 has tiny error, 41.75 is 0.5 -{ "4:4K:16:512", {39.50f, 39.50f, 39.50f, 39.50f, 39.50f, 39.50f}}, // Estimated -{ "4:4K:16:1K", {37.50f, 37.50f, 37.50f, 37.50f, 37.50f, 37.50f}}, // Estimated +{ "4:1K:16:512", {45.02f, 45.02f, 45.02f, 45.02f, 45.02f, 45.02f}}, +{ "4:1K:8:1K", {45.02f, 45.02f, 45.02f, 45.02f, 45.02f, 45.02f}}, +{ "4:1K:16:1K", {44.70f, 44.70f, 44.70f, 44.70f, 44.70f, 44.70f}}, +{ "4:4K:16:512", {44.42f, 44.42f, 44.42f, 44.42f, 44.42f, 44.42f}}, +{ "4:4K:16:1K", {44.12f, 44.12f, 44.12f, 44.12f, 44.12f, 44.12f}}, // Estimated // FFT3231 - Computed with -use TABMUL_CHAIN32=0,TAIL_TRIGS32=0 and targeting maxROE of ~0.35 over 1000 iterations, probably could go higher { "50:256:2:256", {19.57f, 19.57f, 19.57f, 19.57f, 19.57f, 19.57f}}, { "50:256:4:256", {19.23f, 19.23f, 19.23f, 19.23f, 19.23f, 19.23f}}, From f7e11d87b8ef8292eeb59f4ad4811c692113bb14 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 4 May 2026 14:44:17 +0000 Subject: [PATCH 066/214] Added tdulcet's suggested fix for the Windows-buffering-logfile-output problem. --- src/File.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/File.cpp b/src/File.cpp index 2d54bda2..2ab37b7c 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -18,9 +18,11 @@ File::File(const std::filesystem::path& path, const string& mode, bool throwOnEr if (mode == "ab") { assert(f); -#if HAS_SETLINEBUF - setlinebuf(f); -#endif +//#if HAS_SETLINEBUF +// setlinebuf(f); +//#endif + // tdulcet's suggested portable replacement for the lines above + setvbuf(f, nullptr, _IOLBF, 0); } } From da51b2a4914e4b497258e79d4a5fabf0caafbba9 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 7 May 2026 00:32:33 +0000 Subject: [PATCH 067/214] If proof generation fails, lower and lower proof powers are tried. Untested. I have a GPU that has twice failed proof generation and the next lower proof power succeeded. This will automate the process rather than raising an error and aborting which requires manual intervention. Up next see if we figure out how to catch this problem earlier (during write of proof residues?) --- src/Gpu.cpp | 32 +++++++++++++++++++------------- src/Gpu.h | 2 +- src/Proof.h | 3 ++- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 4e7dafc7..42a0caa3 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -1705,21 +1705,27 @@ void Gpu::doDiv9(u64 E, Words& words) { doDiv3(E, words); } -fs::path Gpu::saveProof(const Args& args, const ProofSet& proofSet) { - for (int retry = 0; retry < 2; ++retry) { - auto [proof, hashes] = proofSet.computeProof(this); - fs::path tmpFile = proof.file(args.proofToVerifyDir); - proof.save(tmpFile); +fs::path Gpu::saveProof(const Args& args, ProofSet& proofSet) { + bool problem_proof = false; + for ( ; ; ) { + for (int retry = 0; retry == 0 || (retry == 1 && !problem_proof); ++retry) { + auto [proof, hashes] = proofSet.computeProof(this); + fs::path tmpFile = proof.file(args.proofToVerifyDir); + proof.save(tmpFile); - fs::path proofFile = proof.file(args.proofResultDir); - - bool ok = Proof::load(tmpFile).verify(this, hashes); - log("Proof '%s' verification %s\n", tmpFile.string().c_str(), ok ? "OK" : "FAILED"); - if (ok) { - fancyRename(tmpFile, proofFile); - log("Proof '%s' generated\n", proofFile.string().c_str()); - return proofFile; + fs::path proofFile = proof.file(args.proofResultDir); + + bool ok = Proof::load(tmpFile).verify(this, hashes); + log("Proof '%s' verification %s\n", tmpFile.string().c_str(), ok ? "OK" : "FAILED"); + if (ok) { + fancyRename(tmpFile, proofFile); + log("Proof '%s' generated\n", proofFile.string().c_str()); + return proofFile; + } } + problem_proof = true; + proofSet.reducePower(); + if (proofSet.power < 4) break; } throw "bad proof generation"; } diff --git a/src/Gpu.h b/src/Gpu.h index a9ee95de..9553fec1 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -270,7 +270,7 @@ class Gpu { void modMul(Buffer& ioA, Buffer& inB, bool mul3 = false); void modMul(Buffer& ioA, Buffer& inB, enum LEAD_TYPE leadInB, bool mul3 = false); - fs::path saveProof(const Args& args, const ProofSet& proofSet); + fs::path saveProof(const Args& args, ProofSet& proofSet); std::pair readROE(); RoeInfo readCarryStats(); diff --git a/src/Proof.h b/src/Proof.h index c83d8ace..e2df024c 100644 --- a/src/Proof.h +++ b/src/Proof.h @@ -55,7 +55,7 @@ class Proof { class ProofSet { public: const u64 E; - const u32 power; + u32 power; private: vector points; @@ -86,5 +86,6 @@ class ProofSet { void save(u64 k, const Words& words) const { return save(E, power, k, words); } Words load(u64 k) const { return load(E, power, k); } + void reducePower(void) { power--; } std::pair> computeProof(Gpu *gpu) const; }; From 2c9a2e8e3da4cb1aa1082760d23252f676470f4f Mon Sep 17 00:00:00 2001 From: george Date: Sat, 9 May 2026 21:45:33 +0000 Subject: [PATCH 068/214] Fixed bug where worktodo-0.txt file name was not printed properly. fs:path.c_str() returns a wide char in Windows. --- src/Worktodo.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 0801ffec..7c31361c 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -109,14 +109,16 @@ static std::optional bestTask(const fs::path& fileName, bool smallest) { string workName(i32 instance) { return "worktodo-" + to_string(instance) + ".txt"; } optional getWork(Args& args, i32 instance) { - fs::path localWork = workName(instance); + string filename = workName(instance); // Used for printf statements. Using fd::path is problematic because it 8-bit char in Linux and 16-bit char in Windows. + fs::path localWork = filename; // Try to get a task from the local worktodo- file. if (optional task = bestTask(localWork, args.smallest)) { return task; } - if (args.masterDir.empty()) { log("No work to do found. Add work to %s.\n", localWork.c_str()); return {}; } + if (args.masterDir.empty()) { log("No work to do found. Add work to %s.\n", filename.c_str()); return {}; } - fs::path worktodo = args.masterDir / "worktodo.txt"; + filename = args.masterDir / "worktodo.txt"; + fs::path worktodo = filename; /* We need to aquire a task from the global worktodo.txt, and "atomically" @@ -156,7 +158,7 @@ optional getWork(Args& args, i32 instance) { if (!found) { return {}; } } - log("Could not extract a task from '%s'\n", worktodo.string().c_str()); + log("Could not extract a task from '%s'\n", filename.c_str()); // must be tough luck to be preempted twice while mutating the global worktodo assert(false); return {}; From 53b35ff7759e745277e317aca431390c33dfc0cc Mon Sep 17 00:00:00 2001 From: george Date: Sat, 9 May 2026 22:45:08 +0000 Subject: [PATCH 069/214] Trying again with Windows pathname bug fix. --- src/Worktodo.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 7c31361c..63762490 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -117,8 +117,8 @@ optional getWork(Args& args, i32 instance) { if (args.masterDir.empty()) { log("No work to do found. Add work to %s.\n", filename.c_str()); return {}; } - filename = args.masterDir / "worktodo.txt"; - fs::path worktodo = filename; + filename = "worktodo.txt"; + fs::path worktodo = args.masterDir / filename; /* We need to aquire a task from the global worktodo.txt, and "atomically" From 73172dfdd2b532b7c3e0189cf6f7fdc4e82b0c1a Mon Sep 17 00:00:00 2001 From: george Date: Wed, 13 May 2026 04:56:13 +0000 Subject: [PATCH 070/214] Added ETA to LL status lines --- src/Gpu.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 42a0caa3..7fd00a7b 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -2235,7 +2235,7 @@ LLResult Gpu::isPrimeLL(const Task& task) { float secsPerIt = iterationTimer.reset(k); queue->setSquareTime((int) (secsPerIt * 1'000'000)); - log("%9" PRIu64 " %016" PRIx64 " %4.0f\n", k, res64, secsPerIt * 1'000'000); + log("%9" PRIu64 " %016" PRIx64 " %4.0f ETA %s\n", k, res64, secsPerIt * 1'000'000, getETA(k, kEnd, secsPerIt).c_str()); if (k >= kEnd) { return {isAllZero, res64}; } From 445f17a38ba6ee1caf09e167e438051cb472534c Mon Sep 17 00:00:00 2001 From: george Date: Thu, 14 May 2026 03:02:27 +0000 Subject: [PATCH 071/214] Outpu per-iteration timing down to the 1/10th of a microsecond. Batalov's suggestion. --- src/Gpu.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 7fd00a7b..f7f57c24 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -522,6 +522,17 @@ string toHex(const vector& v) { return s; } +string formatSecsPerIter(float secsPerIter) { + char buf[64]; + float usecsPerIter = secsPerIter * 1.0e6f; // Convert to micro-seconds + if (usecsPerIter > 1000.0f) { + snprintf(buf, sizeof(buf), "%4.0f", usecsPerIter); + } else { + snprintf(buf, sizeof(buf), "%6.1f", usecsPerIter); + } + return string(buf); +} + } // namespace // -------- @@ -1315,7 +1326,7 @@ Words Gpu::expExp2(const Words& A, u32 n) { queue->finish(); if (k % logStep == 0) { float secsPerIt = timer.reset(k); - log("%u / %u, %.0f us/it\n", k, n, secsPerIt * 1'000'000); + log("%u / %u, %s us/it\n", k, n, formatSecsPerIter(secsPerIt).c_str()); } } return readData(); @@ -1556,9 +1567,9 @@ string RoeInfo::toString() const { static string makeLogStr(const string& status, u64 k, u64 res, float secsPerIt, u64 nIters) { char buf[256]; - snprintf(buf, sizeof(buf), "%2s %9" PRIu64 " %016" PRIx64 " %4.0f ETA %s; ", + snprintf(buf, sizeof(buf), "%2s %9" PRIu64 " %016" PRIx64 " %s ETA %s; ", status.c_str(), k, res, /* k / float(nIters) * 100, */ - secsPerIt * 1'000'000, getETA(k, nIters, secsPerIt).c_str()); + formatSecsPerIter(secsPerIt).c_str(), getETA(k, nIters, secsPerIt).c_str()); return buf; } @@ -2109,7 +2120,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { elapsedBefore + elapsedTimer.at()}); }); - log(" %9" PRIu64 " %016" PRIx64 " %4.0f\n", k, res, /*k / float(kEndEnd) * 100*,*/ secsPerIt * 1'000'000); + log(" %9" PRIu64 " %016" PRIx64 " %s\n", k, res, formatSecsPerIter(secsPerIt).c_str()); RoeInfo carryStats = readCarryStats(); if (carryStats.N) { u32 m = ldexp(carryStats.max, 32); @@ -2235,7 +2246,7 @@ LLResult Gpu::isPrimeLL(const Task& task) { float secsPerIt = iterationTimer.reset(k); queue->setSquareTime((int) (secsPerIt * 1'000'000)); - log("%9" PRIu64 " %016" PRIx64 " %4.0f ETA %s\n", k, res64, secsPerIt * 1'000'000, getETA(k, kEnd, secsPerIt).c_str()); + log("%9" PRIu64 " %016" PRIx64 " %s ETA %s\n", k, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); if (k >= kEnd) { return {isAllZero, res64}; } @@ -2295,7 +2306,7 @@ array Gpu::isCERT(const Task& task) { float secsPerIt = iterationTimer.reset(k); queue->setSquareTime((int) (secsPerIt * 1'000'000)); - log("%7u / %7u %016" PRIx64 " %4.0f ETA %s\n", k, kEnd, res64, secsPerIt * 1'000'000, getETA(k, kEnd, secsPerIt).c_str()); + log("%7u / %7u %016" PRIx64 " %s ETA %s\n", k, kEnd, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); if (k >= kEnd) { fs::remove (fname); From 59aaf73d8d60a5c5dad657c17e9abea052326c14 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 15 May 2026 04:51:20 +0000 Subject: [PATCH 072/214] Moved Context to GpuCommon. Made Gpu class responsible for creating command queues. Prep work eeded for trying dispatching kernels to multiple queues. --- src/Gpu.cpp | 51 +++++++++++++++---------------- src/Gpu.h | 8 ++--- src/GpuCommon.h | 2 ++ src/Task.cpp | 4 +-- src/Task.h | 3 +- src/main.cpp | 14 ++++----- src/tune.cpp | 80 ++++++++++++++++++++++++------------------------- src/tune.h | 4 +-- 8 files changed, 82 insertions(+), 84 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index f7f57c24..27fa8a73 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -537,8 +537,8 @@ string formatSecsPerIter(float secsPerIter) { // -------- -unique_ptr Gpu::make(Queue* q, u64 E, GpuCommon shared, FFTConfig fftConfig, const vector& extraConf, bool logFftSize) { - return make_unique(q, shared, fftConfig, E, extraConf, logFftSize); +unique_ptr Gpu::make(u64 E, GpuCommon shared, FFTConfig fftConfig, const vector& extraConf, bool logFftSize) { + return make_unique(shared, fftConfig, E, extraConf, logFftSize); } Gpu::~Gpu() { @@ -733,8 +733,8 @@ string Gpu::kernelDefines(enum WHICH_KERNEL_TYPE which_kernel) { #define ROE_SIZE 100000 #define CARRY_SIZE 100000 -Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : - queue(q), +Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : + shared(s), background{shared.background}, args{*shared.args}, E(E), @@ -747,9 +747,10 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& nW(fft.shape.nW()), nH(fft.shape.nH()), useLongCarry{args.carry == CARRY_64}, - compiler{args, queue->context, clDefines(args, queue->context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size, wmul)}, + queue{*shared.context, args.profile}, + compiler{args, shared.context, clDefines(args, shared.context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size, wmul)}, -#define K(name, ...) name(#name, &compiler, profile.make(#name), queue, __VA_ARGS__) +#define K(name, ...) name(#name, &compiler, profile.make(#name), &queue, __VA_ARGS__) K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), (kernelDefines(KFP) + numCudaRegisters(MIDIN)).c_str()), K(kfftHin, "ffthin.cl", "fftHin", hN / nH, kernelDefines(KFP).c_str()), @@ -827,11 +828,11 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& bufTrigM{shared.bufCache->middleTrig(shared.args, fft, SMALL_H, BIG_H / SMALL_H, WIDTH)}, bufTrigW{shared.bufCache->smallTrig(shared.args, fft, WIDTH, nW, fft.shape.middle, SMALL_H, nH, tail_single_wide)}, - weights{genWeights(fft, E, WIDTH, BIG_H, nW, isNvidiaGpu(q->context->deviceId()))}, - bufConstWeights{q->context, std::move(weights.weightsConstIF)}, - bufWeights{q->context, std::move(weights.weightsIF)}, + weights{genWeights(fft, E, WIDTH, BIG_H, nW, isNvidiaGpu(shared.context->deviceId()))}, + bufConstWeights{shared.context, std::move(weights.weightsConstIF)}, + bufWeights{shared.context, std::move(weights.weightsIF)}, -#define BUF(name, ...) name{profile.make(#name), queue, __VA_ARGS__} +#define BUF(name, ...) name{profile.make(#name), &queue, __VA_ARGS__} // GPU Buffers containing integer data. Since this buffer is type i64, if fft.WordSize < 8 then we need less memory allocated. BUF(bufData, N * fft.WordSize / sizeof(Word)), @@ -944,8 +945,8 @@ Gpu::Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& selftestTrig(); } - queue->setSquareKernels(1 + 3 * (fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61)); - queue->finish(); + queue.setSquareKernels(1 + 3 * (fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61)); + queue.finish(); } @@ -1039,7 +1040,7 @@ void Gpu::carryFusedLL(Buffer& out, Buffer& in) { void Gpu::measureTransferSpeed() { u32 SIZE_MB = 16; vector data(SIZE_MB * 1024 * 1024, 1); - Buffer buf{profile.make("DMA"), queue, SIZE}; + Buffer buf{profile.make("DMA"), &queue, SIZE}; Timer t; for (int i = 0; i < 4; ++i) { @@ -1049,11 +1050,11 @@ void Gpu::measureTransferSpeed() { for (int i = 0; i < 4; ++i) { buf.read(data); - // queue->finish(); + // queue.finish(); log("buffer READ : %f GB/s\n", double(SIZE / 1024 / 1024) * sizeof(double) / (1024 * t.reset())); } - queue->finish(); + queue.finish(); } #endif @@ -1063,7 +1064,7 @@ u32 Gpu::updateCarryPos(u32 bit) { vector> Gpu::makeBufVector(u32 size) { vector> r; - for (u32 i = 0; i < size; ++i) { r.emplace_back(timeBufVect, queue, N); } + for (u32 i = 0; i < size; ++i) { r.emplace_back(timeBufVect, &queue, N); } return r; } @@ -1323,7 +1324,7 @@ Words Gpu::expExp2(const Words& A, u32 n) { u32 its = std::min(blockSize, n - k); squareLoop(bufData, 0, its); k += its; - queue->finish(); + queue.finish(); if (k % logStep == 0) { float secsPerIt = timer.reset(k); log("%u / %u, %s us/it\n", k, n, formatSecsPerIter(secsPerIt).c_str()); @@ -1661,7 +1662,7 @@ void Gpu::selftestTrig() { log("TRIG norm: up %d, down %d\n", oneUp, oneDown); #endif - if (isAmdGpu(queue->context->deviceId())) { + if (isAmdGpu(shared.context->deviceId())) { vector WHATS {"V_NOP", "V_ADD_I32", "V_FMA_F32", "V_ADD_F64", "V_FMA_F64", "V_MUL_F64", "V_MAD_U64_U32"}; for (int w = 0; w < int(WHATS.size()); ++w) { const int what = w; @@ -1957,11 +1958,11 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest leadIn = leadOut; ++k; } - queue->finish(); + queue.finish(); if (Signal::stopRequested()) { throw "stop requested"; } Timer t; - queue->setSquareTime(0); // Busy wait on nVidia to get the most accurate timings while tuning + queue.setSquareTime(0); // Busy wait on nVidia to get the most accurate timings while tuning while (true) { while (k % blockSize < blockSize-1) { square(bufData, bufData, leadIn, leadOut); @@ -1978,7 +1979,7 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest leadIn = LEAD_MIDDLE; if (Signal::stopRequested()) { throw "stop requested"; } } - queue->finish(); + queue.finish(); double secsPerIt = t.reset() / (iters - warmup); if (Signal::stopRequested()) { throw "stop requested"; } @@ -2104,7 +2105,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { u64 res = dataResidue(); float secsPerIt = iterationTimer.reset(k); - queue->setSquareTime((int) (secsPerIt * 1'000'000)); + queue.setSquareTime((int) (secsPerIt * 1'000'000)); vector rawCheck = readChecked(bufCheck); if (rawCheck.empty()) { @@ -2166,7 +2167,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { logTimeKernels(); if (doStop) { - queue->finish(); + queue.finish(); throw "stop requested"; } @@ -2245,7 +2246,7 @@ LLResult Gpu::isPrimeLL(const Task& task) { } float secsPerIt = iterationTimer.reset(k); - queue->setSquareTime((int) (secsPerIt * 1'000'000)); + queue.setSquareTime((int) (secsPerIt * 1'000'000)); log("%9" PRIu64 " %016" PRIx64 " %s ETA %s\n", k, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); if (k >= kEnd) { return {isAllZero, res64}; } @@ -2305,7 +2306,7 @@ array Gpu::isCERT(const Task& task) { u64 res64 = (u64(data[1]) << 32) | data[0]; float secsPerIt = iterationTimer.reset(k); - queue->setSquareTime((int) (secsPerIt * 1'000'000)); + queue.setSquareTime((int) (secsPerIt * 1'000'000)); log("%7u / %7u %016" PRIx64 " %s ETA %s\n", k, kEnd, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); if (k >= kEnd) { diff --git a/src/Gpu.h b/src/Gpu.h index 9553fec1..c3e6f766 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -83,7 +83,7 @@ struct Weights { }; class Gpu { - Queue* queue; + GpuCommon shared; Background* background; public: @@ -106,6 +106,7 @@ class Gpu { Profile profile{}; + Queue queue; KernelCompiler compiler; /* Kernels for FFT_FP64 or FFT_FP32 */ @@ -287,9 +288,8 @@ class Gpu { void selftestTrig(); public: - Gpu(Queue* q, GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize); - static unique_ptr make(Queue* q, u64 E, GpuCommon shared, FFTConfig fft, - const vector& extraConf = {}, bool logFftSize = true); + Gpu(GpuCommon shared, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize); + static unique_ptr make(u64 E, GpuCommon shared, FFTConfig fft, const vector& extraConf = {}, bool logFftSize = true); ~Gpu(); diff --git a/src/GpuCommon.h b/src/GpuCommon.h index 5a9f4756..76ba072b 100644 --- a/src/GpuCommon.h +++ b/src/GpuCommon.h @@ -2,6 +2,7 @@ #pragma once +class Context; class Args; class TrigBufCache; class Background; @@ -9,6 +10,7 @@ class Background; // Data that's normally shared between Gpu instances class GpuCommon { public: + Context* context; Args* args; TrigBufCache* bufCache; Background* background; diff --git a/src/Task.cpp b/src/Task.cpp index 95c23e98..28794956 100644 --- a/src/Task.cpp +++ b/src/Task.cpp @@ -209,7 +209,7 @@ void Task::writeResultCERT(FFTConfig fft, const Args &args, u32 instance, array File::append(resultsFile, s + '\n'); } -void Task::execute(GpuCommon shared, Queue *q, u32 instance) { +void Task::execute(GpuCommon shared, u32 instance) { if (kind == VERIFY) { exponent = proof::getInfo(verifyPath).exp; } assert(exponent); @@ -232,7 +232,7 @@ void Task::execute(GpuCommon shared, Queue *q, u32 instance) { FFTConfig fft = FFTConfig::bestFit(*shared.args, exponent, shared.args->fftSpec); - auto gpu = Gpu::make(q, exponent, shared, fft); + auto gpu = Gpu::make(exponent, shared, fft); if (kind == VERIFY) { Proof proof{Proof::load(verifyPath)}; diff --git a/src/Task.h b/src/Task.h index 4c130446..588acd3a 100644 --- a/src/Task.h +++ b/src/Task.h @@ -12,7 +12,6 @@ class Args; class Result; class Context; -class Queue; class TrigBufCache; class Task { @@ -26,7 +25,7 @@ class Task { u32 squarings; // For CERTs string verifyPath; // For Verify - void execute(GpuCommon shared, Queue* q, u32 instance); + void execute(GpuCommon shared, u32 instance); void writeResultPRP(FFTConfig fft, const Args&, u32 instance, bool isPrime, u64 res64, const std::string& res2048, u32 nErrors, const fs::path& proofPath) const; void writeResultLL(FFTConfig fft, const Args&, u32 instance, bool isPrime, u64 res64) const; diff --git a/src/main.cpp b/src/main.cpp index 259b7f9b..efdfc38d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -20,7 +20,7 @@ #include // #include from GCC-13 onwards -void gpuWorker(GpuCommon shared, Queue *q, i32 instance) { +void gpuWorker(GpuCommon shared, i32 instance) { // LogContext context{(instance ? shared.args->tailDir() : ""s) + to_string(instance) + ' '}; // log("Starting worker %d\n", instance); if (instance > 0) { @@ -29,7 +29,7 @@ void gpuWorker(GpuCommon shared, Queue *q, i32 instance) { } try { - while (auto task = Worktodo::getTask(*shared.args, instance)) { task->execute(shared, q, instance); } + while (auto task = Worktodo::getTask(*shared.args, instance)) { task->execute(shared, instance); } } catch (const char *mes) { log("Exception \"%s\"\n", mes); } catch (const string& mes) { @@ -97,14 +97,14 @@ int main(int argc, char **argv) { Signal signal; Background background; GpuCommon shared; + shared.context = &context; shared.args = &args; TrigBufCache bufCache{&context}; shared.bufCache = &bufCache; shared.background = &background; if (args.doCtune || args.doTune || args.doZtune || args.carryTune) { - Queue q(context, args.profile); - Tune tune{&q, shared}; + Tune tune{shared}; if (args.doCtune) { tune.ctune(); @@ -117,13 +117,11 @@ int main(int argc, char **argv) { } } else { { - vector queues; - for (int i = 0; i < int(args.workers); ++i) { queues.emplace_back(context, args.profile); } vector threads; for (int i = 1; i < int(args.workers); ++i) { - threads.emplace_back(gpuWorker, shared, &queues[i], i); + threads.emplace_back(gpuWorker, shared, i); } - gpuWorker(shared, &queues[0], 0); + gpuWorker(shared, 0); } // log("No more work. Add work to worktodo.txt , see -h for details.\n"); diff --git a/src/tune.cpp b/src/tune.cpp index c00956f0..dea4f864 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -176,7 +176,7 @@ float Tune::zForBpw(float bpw, FFTConfig fft, u32 count) { u64 exponent = (count == 1) ? primes.prevPrime(fft.size() * bpw) : primes.nextPrime(fft.size() * bpw); float total_z = 0.0f; for (u32 i = 0; i < count; i++, exponent = primes.nextPrime (exponent + 1)) { - auto [ok, res, roeSq, roeMul] = Gpu::make(q, exponent, shared, fft, {}, false)->measureROE(true); + auto [ok, res, roeSq, roeMul] = Gpu::make(exponent, shared, fft, {}, false)->measureROE(true); float z = roeSq.z(); total_z += z; log("Zforbpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); @@ -250,7 +250,7 @@ void Tune::carryTune() { const float mid = fft.shape.carry32BPW(); for (float bpw : {mid - 0.05f, mid + 0.05f}) { u64 exponent = primes.nearestPrime(fft.size() * bpw); - auto [ok, carry] = Gpu::make(q, exponent, shared, fft, {}, false)->measureCarry(); + auto [ok, carry] = Gpu::make(exponent, shared, fft, {}, false)->measureCarry(); m = carry.max; if (!ok) { log("Error %s at %f\n", fft.spec().c_str(), bpw); } zv.push_back(carry.z()); @@ -309,7 +309,7 @@ void Tune::ctune() { for (u32 k = i + 1; k < configsVect.size(); ++k) { add(c, configsVect[k][bestPos[k]]); } - auto cost = Gpu::make(q, exponent, shared, fft, c, false)->timePRP(); + auto cost = Gpu::make(exponent, shared, fft, c, false)->timePRP(); bool isBest = (cost < best.cost); if (isBest) { @@ -342,8 +342,8 @@ void Tune::tune() { vector shapes = FFTShape::multiSpec(args->fftSpec); // There are some options and variants that are different based on GPU manufacturer - bool AMDGPU = isAmdGpu(q->context->deviceId()); - bool NVIDIAGPU = isNvidiaGpu(q->context->deviceId()); + bool AMDGPU = isAmdGpu(shared.context->deviceId()); + bool NVIDIAGPU = isNvidiaGpu(shared.context->deviceId()); int NO_ASM = args->value("NO_ASM", 0); bool tune_config = 1; @@ -410,11 +410,11 @@ void Tune::tune() { log("Checking whether this GPU is better suited for double-precision FFTs or integer NTTs.\n"); defaultFFTShape = FFTShape(FFT64, 512, 16, 512); FFTConfig fft{defaultFFTShape, 101, CARRY_32}; - double fp64_time = Gpu::make(q, 141000001, shared, fft, {}, false)->timePRP(quick); + double fp64_time = Gpu::make(141000001, shared, fft, {}, false)->timePRP(quick); log("Time for FP64 FFT %12s is %6.1f\n", fft.spec().c_str(), fp64_time); defaultNTTShape = FFTShape(FFT3161, 512, 8, 512); FFTConfig ntt{defaultNTTShape, 202, CARRY_AUTO}; - double ntt_time = Gpu::make(q, 141000001, shared, ntt, {}, false)->timePRP(quick); + double ntt_time = Gpu::make(141000001, shared, ntt, {}, false)->timePRP(quick); log("Time for M31*M61 NTT %12s is %6.1f\n", ntt.spec().c_str(), ntt_time); if (fp64_time < ntt_time) { defaultShape = &defaultFFTShape; @@ -463,7 +463,7 @@ void Tune::tune() { for (u32 in_sizex : {8, 16, 32}) { args->flags["IN_WG"] = to_string(in_wg); args->flags["IN_SIZEX"] = to_string(in_sizex); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using IN_WG=%u, IN_SIZEX=%u is %6.1f\n", fft.spec().c_str(), in_wg, in_sizex, cost); if (in_wg == current_in_wg && in_sizex == current_in_sizex) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_in_wg = in_wg; best_in_sizex = in_sizex; } @@ -485,7 +485,7 @@ void Tune::tune() { for (u32 out_sizex : {8, 16, 32}) { args->flags["OUT_WG"] = to_string(out_wg); args->flags["OUT_SIZEX"] = to_string(out_sizex); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using OUT_WG=%u, OUT_SIZEX=%u is %6.1f\n", fft.spec().c_str(), out_wg, out_sizex, cost); if (out_wg == current_out_wg && out_sizex == current_out_sizex) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_out_wg = out_wg; best_out_sizex = out_sizex; } @@ -508,7 +508,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 pad : {0, 64, 128, 256, 512}) { args->flags["PAD"] = to_string(pad); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using PAD=%u is %6.1f\n", fft.spec().c_str(), pad, cost); if (pad == current_pad) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_pad = pad; } @@ -528,7 +528,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 middle_in_lds_transpose : {0, 1}) { args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(middle_in_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MIDDLE_IN_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_in_lds_transpose, cost); if (middle_in_lds_transpose == current_middle_in_lds_transpose) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_in_lds_transpose = middle_in_lds_transpose; } @@ -548,7 +548,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 middle_out_lds_transpose : {0, 1}) { args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(middle_out_lds_transpose); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MIDDLE_OUT_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_out_lds_transpose, cost); if (middle_out_lds_transpose == current_middle_out_lds_transpose) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_out_lds_transpose = middle_out_lds_transpose; } @@ -572,7 +572,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 inplace : {0, 1}) { args->flags["INPLACE"] = to_string(inplace); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using INPLACE=%u is %6.1f\n", fft.spec().c_str(), inplace, cost); if (inplace == current_inplace) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_inplace = inplace; } @@ -596,7 +596,7 @@ void Tune::tune() { for (u32 fft_load : {0, 1, 2, 3, 4}) { if (fft_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10 * 10 + fft_load); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FFT load=%u is %6.1f\n", fft.spec().c_str(), fft_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_load = fft_load; } } @@ -614,7 +614,7 @@ void Tune::tune() { for (u32 fft_store : {0, 1, 2, 3, 4}) { if (fft_store >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["STORES"] = to_string(stores / 10 * 10 + fft_store); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FFT store=%u is %6.1f\n", fft.spec().c_str(), fft_store, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_store = fft_store; } } @@ -635,7 +635,7 @@ void Tune::tune() { u32 cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; args->flags["LOADS"] = to_string(loads / 100 * 100 + cs_load * 10 + loads % 10); args->flags["STORES"] = to_string(stores / 100 * 100 + cs_store * 10 + stores % 10); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using carry shuttle load=%u, store=%u is %6.1f\n", fft.spec().c_str(), cs_load, cs_store, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_cs_load = cs_load; best_cs_store = cs_store; } } @@ -655,7 +655,7 @@ void Tune::tune() { for (u32 trig_load : {0, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 1000 * 1000 + trig_load * 100 + loads % 100); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig frequently used load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } } @@ -673,7 +673,7 @@ void Tune::tune() { for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10000 * 10000 + trig_load * 1000 + loads % 1000); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig several uses load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } } @@ -691,7 +691,7 @@ void Tune::tune() { for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(trig_load * 10000 + loads % 10000); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig used once load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } } @@ -717,7 +717,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 fast_barrier : {0, 1}) { args->flags["FAST_BARRIER"] = to_string(fast_barrier); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FAST_BARRIER=%u is %6.1f\n", fft.spec().c_str(), fast_barrier, cost); if (fast_barrier == current_fast_barrier) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fast_barrier = fast_barrier; } @@ -737,7 +737,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tail_kernels : {0, 1, 2, 3}) { args->flags["TAIL_KERNELS"] = to_string(tail_kernels); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_KERNELS=%u is %6.1f\n", fft.spec().c_str(), tail_kernels, cost); if (tail_kernels == current_tail_kernels) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_kernels = tail_kernels; } @@ -760,7 +760,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tail_trigs : {0, 1, 2}) { args->flags["TAIL_TRIGS"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -781,7 +781,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tail_trigs : {0, 1}) { args->flags["TAIL_TRIGS31"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS31=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -802,7 +802,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tail_trigs : {0, 1, 2}) { args->flags["TAIL_TRIGS32"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS32=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -823,7 +823,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tail_trigs : {0, 1}) { args->flags["TAIL_TRIGS61"] = to_string(tail_trigs); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS61=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -843,7 +843,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -864,7 +864,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN31"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN31=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -885,7 +885,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN32"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN32=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -906,7 +906,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN61"] = to_string(tabmul_chain); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN61=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -927,7 +927,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 modm31 : {0, 1, 2}) { args->flags["MODM31"] = to_string(modm31); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MODM31=%u is %6.1f\n", fft.spec().c_str(), modm31, cost); if (modm31 == current_modm31) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_modm31 = modm31; } @@ -947,7 +947,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 unroll_w : {0, 1}) { args->flags["UNROLL_W"] = to_string(unroll_w); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using UNROLL_W=%u is %6.1f\n", fft.spec().c_str(), unroll_w, cost); if (unroll_w == current_unroll_w) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_unroll_w = unroll_w; } @@ -967,7 +967,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 unroll_h : {0, 1}) { args->flags["UNROLL_H"] = to_string(unroll_h); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using UNROLL_H=%u is %6.1f\n", fft.spec().c_str(), unroll_h, cost); if (unroll_h == current_unroll_h) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_unroll_h = unroll_h; } @@ -987,7 +987,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 zerohack_w : {0, 1}) { args->flags["ZEROHACK_W"] = to_string(zerohack_w); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using ZEROHACK_W=%u is %6.1f\n", fft.spec().c_str(), zerohack_w, cost); if (zerohack_w == current_zerohack_w) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_zerohack_w = zerohack_w; } @@ -1007,7 +1007,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 zerohack_h : {0, 1}) { args->flags["ZEROHACK_H"] = to_string(zerohack_h); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using ZEROHACK_H=%u is %6.1f\n", fft.spec().c_str(), zerohack_h, cost); if (zerohack_h == current_zerohack_h) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_zerohack_h = zerohack_h; } @@ -1027,7 +1027,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 wmul : {1, 2, 4}) { args->flags["WMUL"] = to_string(wmul); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using WMUL=%u is %6.1f\n", fft.spec().c_str(), wmul, cost); if (wmul == current_wmul) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_wmul = wmul; } @@ -1048,7 +1048,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 noreg : {0, 1}) { args->flags["NOREG"] = to_string(noreg); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using NOREG=%u is %6.1f\n", fft.spec().c_str(), noreg, cost); if (noreg == current_noreg) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_noreg = noreg; } @@ -1069,7 +1069,7 @@ void Tune::tune() { double current_cost = -1.0; for (u32 biglit : {0, 1}) { args->flags["BIGLIT"] = to_string(biglit); - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using BIGLIT=%u is %6.1f\n", fft.spec().c_str(), biglit, cost); if (biglit == current_biglit) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_biglit = biglit; } @@ -1198,7 +1198,7 @@ skip_1K_256 = 0; if (w == 0 && !AMDGPU) continue; if (w == 0 && test.width > 1024) continue; FFTConfig fft{test, variant_WMH (w, 0, 1), CARRY_32}; - cost = Gpu::make(q, primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(adjusted_quick); + cost = Gpu::make(primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(adjusted_quick); log("Fast width search %6.1f %12s\n", cost, fft.spec().c_str()); if (min_cost < 0.0 || cost < min_cost) { min_cost = cost; fastest_width = w; } } @@ -1220,7 +1220,7 @@ skip_1K_256 = 0; if (h == 0 && !AMDGPU) continue; if (h == 0 && test.height > 1024) continue; FFTConfig fft{test, variant_WMH (1, 0, h), CARRY_32}; - cost = Gpu::make(q, primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(quick); + cost = Gpu::make(primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(quick); log("Fast height search %6.1f %12s\n", cost, fft.spec().c_str()); if (min_cost < 0.0 || cost < min_cost) { min_cost = cost; fastest_height = h; } } @@ -1250,7 +1250,7 @@ skip_1K_256 = 0; // Skip middle = 1, CARRY_32 if maximum exponent would be the same as middle = 0, CARRY_32 if (variant_M(variant) > 0 && carry == CARRY_32 && fft.maxExp() <= FFTConfig{shape, variant - 10, CARRY_32}.maxExp()) continue; - double cost = Gpu::make(q, exponent, shared, fft, {}, false)->timePRP(quick); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); bool isUseful = TuneEntry{cost, fft}.update(results); log("%c %6.1f %12s %9" PRIu64 "\n", isUseful ? '*' : ' ', cost, fft.spec().c_str(), fft.maxExp()); if (isUseful) TuneEntry::writeTuneFile(results); diff --git a/src/tune.h b/src/tune.h index a64f2100..b3ce6119 100644 --- a/src/tune.h +++ b/src/tune.h @@ -9,7 +9,6 @@ #include #include -class Queue; class GpuCommon; class RoeInfo; class Gpu; @@ -18,7 +17,6 @@ using TuneConfig = vector; class Tune { private: - Queue *q; GpuCommon shared; Primes primes; @@ -26,7 +24,7 @@ class Tune { float zForBpw(float bpw, FFTConfig fft, u32); public: - Tune(Queue *q, GpuCommon shared) : q{q}, shared{shared} {} + Tune(GpuCommon shared) : shared{shared} {} // Find the max-BPW for each FFT void ztune(); From e213b6f497bff78f4b3f7d7ac5f172421346d5cf Mon Sep 17 00:00:00 2001 From: george Date: Fri, 15 May 2026 06:03:12 +0000 Subject: [PATCH 073/214] Potential fix for the "Read GPU error" problem! --- src/Buffer.h | 11 ++++++----- src/Gpu.cpp | 1 + src/cl/etc.cl | 2 -- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Buffer.h b/src/Buffer.h index 7866975d..4bd57c6d 100644 --- a/src/Buffer.h +++ b/src/Buffer.h @@ -35,9 +35,10 @@ class Buffer { , tInfo{tInfo} {} - void fill(T value, u32 len) { - assert(len <= size); - queue->fillBuf(get(), value, (len ? len : size) * sizeof(T), tInfo); + void fill(T value, u32 sizeOrFull = 0) { + assert(sizeOrFull <= size); + auto fillSize = sizeOrFull ? sizeOrFull : size; + queue->fillBuf(get(), value, fillSize * sizeof(T), tInfo); } public: @@ -84,8 +85,8 @@ class Buffer { void write(const vector& vect) { queue->write(get(), vect, tInfo); } - void zero(size_t len = 0) { - fill(0, len); + void zero(size_t sizeOrFull = 0) { + fill(0, sizeOrFull); } void set(T value) { diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 27fa8a73..22fa5e71 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -1123,6 +1123,7 @@ static bool isAllZero(vector v) { return std::all_of(v.begin(), v.end(), [](T // Read from GPU, verifying the transfer with a sum, and retry on failure. vector Gpu::readChecked(Buffer& buf) { for (int nRetry = 0; nRetry < 3; ++nRetry) { + bufSumOut.zero(); sum64(bufSumOut, u32(buf.size * sizeof(Word)), buf); vector expectedVect(1); diff --git a/src/cl/etc.cl b/src/cl/etc.cl index ae4fd857..2f5fb43a 100644 --- a/src/cl/etc.cl +++ b/src/cl/etc.cl @@ -17,8 +17,6 @@ KERNEL(32) readResidue(P(Word2) out, CP(Word2) in) { #if SUM64 KERNEL(64) sum64(global ulong* out, u32 sizeBytes, global ulong* in) { - if (get_global_id(0) == 0) { out[0] = 0; } - ulong sum = 0; for (i32 p = get_global_id(0); p < sizeBytes / sizeof(u64); p += get_global_size(0)) { sum += in[p]; From d76e4a70ce02b88d426e99fa20b859b93b423d31 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 17 May 2026 11:38:46 +0000 Subject: [PATCH 074/214] Windows asserted using _IOLBF, 0 in setvbuf. Switched to _IONBF. --- src/File.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/File.cpp b/src/File.cpp index 2ab37b7c..1360b3f3 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -22,7 +22,7 @@ File::File(const std::filesystem::path& path, const string& mode, bool throwOnEr // setlinebuf(f); //#endif // tdulcet's suggested portable replacement for the lines above - setvbuf(f, nullptr, _IOLBF, 0); + setvbuf(f, nullptr, _IONBF, 0); } } From e98adb1c623d2005944304a537ab1e6927cb6b67 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 17 May 2026 11:48:08 +0000 Subject: [PATCH 075/214] First cut at multiple command queues. Off by default. Can be turned on with -use MULTI_Q. Slower on 5070Ti due to increase L2 cache hits. Significantly faster on a 5080. No change on a 4080 which has same L2 cache size as a 5080. Needs more study. --- src/Gpu.cpp | 134 ++++++++++++++++++++++++++++++++++++++- src/Gpu.h | 5 +- src/Kernel.h | 5 +- src/Queue.cpp | 8 ++- src/Queue.h | 6 +- src/clwrap.cpp | 6 ++ src/clwrap.h | 1 + src/cuda/clwrap_cuda.cpp | 10 ++- 8 files changed, 166 insertions(+), 9 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 22fa5e71..335b69bd 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -345,7 +345,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< defines += toDefine("FFT_VARIANT", fft.variant); defines += toDefine("MAXBPW", (u32)(fft.maxBpw() * 100.0f)); - if (fft.FFT_FP64 | fft.FFT_FP32) { + if (fft.FFT_FP64 || fft.FFT_FP32) { defines += toDefine("WEIGHT_STEP", weightM1(N, E, fft.shape.height * fft.shape.middle, 0, 0, 1)); defines += toDefine("IWEIGHT_STEP", invWeightM1(N, E, fft.shape.height * fft.shape.middle, 0, 0, 1)); if (fft.FFT_FP64) defines += toDefine("TAILT", root1Fancy(fft.shape.height * 2, 1)); @@ -748,6 +748,7 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo nH(fft.shape.nH()), useLongCarry{args.carry == CARRY_64}, queue{*shared.context, args.profile}, + auxQueues{}, compiler{args, shared.context, clDefines(args, shared.context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size, wmul)}, #define K(name, ...) name(#name, &compiler, profile.make(#name), &queue, __VA_ARGS__) @@ -945,10 +946,127 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo selftestTrig(); } - queue.setSquareKernels(1 + 3 * (fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61)); + // If MULTI_Q option is set there are fewer kernels executed in the main queue, but there are some additional syncEvents and waits. + // That's a total of 4 kernels (carryFused, MidIn/Out, TailSquare) plus 1 syncEvent plus 1 or more syncWaits. + // If MULTI_Q option is not set there is carryFused + MidIn/Out and TailSquare for each NTT modulus. + // NOTE: We dont take into account the optional tailSquareZero kernel. We theoretically should. + if (args.value("MULTI_Q", 0)) + queue.setSquareKernels(5 + ((fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61) - 1)); + else + queue.setSquareKernels(1 + 3 * (fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61)); queue.finish(); } +// Optionallly split some of the MiddleIn/Tail/MiddleOut kernels off od executing on the main queue to run on an auxiliary queue. +// This will increase GPU occupancy but will negatively impact L2 cache coherency. +// If the L2 cache is large enough so that all FFT data fits in the cache, this ought to be a win. +// If the L2 cache is small enough such that L2 cache hits are very low anyway, this might be a win. + +void Gpu::splitQueue(void) { + + // If MULTI_Q -use not set, return + if (!args.value("MULTI_Q", 0)) return; + + // Create aux queues. For now, we only have one auxiliary queue. We could do more. + if (auxQueues.size() == 0) { + auxQueues.push_back(Queue{*shared.context, args.profile, true}); + } + + // Queue a sync event in the main queue. Have all auxiliary queues wait on the event. + EventHolder event = queue.createSyncEvent(); + for (size_t i = 0; i < auxQueues.size(); ++i) { + auxQueues[i].waitForSyncEvent(&event); + } + + // Assign kernels to running on the main queue or an auxiliary queue + + int which_queue = -1; + + // For no particularly good reason, put a kernel that operates on 64-bit vaules in the main queue. + if (fft.NTT_GF61) { + if (which_queue != -1) { + kfftWGF61.setQueue(&auxQueues[which_queue]); + kfftMidInGF61.setQueue(&auxQueues[which_queue]); + kfftMidOutGF61.setQueue(&auxQueues[which_queue]); + ktailSquareZeroGF61.setQueue(&auxQueues[which_queue]); + ktailSquareGF61.setQueue(&auxQueues[which_queue]); + ktailMulGF61.setQueue(&auxQueues[which_queue]); + ktailMulLowGF61.setQueue(&auxQueues[which_queue]); + } + which_queue++; + } + + if (fft.FFT_FP64 || fft.FFT_FP32) { + if (which_queue != -1) { + kfftW.setQueue(&auxQueues[which_queue]); + kfftMidIn.setQueue(&auxQueues[which_queue]); + kfftMidOut.setQueue(&auxQueues[which_queue]); + ktailSquareZero.setQueue(&auxQueues[which_queue]); + ktailSquare.setQueue(&auxQueues[which_queue]); + ktailMul.setQueue(&auxQueues[which_queue]); + ktailMulLow.setQueue(&auxQueues[which_queue]); + } + // For no particularly good reason, put kernels that operate on 32-bit value in the same queue unless there are no kernels operating on 64-bit values + if (fft.FFT_FP64 || (fft.FFT_FP32 && which_queue == -1)) { + which_queue++; + } + } + + if (fft.NTT_GF31) { + if (which_queue != -1) { + kfftWGF31.setQueue(&auxQueues[which_queue]); + kfftMidInGF31.setQueue(&auxQueues[which_queue]); + kfftMidOutGF31.setQueue(&auxQueues[which_queue]); + ktailSquareZeroGF31.setQueue(&auxQueues[which_queue]); + ktailSquareGF31.setQueue(&auxQueues[which_queue]); + ktailMulGF31.setQueue(&auxQueues[which_queue]); + ktailMulLowGF31.setQueue(&auxQueues[which_queue]); + } + //which_queue++; + } +} + +void Gpu::mergeQueue(void) { + + // If MULTI_Q -use not set, return + if (!args.value("MULTI_Q", 0)) return; + + // Queue a sync event in each auxiliary queue(s). Wait on the event(s) in the main queue. + for (size_t i = 0; i < auxQueues.size(); ++i) { + EventHolder event = auxQueues[i].createSyncEvent(); + queue.waitForSyncEvent(&event); + } + + // Return kernels to running on the main queue + // NOTE: I believe there is no need to switch queues back and forth between the main and auxiliary queues. No one currently uses the cache_group == 0 option. + if (fft.NTT_GF61) { + kfftWGF61.setQueue(&queue); + kfftMidInGF61.setQueue(&queue); + kfftMidOutGF61.setQueue(&queue); + ktailSquareZeroGF61.setQueue(&queue); + ktailSquareGF61.setQueue(&queue); + ktailMulGF61.setQueue(&queue); + ktailMulLowGF61.setQueue(&queue); + } + if (fft.FFT_FP64 || fft.FFT_FP32) { + kfftW.setQueue(&queue); + kfftMidIn.setQueue(&queue); + kfftMidOut.setQueue(&queue); + ktailSquareZero.setQueue(&queue); + ktailSquare.setQueue(&queue); + ktailMul.setQueue(&queue); + ktailMulLow.setQueue(&queue); + } + if (fft.NTT_GF31) { + kfftWGF31.setQueue(&queue); + kfftMidInGF31.setQueue(&queue); + kfftMidOutGF31.setQueue(&queue); + ktailSquareZeroGF31.setQueue(&queue); + ktailSquareGF31.setQueue(&queue); + ktailMulGF31.setQueue(&queue); + ktailMulLowGF31.setQueue(&queue); + } +} // Call the appropriate kernels to support hybrid FFTs and NTTs @@ -1162,21 +1280,25 @@ vector Gpu::readData() { return readAndCompress(bufData); } void Gpu::mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, Buffer& tmp2, bool mul3) { if (!in_place) { fftP(tmp2, ioA); + splitQueue(); for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { fftMidIn(tmp1, tmp2, cache_group); tailMul(tmp2, inB, tmp1, cache_group); fftMidOut(tmp1, tmp2, cache_group); fftW(tmp2, tmp1, cache_group); } + mergeQueue(); } else { fftP(tmp1, ioA); + splitQueue(); for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { fftMidIn(tmp1, tmp1, cache_group); tailMul(tmp1, inB, tmp1, cache_group); fftMidOut(tmp1, tmp1, cache_group); fftW(tmp2, tmp1, cache_group); } + mergeQueue(); } // Register the current ROE pos as multiplication (vs. a squaring) @@ -1371,6 +1493,7 @@ void Gpu::exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Bu while (!testBit(exp, p)) { --p; } for (--p; ; --p) { + splitQueue(); for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { if (!in_place) { if (!midInAlreadyDone) fftMidIn(buf2, buf3, cache_group); @@ -1382,10 +1505,12 @@ void Gpu::exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Bu fftMidOut(buf2, buf2, cache_group); } } + mergeQueue(); midInAlreadyDone = 0; if (testBit(exp, p)) { doCarry(buf3, buf2, bufInOut); + splitQueue(); for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { if (!in_place) { fftMidIn(buf2, buf3, cache_group); @@ -1397,6 +1522,7 @@ void Gpu::exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Bu fftMidOut(buf2, buf2, cache_group); } } + mergeQueue(); } if (!p) { break; } @@ -1447,12 +1573,14 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu // If leadOut is LEAD_WIDTH, then will buf2 contain the output of carryFused -- to be used as input to the next squaring. if (!in_place) { if (leadIn == LEAD_NONE) fftP(buf2, in); + splitQueue(); for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { if (leadIn != LEAD_MIDDLE) fftMidIn(buf1, buf2, cache_group); tailSquare(buf2, buf1, cache_group); fftMidOut(buf1, buf2, cache_group); if (leadOut == LEAD_NONE) fftW(buf2, buf1, cache_group); } + mergeQueue(); } // In place FFTs use buf1. @@ -1462,12 +1590,14 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu // If leadOut is LEAD_WIDTH, then buf1 will contain the output of carryFused -- to be used as input to the next squaring. else { if (leadIn == LEAD_NONE) fftP(buf1, in); + splitQueue(); for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { if (leadIn != LEAD_MIDDLE) fftMidIn(buf1, buf1, cache_group); tailSquare(buf1, buf1, cache_group); fftMidOut(buf1, buf1, cache_group); if (leadOut == LEAD_NONE) fftW(buf2, buf1, cache_group); } + mergeQueue(); } // If leadOut is not allowed then we cannot use the faster carryFused kernel diff --git a/src/Gpu.h b/src/Gpu.h index c3e6f766..6035d0a9 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -107,6 +107,7 @@ class Gpu { Profile profile{}; Queue queue; + vector auxQueues; KernelCompiler compiler; /* Kernels for FFT_FP64 or FFT_FP32 */ @@ -218,7 +219,9 @@ class Gpu { TimeInfo* timeBufVect; ZAvg zAvg; - int NUM_CACHE_GROUPS = 3; + const int NUM_CACHE_GROUPS = 3; + void splitQueue(void); + void mergeQueue(void); void fftP(Buffer& out, Buffer& in) { fftP(out, reinterpret_cast&>(in)); } void fftP(Buffer& out, Buffer& in); diff --git a/src/Kernel.h b/src/Kernel.h index a1505d4d..67a7bf50 100644 --- a/src/Kernel.h +++ b/src/Kernel.h @@ -42,7 +42,10 @@ class Kernel { void startLoad(KernelCompiler* compiler); void finishLoad(); - + + // Change which queue is used to run a kernel + void setQueue(Queue *q) { queue = q; } + template void setFixedArgs(int pos, const Args &...tail) { setArgs(pos, tail...); } template void operator()(const Args &...args) { diff --git a/src/Queue.cpp b/src/Queue.cpp index 27f461ad..0b2d0baf 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -17,9 +17,10 @@ void Events::synced() { assert(empty()); } -Queue::Queue(const Context& context, bool profile) : +Queue::Queue(const Context& context, bool profile, bool auxQueue) : QueueHolder{makeQueue(context.deviceId(), context.get(), profile)}, hasEvents{profile}, + isAuxQueue(auxQueue), context{&context}, markerEvent{}, markerQueued(false), @@ -55,6 +56,7 @@ void Queue::print() { void Queue::add(EventHolder&& e, TimeInfo* ti) { if (hasEvents) { events.emplace_back(std::move(e), ti); } + if (isAuxQueue) return; queueCount++; if (queueCount == MAX_QUEUE_COUNT) queueMarkerEvent(); } @@ -77,6 +79,7 @@ void Queue::run(cl_kernel kernel, size_t groupSize, size_t workSize, TimeInfo* t } void Queue::finish() { + assert (!isAuxQueue); waitForMarkerEvent(); ::finish(get()); events.synced(); @@ -84,6 +87,7 @@ void Queue::finish() { } void Queue::queueMarkerEvent() { + assert (!isAuxQueue); waitForMarkerEvent(); if (queueCount) { // AMD GPUs have no trouble waiting for a finish without a CPU busy wait. So, instead of markers and events, simply run finish every now and then. @@ -100,6 +104,7 @@ void Queue::queueMarkerEvent() { } void Queue::waitForMarkerEvent() { + assert (!isAuxQueue); if (!markerQueued) return; // By default, nVidia finish causes a CPU busy wait. Instead, sleep for a while. Since we know how many items are enqueued after the marker we can make an // educated guess of how long to sleep to keep CPU overhead low. @@ -111,6 +116,7 @@ void Queue::waitForMarkerEvent() { } void Queue::setSquareTime(int time) { + assert (!isAuxQueue); if (firstSetTime) { // Ignore first setSquareTime call. First measured times are wrong because of startup costs firstSetTime = false; return; diff --git a/src/Queue.h b/src/Queue.h index be8341d6..3e0b3ab0 100644 --- a/src/Queue.h +++ b/src/Queue.h @@ -23,6 +23,7 @@ class Events : public std::deque { class Queue : public QueueHolder { Events events; bool hasEvents; + bool isAuxQueue; void writeTE(cl_mem buf, u64 size, const void* data, TimeInfo *tInfo); void fillBufTE(cl_mem buf, u32 patSize, const void* pattern, u64 size, TimeInfo* tInfo); @@ -33,7 +34,7 @@ class Queue : public QueueHolder { public: const Context* context; - Queue(const Context& context, bool profile); + Queue(const Context& context, bool profile, bool auxQueue = false); static int registerThread(); static int tid(); @@ -50,6 +51,9 @@ class Queue : public QueueHolder { void copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo); void finish(); + EventHolder createSyncEvent(void) { return enqueueMarker(get()); } // Enqueue a synchronization event. Used to sync work among multiple queues. + void waitForSyncEvent(EventHolder* e) { enqueueMarkerWithWaits(get(), {e->get()}); } // Wait for a synchronization event to complete. + void setSquareTime(int); // Update the time to do one squaring (in microseconds) void setSquareKernels(int n) { squareKernels = n; firstSetTime = true; } diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 93385e64..602a317c 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -369,6 +369,12 @@ EventHolder enqueueMarker(cl_queue q) { return EventHolder{event}; } +EventHolder enqueueMarkerWithWaits(cl_queue q, vector&& waits) { + cl_event event{}; + CHECK1(clEnqueueMarkerWithWaitList(q, waits.size(), waits.empty() ? 0 : waits.data(), &event)); + return EventHolder{event}; +} + void waitForEvents(vector&& waits) { if (!waits.empty()) { CHECK1(clWaitForEvents(waits.size(), waits.data())); diff --git a/src/clwrap.h b/src/clwrap.h index f5cf78fd..0b005879 100644 --- a/src/clwrap.h +++ b/src/clwrap.h @@ -111,6 +111,7 @@ EventHolder copyBuf(cl_queue queue, vector&& waits, const cl_mem src, EventHolder fillBuf(cl_queue q, vector&& waits, cl_mem buf, const void *pat, size_t patSize, size_t size, bool genEvent); EventHolder enqueueMarker(cl_queue q); +EventHolder enqueueMarkerWithWaits(cl_queue q, vector&& waits); void waitForEvents(vector&& waits); diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 1e3157af..c8e20bd2 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -543,7 +543,7 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { // Also write to file since WSL2+CUDA swallows stderr if (dumpPrefix) { FILE* regLog = fopen("kernel_regs.log", "a"); - if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); fclose(regLog); } + if (regLog) { fprintf(regLog, " %-25s: %3d regs, %5d shmem, %d localmem, maxThreads=%d\n", name, numRegs, shmem, localmem, maxThreads); fclose(regLog); } } } } @@ -804,8 +804,12 @@ int clEnqueueFillBuffer(cl_command_queue q, cl_mem buf, const void* pattern, return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; } -int clEnqueueMarkerWithWaitList(cl_command_queue q, unsigned nWaits, - const cl_event* waits, cl_event* event) { +int clEnqueueMarkerWithWaitList(cl_command_queue q, unsigned nWaits, const cl_event* waits, cl_event* event) { + if (nWaits) { + for (unsigned int i = 0; i < nWaits; ++i) { + cuStreamWaitEvent(q->stream, waits[i]->end, 0); + } + } if (event) { auto* ev = new _cl_event; cuEventCreate(&ev->end, CU_EVENT_DEFAULT); From f9a2c069b6227e9f1033c27de0fac704005c7a8b Mon Sep 17 00:00:00 2001 From: george Date: Sun, 17 May 2026 20:40:29 +0000 Subject: [PATCH 076/214] Used #if to not include some code that was giving some OpenCL ccmpilers trouble (fma on floats). The code was not used anyway. --- src/cl/fftbase.cl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index daed1895..e6c3c3a3 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -1189,6 +1189,11 @@ void OVERLOAD tabMul(TrigFP32 trig, F2 *u, u32 f, u32 me) { // New fft WIDTH and HEIGHT macros to support radix-4 FFTs with more FMA instructions //************************************************************************************ +// Some OpenCL compilers are having trouble with fma on floats. Specifically, line "X2_via_FMA(u[3], preloads[7], u[7]); u[7] = mul_3t8_delayed(u[7]);". +// Since we're not enabling FP32 variant 2 by default, don't include these "more FMA" routines. + +#if ENABLE_FP32_VARIANT_2 + // Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. // We're trying to calculate u * U2(cosine,sine). // real = (u.x - u.y*sine_over_cosine) * cosine @@ -1422,6 +1427,8 @@ void finish_tabMul8_fft8(TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u3 SWAP(u[3], u[6]); } +#endif + // Variant 2 code uses more FMA instructions than the original fft version. // The tabMul after fft8 only does a partial complex multiply, saving a mul-by-cosine for the next fft8 using FMA instructions. // To maximize FMA opportunities we precompute trig values as cosine and sine/cosine rather than cosine and sine. From 6e22d6ab4e6dc26b6c06a6a8278ccb89f2dabb31 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 20 May 2026 03:33:06 +0000 Subject: [PATCH 077/214] Reworked kernel calls. The ping-pong nature of !in_place and the favorable L2 cache locality of working on NTT data one data type at a time has been moved to a central location. Code is much more readable. Paves the way for further improvements in L2 cache locality work. --- src/Gpu.cpp | 425 ++++++++++++++++++++++++++--------------------- src/Gpu.h | 30 ++-- src/cl/middle.cl | 4 +- 3 files changed, 254 insertions(+), 205 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 335b69bd..ea6be69e 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -855,7 +855,10 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo #undef BUF statsBits{u32(args.value("STATS", 0))}, - timeBufVect{profile.make("proofBufVect")} + timeBufVect{profile.make("proofBufVect")}, + + recorded_kernels{}, + recorded_kernel_args{} { float bitsPerWord = E / float(N); @@ -985,26 +988,28 @@ void Gpu::splitQueue(void) { // For no particularly good reason, put a kernel that operates on 64-bit vaules in the main queue. if (fft.NTT_GF61) { if (which_queue != -1) { - kfftWGF61.setQueue(&auxQueues[which_queue]); kfftMidInGF61.setQueue(&auxQueues[which_queue]); - kfftMidOutGF61.setQueue(&auxQueues[which_queue]); + kfftHinGF61.setQueue(&auxQueues[which_queue]); ktailSquareZeroGF61.setQueue(&auxQueues[which_queue]); ktailSquareGF61.setQueue(&auxQueues[which_queue]); ktailMulGF61.setQueue(&auxQueues[which_queue]); ktailMulLowGF61.setQueue(&auxQueues[which_queue]); + kfftMidOutGF61.setQueue(&auxQueues[which_queue]); + kfftWGF61.setQueue(&auxQueues[which_queue]); } which_queue++; } if (fft.FFT_FP64 || fft.FFT_FP32) { if (which_queue != -1) { - kfftW.setQueue(&auxQueues[which_queue]); kfftMidIn.setQueue(&auxQueues[which_queue]); - kfftMidOut.setQueue(&auxQueues[which_queue]); + kfftHin.setQueue(&auxQueues[which_queue]); ktailSquareZero.setQueue(&auxQueues[which_queue]); ktailSquare.setQueue(&auxQueues[which_queue]); ktailMul.setQueue(&auxQueues[which_queue]); ktailMulLow.setQueue(&auxQueues[which_queue]); + kfftMidOut.setQueue(&auxQueues[which_queue]); + kfftW.setQueue(&auxQueues[which_queue]); } // For no particularly good reason, put kernels that operate on 32-bit value in the same queue unless there are no kernels operating on 64-bit values if (fft.FFT_FP64 || (fft.FFT_FP32 && which_queue == -1)) { @@ -1014,13 +1019,14 @@ void Gpu::splitQueue(void) { if (fft.NTT_GF31) { if (which_queue != -1) { - kfftWGF31.setQueue(&auxQueues[which_queue]); kfftMidInGF31.setQueue(&auxQueues[which_queue]); - kfftMidOutGF31.setQueue(&auxQueues[which_queue]); + kfftHinGF31.setQueue(&auxQueues[which_queue]); ktailSquareZeroGF31.setQueue(&auxQueues[which_queue]); ktailSquareGF31.setQueue(&auxQueues[which_queue]); ktailMulGF31.setQueue(&auxQueues[which_queue]); ktailMulLowGF31.setQueue(&auxQueues[which_queue]); + kfftMidOutGF31.setQueue(&auxQueues[which_queue]); + kfftWGF31.setQueue(&auxQueues[which_queue]); } //which_queue++; } @@ -1040,85 +1046,194 @@ void Gpu::mergeQueue(void) { // Return kernels to running on the main queue // NOTE: I believe there is no need to switch queues back and forth between the main and auxiliary queues. No one currently uses the cache_group == 0 option. if (fft.NTT_GF61) { - kfftWGF61.setQueue(&queue); kfftMidInGF61.setQueue(&queue); - kfftMidOutGF61.setQueue(&queue); + kfftHinGF61.setQueue(&queue); ktailSquareZeroGF61.setQueue(&queue); ktailSquareGF61.setQueue(&queue); ktailMulGF61.setQueue(&queue); ktailMulLowGF61.setQueue(&queue); + kfftMidOutGF61.setQueue(&queue); + kfftWGF61.setQueue(&queue); } if (fft.FFT_FP64 || fft.FFT_FP32) { - kfftW.setQueue(&queue); kfftMidIn.setQueue(&queue); - kfftMidOut.setQueue(&queue); + kfftHin.setQueue(&queue); ktailSquareZero.setQueue(&queue); ktailSquare.setQueue(&queue); ktailMul.setQueue(&queue); ktailMulLow.setQueue(&queue); + kfftMidOut.setQueue(&queue); + kfftW.setQueue(&queue); } if (fft.NTT_GF31) { - kfftWGF31.setQueue(&queue); kfftMidInGF31.setQueue(&queue); - kfftMidOutGF31.setQueue(&queue); + kfftHinGF31.setQueue(&queue); ktailSquareZeroGF31.setQueue(&queue); ktailSquareGF31.setQueue(&queue); ktailMulGF31.setQueue(&queue); ktailMulLowGF31.setQueue(&queue); + kfftMidOutGF31.setQueue(&queue); + kfftWGF31.setQueue(&queue); + } +} + +// Replay the recorded bottom half kernels in a cache friendly order. We support several +// options here using multiple openCl command queues. +void Gpu::replay(void) { + + // If using multiple command queues, handle that now. + splitQueue(); + + // For better L2 cache locality, operate on all the FP data, then operate on all the GF31 data, then GF61. + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + + // Check for irrelevant cache gouup + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Iterate over the recorded kernels + int arg = 0; + for (auto kern : recorded_kernels) { + + // Call the appropriate kernel + if (kern == KMIDIN) { + Buffer *buf = recorded_kernel_args[arg++]; + // If not in place, the input is from the scratch buffer + Buffer *in = in_place ? buf : &buf3; + Buffer *out = buf; + if (cache_group == 1) kfftMidIn(*out, *in); + if (cache_group == 2) kfftMidInGF31(*out, *in); + if (cache_group == 3) kfftMidInGF61(*out, *in); + } + + if (kern == KFFTHIN) { + Buffer *out = recorded_kernel_args[arg++]; + Buffer *in = recorded_kernel_args[arg++]; + if (cache_group == 1) kfftHin(*out, *in); + if (cache_group == 2) kfftHinGF31(*out, *in); + if (cache_group == 3) kfftHinGF61(*out, *in); + } + + if (kern == KTAILSQUARE) { + Buffer *buf = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer *in = buf; + Buffer *out = in_place ? buf : &buf3; + if (!tail_single_kernel) { + if (cache_group == 1) ktailSquareZero(*out, *in); + if (cache_group == 2) ktailSquareZeroGF31(*out, *in); + if (cache_group == 3) ktailSquareZeroGF61(*out, *in); + } + if (cache_group == 1) ktailSquare(*out, *in); + if (cache_group == 2) ktailSquareGF31(*out, *in); + if (cache_group == 3) ktailSquareGF61(*out, *in); + } + + if (kern == KTAILMUL) { + Buffer *buf = recorded_kernel_args[arg++]; + Buffer *in2 = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer *in1 = buf; + Buffer *out = in_place ? buf : &buf3; + if (cache_group == 1) ktailMul(*out, *in1, *in2); + if (cache_group == 2) ktailMulGF31(*out, *in1, *in2); + if (cache_group == 3) ktailMulGF61(*out, *in1, *in2); + } + + if (kern == KTAILMULLOW) { + Buffer *buf = recorded_kernel_args[arg++]; + Buffer *in2 = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer *in1 = buf; + Buffer *out = in_place ? buf : &buf3; + if (cache_group == 1) ktailMulLow(*out, *in1, *in2); + if (cache_group == 2) ktailMulLowGF31(*out, *in1, *in2); + if (cache_group == 3) ktailMulLowGF61(*out, *in1, *in2); + } + + if (kern == KMIDOUT) { + Buffer *buf = recorded_kernel_args[arg++]; + // If not in place, the input is from the scratch buffer + Buffer *in = in_place ? buf : &buf3; + Buffer *out = buf; + if (cache_group == 1) kfftMidOut(*out, *in); + if (cache_group == 2) kfftMidOutGF31(*out, *in); + if (cache_group == 3) kfftMidOutGF61(*out, *in); + } + + if (kern == KFFTW) { + Buffer *out = recorded_kernel_args[arg++]; + Buffer *in = recorded_kernel_args[arg++]; + if (cache_group == 1) kfftW(*out, *in); + if (cache_group == 2) kfftWGF31(*out, *in); + if (cache_group == 3) kfftWGF61(*out, *in); + } + } } + + // Empty the recorded kernels queue + recorded_kernels.clear(); + recorded_kernel_args.clear(); + + // If using multiple command queues, go back to a single command queue + mergeQueue(); } // Call the appropriate kernels to support hybrid FFTs and NTTs -void Gpu::fftP(Buffer& out, Buffer& in) { - kfftP(out, in); +void Gpu::fftP(Buffer& buf, Buffer& in) { + // If not in place, instead write the output to the scratch buffer + Buffer *out = in_place ? &buf : &buf3; + kfftP(*out, in); } -void Gpu::fftW(Buffer& out, Buffer& in, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) kfftW(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) kfftWGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) kfftWGF61(out, in); +void Gpu::fftMidIn(Buffer& buf) { + // Record this call for later playback + recorded_kernels.push_back(KMIDIN); + recorded_kernel_args.push_back(&buf); } -void Gpu::fftMidIn(Buffer& out, Buffer& in, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) kfftMidIn(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) kfftMidInGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) kfftMidInGF61(out, in); +void Gpu::fftHin(Buffer& out, Buffer& in) { + // Record this call for later playback + recorded_kernels.push_back(KFFTHIN); + recorded_kernel_args.push_back(&out); + recorded_kernel_args.push_back(&in); } -void Gpu::fftMidOut(Buffer& out, Buffer& in, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) kfftMidOut(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) kfftMidOutGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) kfftMidOutGF61(out, in); +void Gpu::tailSquare(Buffer& buf) { + // Record this call for later playback + recorded_kernels.push_back(KTAILSQUARE); + recorded_kernel_args.push_back(&buf); } -void Gpu::fftHin(Buffer& out, Buffer& in) { - if (fft.FFT_FP64 || fft.FFT_FP32) kfftHin(out, in); - if (fft.NTT_GF31) kfftHinGF31(out, in); - if (fft.NTT_GF61) kfftHinGF61(out, in); +void Gpu::tailMul(Buffer& buf, Buffer& in2) { + // Record this call for later playback + recorded_kernels.push_back(KTAILMUL); + recorded_kernel_args.push_back(&buf); + recorded_kernel_args.push_back(&in2); } -void Gpu::tailSquare(Buffer& out, Buffer& in, int cache_group) { - if (!tail_single_kernel) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailSquareZero(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailSquareZeroGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailSquareZeroGF61(out, in); - } - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailSquare(out, in); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailSquareGF31(out, in); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailSquareGF61(out, in); +void Gpu::tailMulLow(Buffer& buf, Buffer& in2) { + // Record this call for later playback + recorded_kernels.push_back(KTAILMULLOW); + recorded_kernel_args.push_back(&buf); + recorded_kernel_args.push_back(&in2); } -void Gpu::tailMul(Buffer& out, Buffer& in1, Buffer& in2, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailMul(out, in1, in2); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailMulGF31(out, in1, in2); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailMulGF61(out, in1, in2); +void Gpu::fftMidOut(Buffer& buf) { + // Record this call for later playback + recorded_kernels.push_back(KMIDOUT); + recorded_kernel_args.push_back(&buf); } -void Gpu::tailMulLow(Buffer& out, Buffer& in1, Buffer& in2, int cache_group) { - if ((cache_group == 0 || cache_group == 1) && (fft.FFT_FP64 || fft.FFT_FP32)) ktailMulLow(out, in1, in2); - if ((cache_group == 0 || cache_group == 2) && fft.NTT_GF31) ktailMulLowGF31(out, in1, in2); - if ((cache_group == 0 || cache_group == 3) && fft.NTT_GF61) ktailMulLowGF61(out, in1, in2); +void Gpu::fftW(Buffer& out, Buffer& in) { + // Record this call for later playback + recorded_kernels.push_back(KFFTW); + recorded_kernel_args.push_back(&out); + recorded_kernel_args.push_back(&in); + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + replay(); } void Gpu::carryA(Buffer& out, Buffer& in) { @@ -1137,20 +1252,35 @@ void Gpu::carryLL(Buffer& out, Buffer& in) { kCarryLL(out, in, updateCarryPos(1 << 2)); } -void Gpu::carryFused(Buffer& out, Buffer& in) { +void Gpu::carryFused(Buffer& buf) { + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + replay(); assert(roePos <= ROE_SIZE); - roePos < wantROE ? kCarryFusedROE(out, in, roePos++) - : kCarryFused(out, in, updateCarryPos(1 << 0)); + // Like fftP, if not in place write the output to the scratch buffer + Buffer *in = &buf; + Buffer *out = in_place ? &buf : &buf3; + roePos < wantROE ? kCarryFusedROE(*out, *in, roePos++) + : kCarryFused(*out, *in, updateCarryPos(1 << 0)); } -void Gpu::carryFusedMul(Buffer& out, Buffer& in) { +void Gpu::carryFusedMul(Buffer& buf) { + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + replay(); assert(roePos <= ROE_SIZE); - roePos < wantROE ? kCarryFusedMulROE(out, in, roePos++) - : kCarryFusedMul(out, in, updateCarryPos(1 << 1)); + // Like fftP, if not in place write the output to the scratch buffer + Buffer *in = &buf; + Buffer *out = in_place ? &buf : &buf3; + roePos < wantROE ? kCarryFusedMulROE(*out, *in, roePos++) + : kCarryFusedMul(*out, *in, updateCarryPos(1 << 1)); } -void Gpu::carryFusedLL(Buffer& out, Buffer& in) { - kCarryFusedLL(out, in, updateCarryPos(1 << 0)); +void Gpu::carryFusedLL(Buffer& buf) { + // This kernel always ends the "bottom half". Replay the recorded kernel calls. + replay(); + // Like fftP, if not in place write the output to the scratch buffer + Buffer *in = &buf; + Buffer *out = in_place ? &buf : &buf3; + kCarryFusedLL(*out, *in, updateCarryPos(1 << 0)); } @@ -1276,58 +1406,32 @@ Words Gpu::readAndCompress(Buffer& buf) { return compactBits(readChecked( vector Gpu::readCheck() { return readAndCompress(bufCheck); } vector Gpu::readData() { return readAndCompress(bufData); } -// out := inA * inB; inB is preserved -void Gpu::mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, Buffer& tmp2, bool mul3) { - if (!in_place) { - fftP(tmp2, ioA); - splitQueue(); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - fftMidIn(tmp1, tmp2, cache_group); - tailMul(tmp2, inB, tmp1, cache_group); - fftMidOut(tmp1, tmp2, cache_group); - fftW(tmp2, tmp1, cache_group); - } - mergeQueue(); - } - else { - fftP(tmp1, ioA); - splitQueue(); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - fftMidIn(tmp1, tmp1, cache_group); - tailMul(tmp1, inB, tmp1, cache_group); - fftMidOut(tmp1, tmp1, cache_group); - fftW(tmp2, tmp1, cache_group); - } - mergeQueue(); - } +// ioA := ioA * inB; inB must be the output of fftMidIn; inB is preserved +void Gpu::mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, bool mul3) { + fftP(tmp1, ioA); + fftMidIn(tmp1); + tailMul(tmp1, inB); + fftMidOut(tmp1); + fftW(buf3, tmp1); // Register the current ROE pos as multiplication (vs. a squaring) if (mulRoePos.empty() || mulRoePos.back() < roePos) { mulRoePos.push_back(roePos); } - if (mul3) { carryM(ioA, tmp2); } else { carryA(ioA, tmp2); } + if (mul3) { carryM(ioA, buf3); } else { carryA(ioA, buf3); } carryB(ioA); } -void Gpu::mul(Buffer& io, Buffer& buf1) { - // We know that mul() stores double output in buf1; so we're going to use buf2 & buf3 for temps. - mul(io, buf1, buf2, buf3, false); -} - -// out := inA * inB; +// ioA := ioA * inB; inB will end up in buf1 in the LEAD_MIDDLE state void Gpu::modMul(Buffer& ioA, Buffer& inB, bool mul3) { modMul(ioA, inB, LEAD_NONE, mul3); }; -// out := inA * inB; inB will end up in buf1 in the LEAD_MIDDLE state +// ioA := ioA * inB; inB will end up in buf1 in the LEAD_MIDDLE state void Gpu::modMul(Buffer& ioA, Buffer& inB, enum LEAD_TYPE leadInB, bool mul3) { - if (!in_place) { - if (leadInB == LEAD_NONE) fftP(buf2, inB); - if (leadInB != LEAD_MIDDLE) fftMidIn(buf1, buf2); - } else { - if (leadInB == LEAD_NONE) fftP(buf1, inB); - if (leadInB != LEAD_MIDDLE) fftMidIn(buf1, buf1); - } - mul(ioA, buf1, buf2, buf3, mul3); + if (leadInB == LEAD_NONE) fftP(buf1, inB); + if (leadInB != LEAD_MIDDLE) fftMidIn(buf1); + replay(); // Work around an odd bug. The above executed fftP writing to buf3 if !in_place and queued fftMidIn. If we don't replay now, mul will call fftP again overwriting buf3. + mul(ioA, buf1, buf2, mul3); }; void Gpu::writeState(u64 k, const vector& check, u32 blockSize) { @@ -1458,7 +1562,7 @@ Words Gpu::expExp2(const Words& A, u32 n) { // A:= A^h * B void Gpu::expMul(Buffer& A, u64 h, Buffer& B) { - exponentiate(A, h, buf1, buf2, buf3); + exponentiate(A, h); modMul(A, B); } @@ -1475,139 +1579,80 @@ Words Gpu::expMul(const Words& A, u64 h, const Words& B, bool doSquareB) { static bool testBit(u64 x, int bit) { return x & (u64(1) << bit); } // See "left-to-right binary exponentiation" on wikipedia -void Gpu::exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Buffer& buf2, Buffer& buf3) { +void Gpu::exponentiate(Buffer& bufInOut, u64 exp) { if (exp == 0) { bufInOut.set(1); } else if (exp > 1) { - if (!in_place) { - fftP(buf3, bufInOut); - fftMidIn(buf2, buf3); - } else { - fftP(buf2, bufInOut); - fftMidIn(buf2, buf2); - } - fftHin(buf1, buf2); // save "base" to buf1 + fftP(buf1, bufInOut); + fftMidIn(buf1); + fftHin(buf2, buf1); // save fully FFTed "base" to buf2 bool midInAlreadyDone = 1; int p = 63; while (!testBit(exp, p)) { --p; } for (--p; ; --p) { - splitQueue(); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (!in_place) { - if (!midInAlreadyDone) fftMidIn(buf2, buf3, cache_group); - tailSquare(buf3, buf2, cache_group); - fftMidOut(buf2, buf3, cache_group); - } else { - if (!midInAlreadyDone) fftMidIn(buf2, buf2, cache_group); - tailSquare(buf2, buf2, cache_group); - fftMidOut(buf2, buf2, cache_group); - } - } - mergeQueue(); + if (!midInAlreadyDone) fftMidIn(buf1); + tailSquare(buf1); + fftMidOut(buf1); midInAlreadyDone = 0; if (testBit(exp, p)) { - doCarry(buf3, buf2, bufInOut); - splitQueue(); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (!in_place) { - fftMidIn(buf2, buf3, cache_group); - tailMulLow(buf3, buf2, buf1, cache_group); - fftMidOut(buf2, buf3, cache_group); - } else { - fftMidIn(buf2, buf2, cache_group); - tailMulLow(buf2, buf2, buf1, cache_group); - fftMidOut(buf2, buf2, cache_group); - } - } - mergeQueue(); + doCarry(buf1, bufInOut); + fftMidIn(buf1); + tailMulLow(buf1, buf2); + fftMidOut(buf1); } if (!p) { break; } - doCarry(buf3, buf2, bufInOut); + doCarry(buf1, bufInOut); } - fftW(buf3, buf2); + fftW(buf3, buf1); carryA(bufInOut, buf3); carryB(bufInOut); } } // does either carryFused() or the expanded version depending on useLongCarry -void Gpu::doCarry(Buffer& out, Buffer& in, Buffer& tmp) { - if (!in_place) { - if (useLongCarry) { - fftW(out, in); - carryA(tmp, out); - carryB(tmp); - fftP(out, tmp); - } else { - carryFused(out, in); - } +void Gpu::doCarry(Buffer& in, Buffer& wordBuf) { + if (useLongCarry) { + fftW(buf3, in); + carryA(wordBuf, buf3); + carryB(wordBuf); + fftP(in, wordBuf); } else { - if (useLongCarry) { - fftW(out, in); - carryA(tmp, out); - carryB(tmp); - fftP(in, tmp); - } else { - carryFused(in, in); - } + carryFused(in); } } -// Use buf1 and buf2 to do a single squaring. +// Use buf1 (and buf23 if not in place) to do a single squaring. void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut, bool doMul3, bool doLL) { // leadOut = LEAD_MIDDLE is not supported (slower than LEAD_WIDTH) assert(leadOut != LEAD_MIDDLE); // LL does not do Mul3 assert(!(doMul3 && doLL)); - // Not in place FFTs use buf1 and buf2 in a "ping pong" fashion. - // If leadIn is LEAD_NONE, in contains the input data, squaring starts at fftP - // If leadIn is LEAD_WIDTH, buf2 contains the input data, squaring starts at fftMidIn - // If leadIn is LEAD_MIDDLE, buf1 contains the input data, squaring starts at tailSquare - // If leadOut is LEAD_WIDTH, then will buf2 contain the output of carryFused -- to be used as input to the next squaring. - if (!in_place) { - if (leadIn == LEAD_NONE) fftP(buf2, in); - splitQueue(); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (leadIn != LEAD_MIDDLE) fftMidIn(buf1, buf2, cache_group); - tailSquare(buf2, buf1, cache_group); - fftMidOut(buf1, buf2, cache_group); - if (leadOut == LEAD_NONE) fftW(buf2, buf1, cache_group); - } - mergeQueue(); - } - - // In place FFTs use buf1. + // In place FFTs use buf1. Not in place FFTs also use buf3. // If leadIn is LEAD_NONE, in contains the input data, squaring starts at fftP - // If leadIn is LEAD_WIDTH, buf1 contains the input data, squaring starts at fftMidIn + // If leadIn is LEAD_WIDTH, buf1 (or buf3 if not in place) contains the input data, squaring starts at fftMidIn // If leadIn is LEAD_MIDDLE, buf1 contains the input data, squaring starts at tailSquare - // If leadOut is LEAD_WIDTH, then buf1 will contain the output of carryFused -- to be used as input to the next squaring. - else { - if (leadIn == LEAD_NONE) fftP(buf1, in); - splitQueue(); - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - if (leadIn != LEAD_MIDDLE) fftMidIn(buf1, buf1, cache_group); - tailSquare(buf1, buf1, cache_group); - fftMidOut(buf1, buf1, cache_group); - if (leadOut == LEAD_NONE) fftW(buf2, buf1, cache_group); - } - mergeQueue(); - } + // If leadOut is LEAD_WIDTH, then buf1 (or buf3 if not in place) will contain the output of carryFused -- to be used as input to the next squaring. + if (leadIn == LEAD_NONE) fftP(buf1, in); + if (leadIn != LEAD_MIDDLE) fftMidIn(buf1); + tailSquare(buf1); + fftMidOut(buf1); // If leadOut is not allowed then we cannot use the faster carryFused kernel if (leadOut == LEAD_NONE) { + fftW(buf3, buf1); if (!doLL && !doMul3) { - carryA(out, buf2); + carryA(out, buf3); } else if (doLL) { - carryLL(out, buf2); + carryLL(out, buf3); } else { - carryM(out, buf2); + carryM(out, buf3); } carryB(out); } @@ -1617,9 +1662,9 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu assert(!useLongCarry); assert(!doMul3); if (doLL) { - carryFusedLL(in_place ? buf1 : buf2, buf1); + carryFusedLL(buf1); } else { - carryFused(in_place ? buf1 : buf2, buf1); + carryFused(buf1); } } } diff --git a/src/Gpu.h b/src/Gpu.h index 6035d0a9..e7a2b810 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -219,26 +219,31 @@ class Gpu { TimeInfo* timeBufVect; ZAvg zAvg; + enum BOTTOM_HALF_KERNELS {KMIDIN, KFFTHIN, KTAILSQUARE, KTAILMUL, KTAILMULLOW, KMIDOUT, KFFTW}; + vector recorded_kernels; + vector *> recorded_kernel_args; + const int NUM_CACHE_GROUPS = 3; void splitQueue(void); void mergeQueue(void); + void replay(void); void fftP(Buffer& out, Buffer& in) { fftP(out, reinterpret_cast&>(in)); } void fftP(Buffer& out, Buffer& in); - void fftMidIn(Buffer& out, Buffer& in, int cache_group = 0); - void fftMidOut(Buffer& out, Buffer& in, int cache_group = 0); + void fftMidIn(Buffer& buf); + void fftMidOut(Buffer& buf); void fftHin(Buffer& out, Buffer& in); - void tailSquare(Buffer& out, Buffer& in, int cache_group = 0); - void tailMul(Buffer& out, Buffer& in1, Buffer& in2, int cache_group = 0); - void tailMulLow(Buffer& out, Buffer& in1, Buffer& in2, int cache_group = 0); - void fftW(Buffer& out, Buffer& in, int cache_group = 0); + void tailSquare(Buffer& buf); + void tailMul(Buffer& buf, Buffer& in2); + void tailMulLow(Buffer& buf, Buffer& in2); + void fftW(Buffer& out, Buffer& in); void carryA(Buffer& out, Buffer& in) { carryA(reinterpret_cast&>(out), in); } void carryA(Buffer& out, Buffer& in); void carryM(Buffer& out, Buffer& in); void carryLL(Buffer& out, Buffer& in); - void carryFused(Buffer& out, Buffer& in); - void carryFusedMul(Buffer& out, Buffer& in); - void carryFusedLL(Buffer& out, Buffer& in); + void carryFused(Buffer& buf); + void carryFusedMul(Buffer& buf); + void carryFusedLL(Buffer& buf); vector readWords(Buffer &buf); void writeWords(Buffer& buf, vector &words); @@ -261,15 +266,14 @@ class Gpu { vector writeBase(const vector &v); - void exponentiate(Buffer& bufInOut, u64 exp, Buffer& buf1, Buffer& buf2, Buffer& buf3); + void exponentiate(Buffer& bufInOut, u64 exp); void writeState(u64 k, const vector& check, u32 blockSize); // does either carrryFused() or the expanded version depending on useLongCarry - void doCarry(Buffer& out, Buffer& in, Buffer& tmp); + void doCarry(Buffer& in, Buffer& wordBuf); - void mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, Buffer& tmp2, bool mul3 = false); - void mul(Buffer& io, Buffer& inB); + void mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, bool mul3 = false); void modMul(Buffer& ioA, Buffer& inB, bool mul3 = false); void modMul(Buffer& ioA, Buffer& inB, enum LEAD_TYPE leadInB, bool mul3 = false); diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 0236141f..35a251cf 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -680,11 +680,11 @@ void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { // Goals: // 1) In-place transpose. Rather than "ping-pong"ing buffers, an in-place transpose uses half as much memory. This may allow // the entire FFT/NTT data set to reside in the L2 cache on upper end consumer GPUs (circa 2025) which can have 64MB or larger L2 caches. -// 2) We want to have distribute the carryFused and/or tailSquare memory in the L2 cache with minimal cache line collisions. The hope is to (one day) do +// 2) We want to distribute the carryFused and/or tailSquare memory in the L2 cache with minimal cache line collisions. The hope is to (one day) do // fftMiddleOut/carryFused/fftMiddleIn or fftMiddleIn/tailSquare/fftMiddleOut in L2 cache-sized chunks to minimize the slowest memory accesses. // The cost of extra kernel launches may negate any L2 cache benefits. // 3) We use swizzling and/or modest padding to reduce carryFused L2 cache line collisions. Several different memory layouts and padding were tried -// on nVidia Titan V and AMD Radeon VII to find the fastest in-place layout and padding scheme. Hopefully, these will schemes will work well +// on nVidia Titan V and AMD Radeon VII to find the fastest in-place layout and padding scheme. Hopefully, these schemes will work well // on later generation GPUs with different L2 cache dimensions (size and "number-of-ways"). // 4) Apparently cache line collisions in the L1 cache also adversely affect timings. The L1 cache may have a different cache line size and number-of-ways // which makes padding tuning a bit difficult. This is especially true on AMD which has a very strange channel & banks partitioning of memory accesses. From 0bd593e73fed317c7c27b72afe177b94b2f7bcaf Mon Sep 17 00:00:00 2001 From: george Date: Sun, 24 May 2026 00:44:17 +0000 Subject: [PATCH 078/214] Fixed yet another Windws treats long as 32-bits bug. --- src/cl/carryutil.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 4ae4b576..57bf5adb 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -94,7 +94,7 @@ bool test(u32 bits, u32 pos) { return (bits >> pos) & 1; } #if FFT_FP64 // Rounding constant: 3 * 2^51, See https://stackoverflow.com/questions/17035464 -#define RNDVAL (3.0 * (1l << 51)) +#define RNDVAL (3.0 * (1ull << 51)) // Convert a double to long efficiently. Double must be in RNDVAL+integer format. i64 RNDVALdoubleToLong(double d) { From 0b54a8ceb18784881e1ebccb7da8fc33074f30db Mon Sep 17 00:00:00 2001 From: george Date: Mon, 25 May 2026 06:00:29 +0000 Subject: [PATCH 079/214] Detect nVidia compute capability to set HAS_PTX properly. Lets nVidia 1080 Gpus use asm by default. --- src/Gpu.cpp | 1 + src/cl/base.cl | 2 +- src/clwrap.cpp | 8 ++++++++ src/clwrap.h | 1 + src/cuda/clwrap_cuda.cpp | 14 ++++++++++++++ src/cuda/tinycuda.h | 4 ++++ src/tinycl.h | 4 ++++ 7 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ea6be69e..5a2bb181 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -335,6 +335,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< if (isAmdGpu(id)) { defines += toDefine("AMDGPU", 1); } if (isNvidiaGpu(id)) { defines += toDefine("NVIDIAGPU", 1); } + if (isNvidiaGpu(id)) { defines += toDefine("CC", getNvidiaComputeCapability(id)); } if ((fft.carry == CARRY_AUTO && fft.shape.needsLargeCarry(E)) || (fft.carry == CARRY_64)) { if (doLog) { log("Using CARRY64\n"); } diff --git a/src/cl/base.cl b/src/cl/base.cl index 7654be7a..ae20149a 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -75,7 +75,7 @@ G_H "group height" == SMALL_HEIGHT / NH #define HAS_PTX 0 #elif NVIDIAGPU #define HAS_ASM 0 -#define HAS_PTX 1200 // Assume CUDA 12.00 support until we can figure out how to automatically determine this at runtime +#define HAS_PTX CC // C code computed the nVidia GPU's compute capability #else #define HAS_ASM 0 #define HAS_PTX 0 diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 602a317c..1152a7bb 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -178,6 +178,14 @@ bool isNvidiaGpu(cl_device_id id) { return pcieId == 0x10DE; } +u32 getNvidiaComputeCapability(cl_device_id id) { + u32 major = 0; + u32 minor = 0; + GET_INFO(id, CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV, major); + GET_INFO(id, CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV, minor); + return major * 100 + minor; +} + /* static string getFreq(cl_device_id device) { unsigned computeUnits, frequency; diff --git a/src/clwrap.h b/src/clwrap.h index 0b005879..41b2bb93 100644 --- a/src/clwrap.h +++ b/src/clwrap.h @@ -67,6 +67,7 @@ u64 getFreeMem(cl_device_id id); bool hasFreeMemInfo(cl_device_id id); bool isAmdGpu(cl_device_id id); bool isNvidiaGpu(cl_device_id id); +u32 getNvidiaComputeCapability(cl_device_id id); string getDriverVersion(cl_device_id id); string getDriverVersionByPos(int pos); diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index c8e20bd2..5df25791 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -969,6 +969,20 @@ int clGetDeviceInfo(cl_device_id dev, cl_device_info info, size_t size, void* va if (value && size >= sizeof(freeKB)) memcpy(value, &freeKB, sizeof(freeKB)); break; } + case CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV: { + int major = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev->dev); + if (sizeRet) *sizeRet = sizeof(major); + if (value && size >= sizeof(major)) memcpy(value, &major, sizeof(major)); + break; + } + case CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV: { + int minor = 0; + cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev->dev); + if (sizeRet) *sizeRet = sizeof(minor); + if (value && size >= sizeof(minor)) memcpy(value, &minor, sizeof(minor)); + break; + } default: return CL_INVALID_VALUE; } diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index 5a34c15e..d01946df 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -207,6 +207,10 @@ using cl_queue = cl_command_queue; #define CL_KERNEL_ATTRIBUTES 0x1195 #define CL_KERNEL_COMPILE_WORK_GROUP_SIZE 0x11B1 +// nVidia +#define CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV 0x4000 +#define CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV 0x4001 + // AMD-specific (unused but must exist for compilation) #define CL_DEVICE_PCIE_ID_AMD 0x4034 #define CL_DEVICE_TOPOLOGY_AMD 0x4037 diff --git a/src/tinycl.h b/src/tinycl.h index e3317657..90fa8404 100644 --- a/src/tinycl.h +++ b/src/tinycl.h @@ -207,6 +207,10 @@ int clSetKernelArgSVMPointer(cl_kernel, unsigned, const void *); #define CL_KERNEL_COMPILE_WORK_GROUP_SIZE 0x11B1 +// nVidia +#define CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV 0x4000 +#define CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV 0x4001 + // AMD #define CL_DEVICE_PCIE_ID_AMD 0x4034 #define CL_DEVICE_TOPOLOGY_AMD 0x4037 From a3e4c7eecf0c1b446b56da98e502aa6b432e788c Mon Sep 17 00:00:00 2001 From: george Date: Mon, 25 May 2026 19:32:41 +0000 Subject: [PATCH 080/214] Another attempt at fixing the GPU read failed error. --- src/Gpu.cpp | 15 ++++----------- src/cl/etc.cl | 4 ++-- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 5a2bb181..4a491748 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -1373,22 +1373,15 @@ static bool isAllZero(vector v) { return std::all_of(v.begin(), v.end(), [](T vector Gpu::readChecked(Buffer& buf) { for (int nRetry = 0; nRetry < 3; ++nRetry) { bufSumOut.zero(); - sum64(bufSumOut, u32(buf.size * sizeof(Word)), buf); - + sum64(bufSumOut, N, buf); vector expectedVect(1); - bufSumOut.readAsync(expectedVect); - vector data = readOut(buf); - u64 gpuSum = expectedVect[0]; + vector data = readOut(buf); u64 hostSum = 0; + for (auto it = data.begin(), end = data.end(); it < end; ++it) hostSum += u64(*it); - int even = 1; - for (auto it = data.begin(), end = data.end(); it < end; ++it, even = !even) { - if (fft.WordSize == 4) hostSum += even ? u64(u32(*it)) : (u64(*it) << 32); - if (fft.WordSize == 8) hostSum += u64(*it); - } - + u64 gpuSum = expectedVect[0]; if (hostSum == gpuSum) { // A buffer containing all-zero is exceptional, so mark that through the empty vector. if (gpuSum == 0 && isAllZero(data)) { diff --git a/src/cl/etc.cl b/src/cl/etc.cl index 2f5fb43a..e3e0f7c9 100644 --- a/src/cl/etc.cl +++ b/src/cl/etc.cl @@ -16,9 +16,9 @@ KERNEL(32) readResidue(P(Word2) out, CP(Word2) in) { #endif #if SUM64 -KERNEL(64) sum64(global ulong* out, u32 sizeBytes, global ulong* in) { +KERNEL(64) sum64(global ulong* out, u32 count, CP(Word) in) { ulong sum = 0; - for (i32 p = get_global_id(0); p < sizeBytes / sizeof(u64); p += get_global_size(0)) { + for (i32 p = get_global_id(0); p < count; p += get_global_size(0)) { sum += in[p]; } u32 prev = atomic_add((global u32*)out, (u32) sum); From 7b6b2c1c93bed2f228ba70896640fdc9fc4e1fef Mon Sep 17 00:00:00 2001 From: george Date: Tue, 26 May 2026 19:48:24 +0000 Subject: [PATCH 081/214] Restrict usage of WMUL=4 for WIDTH=1K and WMUL=2 for WIDTH=4K --- src/Gpu.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 4a491748..f16fe0e9 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -306,13 +306,16 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< } // Maximum WMUL is 32KB / (WIDTH * SHUFL_BYTES_W). If using the 32KB maximum, LDS padding must be disabled. + // Furthermore, I've seen the CUDA compiler refuse to create a kernel with 1024 threads. Thus, we limit WMUL to 2 for a 1K width and to 1 for a 4K width. { u32 shufl_bytes_w = args.value("SHUFL_BYTES_W", 8); u32 max_wmul = 32768 / (fft.shape.width * shufl_bytes_w); + if (max_wmul > 2 && fft.shape.width >= 1024) max_wmul = 2; + if (max_wmul > 1 && fft.shape.width >= 4096) max_wmul = 1; if (wmul > max_wmul) { wmul = max_wmul; config["WMUL"] = to_string(wmul); - log("Local shared memory limit of 32KB exceeded. Changing to WMUL=%d\n", wmul); + log("WMUL setting too large for this FFT width. Changing to WMUL=%d\n", wmul); } if (fft.shape.width * shufl_bytes_w * wmul >= 32768) { log("Local shared memory limit of 32KB exceeded. Changing to LDSPAD_W=0\n"); From b2d530f99be9e0cdedd2f54f2c0cae815a50e073 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Mon, 30 Mar 2026 10:45:47 -0700 Subject: [PATCH 082/214] Updated CI to support building with CUDA. --- .github/workflows/ci.yml | 73 +++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d43b725..aa7b5f77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - cron: '0 0 1 * *' jobs: - Linux: - name: Linux + Linux-OpenCL: + name: Linux OpenCL runs-on: ${{ matrix.os }} strategy: @@ -26,7 +26,7 @@ jobs: - name: Install run: | sudo apt-get update -y - sudo apt-get install -y cppcheck ocl-icd-opencl-dev pocl-opencl-icd + sudo apt-get install -y ocl-icd-opencl-dev pocl-opencl-icd $CXX --version - name: Script run: | @@ -39,14 +39,71 @@ jobs: with: name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_prpll path: ${{ github.workspace }} - - name: Cppcheck + + Linux-CUDA: + name: Linux CUDA + + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-22.04, ubuntu-24.04] + cxx: [g++, clang++] + exclude: + - os: ubuntu-22.04-arm + cxx: clang++ + fail-fast: false + env: + CXX: ${{ matrix.cxx }} + steps: + - uses: actions/checkout@v6 + - name: Install + run: | + sudo apt-get update -y + sudo apt-get install -y nvidia-cuda-toolkit + $CXX --version + - name: Script + run: | + make CUDA=1 -O -j "$(nproc)" + cd build-cuda + rm -f -- *.o + ./prpll -h + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_cuda_prpll + path: ${{ github.workspace }} + + Cppcheck: + name: Cppcheck + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Install + run: | + sudo apt-get update -y + sudo apt-get install -y cppcheck + - name: Script run: cppcheck --enable=all --force . - - name: Clang-Tidy - if: ${{ matrix.cxx == 'clang++' }} + + Clang-Tidy: + name: Clang-Tidy + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Script run: clang-tidy -checks='bugprone-*,-bugprone-reserved-identifier,cert-*,-cert-dcl37-c,-cert-dcl51-cpp,clang-analyzer-*,concurrency-*,misc-*,-misc-no-recursion,modernize-*,-modernize-use-trailing-return-type,performance-*,portability-*,readability-const-return-type,readability-container-*,readability-duplicate-include,readability-else-after-return,readability-make-member-function-cons,readability-non-const-parameter,readability-redundant-*,readability-simplify-*,readability-string-compare,readability-use-*' -header-filter='.*' src/*.cpp -- -Wall -O3 -std=gnu++20 continue-on-error: true - - name: ShellCheck - run: shopt -s globstar; shellcheck -o avoid-nullary-conditions,check-extra-masked-returns,check-set-e-suppressed,deprecate-which,quote-safe-variables,require-double-brackets -s bash **/*.sh + + ShellCheck: + name: ShellCheck + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Script + run: shopt -s globstar; shellcheck -o avoid-nullary-conditions,check-set-e-suppressed,deprecate-which,quote-safe-variables,require-double-brackets -s bash **/*.sh continue-on-error: true Windows: From ea498d2b63470f12ef580a4f65469fa9bf0a80e8 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Thu, 11 Jun 2026 13:37:12 -0700 Subject: [PATCH 083/214] Updated CI to test on Ubuntu 26.04. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa7b5f77..31d60e10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, ubuntu-24.04, ubuntu-22.04-arm, ubuntu-24.04-arm] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, ubuntu-22.04-arm, ubuntu-24.04-arm, ubuntu-26.04-arm] cxx: [g++, clang++] exclude: - os: ubuntu-22.04-arm @@ -46,7 +46,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04] cxx: [g++, clang++] exclude: - os: ubuntu-22.04-arm From 0bf908092d71bd9da05ec4da09e885f743650340 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Fri, 12 Jun 2026 05:45:57 -0700 Subject: [PATCH 084/214] Made Clang-Tidy fixes. --- src/AllocTrac.cpp | 1 - src/AllocTrac.h | 4 +- src/Args.cpp | 32 +-- src/Args.h | 10 +- src/Background.h | 15 +- src/Buffer.h | 10 +- src/Context.h | 2 +- src/Event.cpp | 3 +- src/Event.h | 2 +- src/FFTConfig.cpp | 88 +++---- src/FFTConfig.h | 34 +-- src/File.cpp | 3 +- src/File.h | 43 ++-- src/Gpu.cpp | 459 ++++++++++++++++++----------------- src/Gpu.h | 35 +-- src/Hash.h | 2 +- src/Kernel.cpp | 1 - src/Kernel.h | 4 +- src/KernelCompiler.cpp | 15 +- src/KernelCompiler.h | 6 +- src/MD5.h | 8 +- src/Primes.cpp | 8 +- src/Primes.h | 10 +- src/Profile.cpp | 2 +- src/Profile.h | 2 +- src/Proof.cpp | 54 +++-- src/Proof.h | 6 +- src/Queue.cpp | 18 +- src/Queue.h | 12 +- src/Saver.cpp | 53 ++-- src/Saver.h | 2 +- src/Sha3Hash.h | 2 +- src/Signal.cpp | 4 +- src/Task.cpp | 37 +-- src/TimeInfo.h | 2 +- src/Trig.cpp | 4 +- src/TrigBufCache.cpp | 294 +++++++++++----------- src/TrigBufCache.h | 27 ++- src/TuneEntry.cpp | 10 +- src/TuneEntry.h | 2 +- src/Worktodo.cpp | 30 +-- src/clwrap.cpp | 71 +++--- src/clwrap.h | 4 +- src/common.cpp | 14 +- src/common.h | 2 +- src/cuda/clwrap_cuda.cpp | 167 +++++++------ src/cuda/cudawrap.cpp | 36 +-- src/cuda/cudawrap.h | 21 +- src/cuda/tinycuda.h | 84 +++---- src/fs.cpp | 5 +- src/gpuid.cpp | 11 +- src/log.cpp | 10 +- src/main.cpp | 13 +- src/md5.cpp | 12 +- src/sha3.cpp | 4 +- src/shared.h | 4 +- src/state.cpp | 21 +- src/timeutil.cpp | 4 +- src/timeutil.h | 4 +- src/tinycl.h | 50 ++-- src/tune.cpp | 509 ++++++++++++++++++++------------------- 61 files changed, 1201 insertions(+), 1201 deletions(-) diff --git a/src/AllocTrac.cpp b/src/AllocTrac.cpp index b951bcd8..1995b719 100644 --- a/src/AllocTrac.cpp +++ b/src/AllocTrac.cpp @@ -1,7 +1,6 @@ // Copyright (C) Mihai Preda. #include "AllocTrac.h" -#include std::atomic AllocTrac::totalAlloc = 0; size_t AllocTrac::maxAlloc = size_t(15) * 1024 * 1024 * 1024; // 15 GB diff --git a/src/AllocTrac.h b/src/AllocTrac.h index 2bcedb4c..0c3169c4 100644 --- a/src/AllocTrac.h +++ b/src/AllocTrac.h @@ -36,8 +36,8 @@ class AllocTrac { AllocTrac(const AllocTrac&) = delete; void operator=(const AllocTrac&) = delete; - AllocTrac(AllocTrac&& rhs) : size(rhs.size) { rhs.size = 0; } - AllocTrac& operator=(AllocTrac&& rhs) { + AllocTrac(AllocTrac&& rhs) noexcept : size(rhs.size) { rhs.size = 0; } + AllocTrac& operator=(AllocTrac&& rhs) noexcept { AllocTrac tmp{std::move(rhs)}; swap(*this, tmp); return *this; diff --git a/src/Args.cpp b/src/Args.cpp index b9084ba3..6e115a16 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -16,7 +16,7 @@ #include // This is a copy of the args.verbose flag. It allows the CUDA wrapper to access the flag. -bool prpll_verbose = 0; +bool prpll_verbose = false; int Args::value(const string& key, int valNotFound) const { auto it = flags.find(key); @@ -52,7 +52,7 @@ vector Args::splitArgLine(const string& inputLine) { ret.push_back({prev, {}}); prev = s; } else { - ret.push_back({prev, s}); + ret.emplace_back(prev, s); prev.clear(); } } @@ -67,14 +67,14 @@ vector Args::splitArgLine(const string& inputLine) { // Splits a string of the form "Foo=bar,C,D=1" into key=value pairs, with value defaulting to "1". vector Args::splitUses(string ss) { // pass by value is intentional vector ret; - std::replace(ss.begin(), ss.end(), ',', ' '); + std::ranges::replace(ss, ',', ' '); std::istringstream iss{ss}; - vector uses{std::istream_iterator{iss}, std::istream_iterator{}}; + vector const uses{std::istream_iterator{iss}, std::istream_iterator{}}; for (const string &s : uses) { auto pos = s.find('='); - string key = (pos == string::npos) ? s : s.substr(0, pos); - string val = (pos == string::npos) ? "1"s : s.substr(pos+1); - ret.push_back({key, val}); + string const key = (pos == string::npos) ? s : s.substr(0, pos); + string const val = (pos == string::npos) ? "1"s : s.substr(pos+1); + ret.emplace_back(key, val); } return ret; } @@ -96,7 +96,7 @@ u32 Args::getProofPow(u64 exponent) const { string Args::tailDir() const { return fs::path{dir}.filename().string(); } -bool Args::hasFlag(const string& key) const { return flags.find(key) != flags.end(); } +bool Args::hasFlag(const string& key) const { return flags.contains(key); } void Args::printHelp() { printf(R"( @@ -223,7 +223,7 @@ Device selection : use one of -uid , -pci , -device , see the list } for (unsigned i = 0; i < deviceIds.size(); ++i) { cl_device_id id = deviceIds[i]; - string bdf = getBdfFromDevice(id); + string const bdf = getBdfFromDevice(id); printf("%2u : %7s | %16s | %-24s | %s | %s\n", i, bdf.c_str(), @@ -242,7 +242,7 @@ Device selection : use one of -uid , -pci , -device , see the list u32 activeSize = 0; float maxBpw = 0; string variants; - for (enum FFT_TYPES type : {FFT64, FFT3161, FFT3261, FFT61}) { + for (enum FFT_TYPES const type : {FFT64, FFT3161, FFT3261, FFT61}) { for (auto c : configs) { if (c.fft_type != type) continue; if (c.size() != activeSize) { @@ -273,8 +273,8 @@ void Args::parse(const string& line) { char fftBuf[32]; char configBuf[256]; sscanf(line.c_str(), "! %31s %255s", fftBuf, configBuf); - string fft = fftBuf; - string config = configBuf; + string const fft = fftBuf; + string const config = configBuf; perFftConfig[fft] = splitUses(config); return; } @@ -288,10 +288,10 @@ void Args::parse(const string& line) { if (key == "-h" || key == "--help") { printHelp(); throw "help"; - } else if (key == "-version") { + } if (key == "-version") { // log("PRPLL %s\n", VERSION); throw "version"; - } else if (key == "-info") { + } if (key == "-info") { if (s.empty()) { log("-info expects an FFT spec, e.g. -info 1K:13:256\n"); throw "-info "; @@ -300,12 +300,12 @@ void Args::parse(const string& line) { for (const FFTShape& shape : FFTShape::multiSpec(s)) { for (u32 variant = 0; variant <= LAST_VARIANT; variant = next_variant (variant)) { if (variant != LAST_VARIANT && shape.fft_type != FFT64) continue; - FFTConfig fft{shape, variant, CARRY_AUTO}; + FFTConfig const fft{shape, variant, CARRY_AUTO}; log("%12s | %.2f | %5.1f\n", fft.spec().c_str(), fft.maxBpw(), fft.maxExp() / 1'000'000.0); } } throw "info"; - } else if (key == "-od") { + } if (key == "-od") { double od = stod(s); fftOverdrive = 1 + od / 1000; } else if (key == "-roe") { diff --git a/src/Args.h b/src/Args.h index 99d3a87d..d876ce79 100644 --- a/src/Args.h +++ b/src/Args.h @@ -26,13 +26,13 @@ class Args { void parse(const string& line); void setDefaults(); - bool uses(const std::string& key) const { return flags.find(key) != flags.end(); } - int value(const std::string& key, int valNotFound = -1) const; + [[nodiscard]] bool uses(const std::string& key) const { return flags.contains(key); } + [[nodiscard]] int value(const std::string& key, int valNotFound = -1) const; void readConfig(const fs::path& path); - u32 getProofPow(u64 exponent) const; - string tailDir() const; + [[nodiscard]] u32 getProofPow(u64 exponent) const; + [[nodiscard]] string tailDir() const; - bool hasFlag(const string& key) const; + [[nodiscard]] bool hasFlag(const string& key) const; bool silent; string user; diff --git a/src/Background.h b/src/Background.h index 96cbe804..c93bf17f 100644 --- a/src/Background.h +++ b/src/Background.h @@ -18,7 +18,7 @@ class Background { std::deque > tasks; std::mutex mut; std::condition_variable cond; - bool stopRequested; + bool stopRequested{false}; std::jthread thread; void run() { @@ -30,9 +30,8 @@ class Background { while (tasks.empty()) { if (stopRequested) { return; - } else { - cond.wait(lock); - } + } cond.wait(lock); + } task = tasks.front(); } @@ -48,7 +47,7 @@ class Background { } { - std::unique_lock lock(mut); + std::unique_lock const lock(mut); assert(!tasks.empty()); tasks.pop_front(); if (tasks.size() == maxSize - 1 || tasks.empty()) { cond.notify_all(); } @@ -59,12 +58,12 @@ class Background { public: Background(unsigned size = 2) : maxSize{size}, - stopRequested(false), + thread{&Background::run, this} { } ~Background() { - std::lock_guard lock(mut); + std::scoped_lock const lock(mut); stopRequested = true; cond.notify_all(); } @@ -74,7 +73,7 @@ class Background { while (!tasks.empty()) { cond.wait(lock); } } - template void operator()(T task) { + template void operator()(const T& task) { std::unique_lock lock(mut); while (tasks.size() >= maxSize) { cond.wait(lock); diff --git a/src/Buffer.h b/src/Buffer.h index 4bd57c6d..8e76dca1 100644 --- a/src/Buffer.h +++ b/src/Buffer.h @@ -28,7 +28,7 @@ class Buffer { TimeInfo *tInfo; Buffer(cl_context context, TimeInfo *tInfo, Queue* queue, size_t size, unsigned flags, const T* ptr = nullptr) - : ptr{size == 0 ? NULL : makeBuf_(context, flags, size * sizeof(T), ptr)} + : ptr{size == 0 ? nullptr : makeBuf_(context, flags, size * sizeof(T), ptr)} , size{size} , allocTrac(size * sizeof(T)) , queue{queue} @@ -50,15 +50,15 @@ class Buffer { Buffer(TimeInfo *tInfo, Queue* queue, size_t size) : Buffer(queue->context->get(), tInfo, queue, size, CL_MEM_READ_WRITE /*| CL_MEM_HOST_NO_ACCESS*/) {} - Buffer(Buffer&& rhs) = default; + Buffer(Buffer&& rhs) noexcept = default; - Buffer& operator=(Buffer&& rhs) { + Buffer& operator=(Buffer&& rhs) noexcept { assert(size == rhs.size); std::swap(ptr, rhs.ptr); return *this; } - cl_mem get() const { return ptr.get(); } + [[nodiscard]] cl_mem get() const { return ptr.get(); } void read(T* out, size_t readSize) const { assert(readSize && readSize <= size); @@ -69,7 +69,7 @@ class Buffer { void read(vector& v) const { read(v.data(), v.size()); } // sync read - vector read(size_t sizeOrFull = 0) const { + [[nodiscard]] vector read(size_t sizeOrFull = 0) const { auto readSize = sizeOrFull ? sizeOrFull : size; vector ret(readSize); read(ret); diff --git a/src/Context.h b/src/Context.h index 3f281061..ec14002d 100644 --- a/src/Context.h +++ b/src/Context.h @@ -11,5 +11,5 @@ class Context : public std::unique_ptr { public: explicit Context(cl_device_id id): unique_ptr{createContext(id)}, id{id} {} - cl_device_id deviceId() const { return id; } + [[nodiscard]] cl_device_id deviceId() const { return id; } }; diff --git a/src/Event.cpp b/src/Event.cpp index f6098085..bd235177 100644 --- a/src/Event.cpp +++ b/src/Event.cpp @@ -4,6 +4,7 @@ #include "TimeInfo.h" #include +#include Event::Event(EventHolder&& e, TimeInfo* tInfo) : event{std::move(e)}, @@ -13,7 +14,7 @@ Event::Event(EventHolder&& e, TimeInfo* tInfo) : } Event::~Event() { - [[maybe_unused]] bool done = isComplete(); + [[maybe_unused]] bool const done = isComplete(); assert(done); } diff --git a/src/Event.h b/src/Event.h index 98810a5b..479391a7 100644 --- a/src/Event.h +++ b/src/Event.h @@ -17,7 +17,7 @@ class Event { Event(Event&& oth) = default; ~Event(); - cl_event get() const { return event.get(); } + [[nodiscard]] cl_event get() const { return event.get(); } bool isComplete(); bool isRunning(); diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index dd0568a7..83603861 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -23,7 +23,7 @@ struct FftBpw { array bpw; }; -map> BPW { +static map> BPW { #include "fftbpw.h" }; @@ -32,8 +32,8 @@ namespace { u32 parseInt(const string& s) { // if (s.empty()) { return 1; } assert(!s.empty()); - char c = s.back(); - u32 multiple = c == 'k' || c == 'K' ? 1024 : c == 'm' || c == 'M' ? 1024 * 1024 : 1; + char const c = s.back(); + u32 const multiple = c == 'k' || c == 'K' ? 1024 : c == 'm' || c == 'M' ? 1024 * 1024 : 1; return u32(strtod(s.c_str(), nullptr) * multiple); } @@ -59,18 +59,18 @@ vector FFTShape::multiSpec(const string& iniSpec) { } assert(parts.size() <= 3); if (parts.size() == 3) { - u32 width = parseInt(parts[0]); - u32 middle = parseInt(parts[1]); - u32 height = parseInt(parts[2]); - ret.push_back({fft_type, width, middle, height}); + u32 const width = parseInt(parts[0]); + u32 const middle = parseInt(parts[1]); + u32 const height = parseInt(parts[2]); + ret.emplace_back(fft_type, width, middle, height); continue; } assert(parts.size() == 1); parts = split(spec, '-'); - assert(parts.size() >= 1 && parts.size() <= 2); - u32 sizeFrom = parseInt(parts[0]); - u32 sizeTo = parts.size() == 2 ? parseInt(parts[1]) : sizeFrom; + assert(!parts.empty() && parts.size() <= 2); + u32 const sizeFrom = parseInt(parts[0]); + u32 const sizeTo = parts.size() == 2 ? parseInt(parts[1]) : sizeFrom; auto shapes = allShapes(sizeFrom, sizeTo); if (shapes.empty()) { log("Could not find a FFT config for '%s'\n", spec.c_str()); @@ -83,21 +83,21 @@ vector FFTShape::multiSpec(const string& iniSpec) { vector FFTShape::allShapes(u32 sizeFrom, u32 sizeTo) { vector configs; - for (enum FFT_TYPES type : {FFT64, FFT3161, FFT3261, FFT61, FFT323161}) { - for (u32 width : {256, 512, 1024, 4096}) { - for (u32 height : {256, 512, 1024}) { + for (enum FFT_TYPES const type : {FFT64, FFT3161, FFT3261, FFT61, FFT323161}) { + for (u32 const width : {256, 512, 1024, 4096}) { + for (u32 const height : {256, 512, 1024}) { if (width == 256 && height == 1024) { continue; } // Skip because we prefer width >= height - for (u32 middle : {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) { + for (u32 const middle : {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) { if (type != FFT64 && (middle & (middle - 1))) continue; // Reject non-power-of-two NTTs - u32 sz = width * height * middle * 2; + u32 const sz = width * height * middle * 2; if (sizeFrom <= sz && sz <= sizeTo) { - configs.push_back({type, width, middle, height}); + configs.emplace_back(type, width, middle, height); } } } } } - std::sort(configs.begin(), configs.end(), + std::ranges::sort(configs, [](const FFTShape &a, const FFTShape &b) { if (a.size() != b.size()) { return (a.size() < b.size()); } if (a.width != b.width) { @@ -133,7 +133,7 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : // Un-initialized shape, don't set BPW if (w == 1 && m == 1 && h == 1) { return; } - string s = spec(); + string const s = spec(); if (auto it = BPW.find(s); it != BPW.end()) { bpw = it->second; } else { @@ -141,9 +141,9 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : bpw = FFTShape{t, h, m, w}.bpw; } else { // Manipulate the shape into something that was likely pre-computed - u32 orig_w = w; - u32 orig_m = m; - u32 orig_h = h; + u32 const orig_w = w; + u32 const orig_m = m; + u32 const orig_h = h; while (m < 9) { m *= 2; w /= 2; } while (w >= 4*h) { w /= 2; h *= 2; } while (w < h || w < 256 || w == 2048) { w *= 2; h /= 2; } @@ -217,9 +217,9 @@ FFTConfig::FFTConfig(const string& spec) { // Sanity check the spec if (v.size() >= 3) { - u32 w = parseInt(v[0]); - u32 m = parseInt(v[1]); - u32 h = parseInt(v[2]); + u32 const w = parseInt(v[0]); + u32 const m = parseInt(v[1]); + u32 const h = parseInt(v[2]); if (w != 256 && w != 512 && w != 1024 && w != 4096) { log("Width must be 256, 512, 1024, or 4096.\n"); throw "Invalid FFT spec"; @@ -245,7 +245,7 @@ FFTConfig::FFTConfig(const string& spec) { } else if (v.size() == 4) { *this = {FFTShape{fft_type, v[0], v[1], v[2]}, parseInt(v[3]), CARRY_AUTO}; } else if (v.size() == 5) { - int c = parseInt(v[4]); + int const c = parseInt(v[4]); assert(c == 0 || c == 1); *this = {FFTShape{fft_type, v[0], v[1], v[2]}, parseInt(v[3]), c == 0 ? CARRY_32 : CARRY_64}; } else { @@ -262,20 +262,20 @@ FFTConfig::FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry) : assert(variant_M(variant) < N_VARIANT_M); assert(variant_H(variant) < N_VARIANT_H); - if (shape.fft_type == FFT64) FFT_FP64 = 1, FFT_FP32 = 0, NTT_GF31 = 0, NTT_GF61 = 0, WordSize = 4; - else if (shape.fft_type == FFT3161) FFT_FP64 = 0, FFT_FP32 = 0, NTT_GF31 = 1, NTT_GF61 = 1, WordSize = 8; - else if (shape.fft_type == FFT3261) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 0, NTT_GF61 = 1, WordSize = 8; - else if (shape.fft_type == FFT61) FFT_FP64 = 0, FFT_FP32 = 0, NTT_GF31 = 0, NTT_GF61 = 1, WordSize = 4; - else if (shape.fft_type == FFT323161) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 1, NTT_GF61 = 1, WordSize = 8; - else if (shape.fft_type == FFT3231) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 1, NTT_GF61 = 0, WordSize = 4; - else if (shape.fft_type == FFT6431) FFT_FP64 = 1, FFT_FP32 = 0, NTT_GF31 = 1, NTT_GF61 = 0, WordSize = 8; - else if (shape.fft_type == FFT31) FFT_FP64 = 0, FFT_FP32 = 0, NTT_GF31 = 1, NTT_GF61 = 0, WordSize = 4; - else if (shape.fft_type == FFT32) FFT_FP64 = 0, FFT_FP32 = 1, NTT_GF31 = 0, NTT_GF61 = 0, WordSize = 4; + if (shape.fft_type == FFT64) FFT_FP64 = true, FFT_FP32 = false, NTT_GF31 = false, NTT_GF61 = false, WordSize = 4; + else if (shape.fft_type == FFT3161) FFT_FP64 = false, FFT_FP32 = false, NTT_GF31 = true, NTT_GF61 = true, WordSize = 8; + else if (shape.fft_type == FFT3261) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = false, NTT_GF61 = true, WordSize = 8; + else if (shape.fft_type == FFT61) FFT_FP64 = false, FFT_FP32 = false, NTT_GF31 = false, NTT_GF61 = true, WordSize = 4; + else if (shape.fft_type == FFT323161) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = true, NTT_GF61 = true, WordSize = 8; + else if (shape.fft_type == FFT3231) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = true, NTT_GF61 = false, WordSize = 4; + else if (shape.fft_type == FFT6431) FFT_FP64 = true, FFT_FP32 = false, NTT_GF31 = true, NTT_GF61 = false, WordSize = 8; + else if (shape.fft_type == FFT31) FFT_FP64 = false, FFT_FP32 = false, NTT_GF31 = true, NTT_GF61 = false, WordSize = 4; + else if (shape.fft_type == FFT32) FFT_FP64 = false, FFT_FP32 = true, NTT_GF31 = false, NTT_GF61 = false, WordSize = 4; else throw "FFT type"; } string FFTConfig::spec() const { - string s = shape.spec() + ":" + to_string(variant_W(variant)) + to_string(variant_M(variant)) + to_string(variant_H(variant)); + string const s = shape.spec() + ":" + to_string(variant_W(variant)) + to_string(variant_M(variant)) + to_string(variant_H(variant)); return carry == CARRY_AUTO ? s : (s + (carry == CARRY_32 ? ":0" : ":1")); } @@ -289,8 +289,8 @@ float FFTConfig::maxBpw() const { } // Interpolate for the maximum bpw. This might could be improved upon. However, I doubt people will use these variants often. else { - float b1 = shape.bpw[variant_M(variant) * 3 + variant_W(variant)]; - float b2 = shape.bpw[variant_M(variant) * 3 + variant_H(variant)]; + float const b1 = shape.bpw[variant_M(variant) * 3 + variant_W(variant)]; + float const b2 = shape.bpw[variant_M(variant) * 3 + variant_H(variant)]; b = (b1 + b2) / 2.0f; } // Only some FFTs support both 32 and 64 bit carries. @@ -308,7 +308,7 @@ FFTConfig FFTConfig::bestFit(const Args& args, u64 E, const string& spec) { } // No FFT-spec given, so choose from tune.txt the fastest FFT that can handle E - vector tunes = TuneEntry::readTuneFile(args); + vector const tunes = TuneEntry::readTuneFile(args); for (const TuneEntry& e : tunes) { // The first acceptable is the best as they're sorted by cost if (E <= e.fft.maxExp() * args.fftOverdrive) { return e.fft; } @@ -318,7 +318,7 @@ FFTConfig FFTConfig::bestFit(const Args& args, u64 E, const string& spec) { // Take the first FFT that can handle E for (const FFTShape& shape : FFTShape::allShapes()) { - for (u32 v : {101, 202}) { + for (u32 const v : {101, 202}) { if (FFTConfig fft{shape, v, CARRY_AUTO}; fft.maxExp() * args.fftOverdrive >= E) { return fft; } } } @@ -329,8 +329,8 @@ FFTConfig FFTConfig::bestFit(const Args& args, u64 E, const string& spec) { string numberK(u64 n) { - u32 K = 1024; - u32 M = K * K; + u32 const K = 1024; + u32 const M = K * K; if (n % M == 0) { return to_string(n / M) + 'M'; } @@ -338,10 +338,10 @@ string numberK(u64 n) { if (n >= M && (n * u64(100)) % M == 0) { snprintf(buf, sizeof(buf), "%.2f", float(n) / M); return string(buf) + 'M'; - } else if (n >= K) { + } if (n >= K) { snprintf(buf, sizeof(buf), "%g", float(n) / K); return string(buf) + 'K'; - } else { + } return to_string(n); - } + } diff --git a/src/FFTConfig.h b/src/FFTConfig.h index e16d610a..5bc719e7 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -12,7 +12,9 @@ // We pre-calculate the maximum BPW for a number of fft specs. From these entries we can either look up or interpolate to get the // maximum BPW for all variants of an FFT spec. The variants for which maximum bpw are precomputed are 000, 101, 202, 010, 111, 212. -#define NUM_BPW_ENTRIES 6 +enum { +NUM_BPW_ENTRIES = 6 +}; class Args; @@ -39,17 +41,17 @@ class FFTShape { FFTShape(enum FFT_TYPES t, const string& w, const string& m, const string& h); explicit FFTShape(const string& spec); - u32 size() const { return width * height * middle * 2; } - u32 nW() const { return (width == 1024 || width == 256 /*|| width == 4096*/) ? 4 : 8; } - u32 nH() const { return (height == 1024 || height == 256 /*|| height == 4096*/) ? 4 : 8; } + [[nodiscard]] u32 size() const { return width * height * middle * 2; } + [[nodiscard]] u32 nW() const { return (width == 1024 || width == 256 /*|| width == 4096*/) ? 4 : 8; } + [[nodiscard]] u32 nH() const { return (height == 1024 || height == 256 /*|| height == 4096*/) ? 4 : 8; } - float minBpw() const { return fft_type != FFT32 ? 3.0f : 1.0f; } - float maxBpw() const { return *max_element(bpw.begin(), bpw.end()); } - std::string spec() const { return (fft_type ? to_string(fft_type) + ':' : "") + numberK(width) + ':' + numberK(middle) + ':' + numberK(height); } + [[nodiscard]] float minBpw() const { return fft_type != FFT32 ? 3.0f : 1.0f; } + [[nodiscard]] float maxBpw() const { return *std::ranges::max_element(bpw); } + [[nodiscard]] std::string spec() const { return (fft_type ? to_string(fft_type) + ':' : "") + numberK(width) + ':' + numberK(middle) + ':' + numberK(height); } - float carry32BPW() const; - bool needsLargeCarry(u64 E) const; - bool isFavoredShape() const; + [[nodiscard]] float carry32BPW() const; + [[nodiscard]] bool needsLargeCarry(u64 E) const; + [[nodiscard]] bool isFavoredShape() const; }; static const u32 N_VARIANT_W = 3; @@ -82,17 +84,17 @@ struct FFTConfig { // Size (in bytes) of integer data passed to FFTs/NTTs on the GPU u32 WordSize; - FFTShape shape{}; + FFTShape shape; u32 variant; enum CARRY_KIND carry; explicit FFTConfig(const string& spec); FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry); - std::string spec() const; - u64 size() const { return shape.size(); } - u64 maxExp() const { return u64(maxBpw() * shape.size()); } + [[nodiscard]] std::string spec() const; + [[nodiscard]] u64 size() const { return shape.size(); } + [[nodiscard]] u64 maxExp() const { return u64(maxBpw() * shape.size()); } - float minBpw() const { return shape.minBpw(); } - float maxBpw() const; + [[nodiscard]] float minBpw() const { return shape.minBpw(); } + [[nodiscard]] float maxBpw() const; }; diff --git a/src/File.cpp b/src/File.cpp index 1360b3f3..56d258b1 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -3,6 +3,7 @@ #include "File.h" #include #include +#include using namespace std; @@ -40,7 +41,7 @@ i64 File::size(const fs::path &name) { return filesystem::file_size(name, dummy); } -File& File::operator=(File&& other) { +File& File::operator=(File&& other) noexcept { assert(this != &other); this->~File(); new (this) File(std::move(other)); diff --git a/src/File.h b/src/File.h index 52b84d7c..fdc3e3ac 100644 --- a/src/File.h +++ b/src/File.h @@ -12,6 +12,7 @@ #include #endif #include +#include #include #include #include @@ -20,7 +21,7 @@ #include #endif -#if defined(__APPLE__) +#ifdef __APPLE__ #include #endif @@ -67,7 +68,7 @@ class File { void datasync() { fflush(f); -#if defined(_MSC_VER) +#ifdef _MSC_VER // We'd really like to use FlushFileBuffers(h), but we do not have easy access to the Windows file handle. // We might could get that by getting the pathname and opening the file with native Windows routines. #elif defined(_WIN32) || defined(__WIN32__) @@ -99,13 +100,13 @@ class File { static void append(const fs::path& name, std::string_view text) { File::openAppend(name).write(text); } - File() : f{}, readOnly{true} {} + File() : readOnly{true} {} - File(FILE* f, const string& name) : f{f}, readOnly{false}, name{name} {} + File(FILE* f, string name) : f{f}, readOnly{false}, name{std::move(name)} {} - File(File&& other) : f{other.f}, readOnly{other.readOnly}, name{other.name} { other.f = nullptr; } + File(File&& other) noexcept : f{other.f}, readOnly{other.readOnly}, name{other.name} { other.f = nullptr; } - File& operator=(File&& other); + File& operator=(File&& other) noexcept ; File(const File& other) = delete; File& operator=(const File& other) = delete; @@ -146,7 +147,7 @@ class File { } void seek(long offset, int whence = SEEK_SET) { - int ret = fseek(this->get(), offset, whence); + int const ret = fseek(this->get(), offset, whence); if (ret) { throw ReadError{name}; } // throw(std::ios_base::failure(("fseek: "s + to_string(ret)).c_str())); } @@ -156,7 +157,7 @@ class File { int printf(const char *fmt, ...) const FORMAT_PRINTF(2, 3) { va_list va; va_start(va, fmt); - int ret = vfprintf(f, fmt, va); + int const ret = vfprintf(f, fmt, va); va_end(va); #if !HAS_LINEBUF @@ -169,7 +170,7 @@ class File { int scanf(const char *fmt, ...) FORMAT_SCANF(2, 3) { va_list va; va_start(va, fmt); - int ret = vfscanf(f, fmt, va); + int const ret = vfscanf(f, fmt, va); va_end(va); return ret; } @@ -179,10 +180,10 @@ class File { void write(string_view s) { write(s.data(), u32(s.size())); } operator bool() const { return f != nullptr; } - FILE* get() const { return f; } + [[nodiscard]] FILE* get() const { return f; } - long ftell() const { - long pos = ::ftell(this->get()); + [[nodiscard]] long ftell() const { + long const pos = ::ftell(this->get()); assert(pos >= 0); return pos; } @@ -193,8 +194,8 @@ class File { } long size() { - long savePos = ftell(); - long retSize = seekEnd(); + long const savePos = ftell(); + long const retSize = seekEnd(); seek(savePos); return retSize; } @@ -205,7 +206,7 @@ class File { std::string readLine() { char buf[1024]; buf[0] = 0; - bool ok = fgets(buf, sizeof(buf), this->get()); + bool const ok = fgets(buf, sizeof(buf), this->get()); if (!ok) { return ""; } // EOF or error string line = buf; if (line.empty() || line.back() != '\n') { @@ -222,7 +223,7 @@ class File { } template - std::vector read(u32 nWords) const { + [[nodiscard]] [[nodiscard]] std::vector read(u32 nWords) const { vector ret; ret.resize(nWords); read(ret.data(), nWords * sizeof(T)); @@ -230,8 +231,8 @@ class File { } template - std::vector readChecked(u32 nWords) const { - u32 expectedCRC = read(1)[0]; + [[nodiscard]] std::vector readChecked(u32 nWords) const { + u32 const expectedCRC = read(1)[0]; return readWithCRC(nWords, expectedCRC); } @@ -242,7 +243,7 @@ class File { } template - std::vector readWithCRC(u32 nWords, u32 crc) const { + [[nodiscard]] std::vector readWithCRC(u32 nWords, u32 crc) const { auto data = read(nWords); if (crc != crc32(data)) { log("File '%s' : CRC: expected %u, actual %u\n", name.c_str(), crc, crc32(data)); @@ -253,7 +254,7 @@ class File { std::vector readBytesLE(u32 nBytes) { assert(nBytes > 0); - u32 nWords = (nBytes - 1) / 4 + 1; + u32 const nWords = (nBytes - 1) / 4 + 1; vector data(nWords); read(data.data(), nBytes); return data; @@ -262,7 +263,7 @@ class File { u32 readUpTo(void* data, u32 nUpToBytes) { return u32(fread(data, 1, nUpToBytes, this->get())); } string readAll() { - u32 sz = u32(size()); + u32 const sz = u32(size()); return {read(sz).data(), sz}; } }; diff --git a/src/Gpu.cpp b/src/Gpu.cpp index f16fe0e9..fd63bdde 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -18,16 +18,16 @@ #include "Sha3Hash.h" #include -#include #include #include #include #include #include -#include #define _USE_MATH_DEFINES #include +#include +#include #ifndef M_PIl #define M_PIl 3.141592653589793238462643383279502884L @@ -45,7 +45,9 @@ #define M_LN2 0.69314718055994530941723212145818 #endif -#define CARRY_LEN 8 +enum { +CARRY_LEN = 8 +}; namespace { @@ -87,8 +89,8 @@ float invWeightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { } Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { - u32 N = 2u * W * H; - u32 groupWidth = W / nW; + u32 const N = 2u * W * H; + u32 const groupWidth = W / nW; vector weightsConstIF; vector weightsIF; @@ -162,7 +164,7 @@ Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { memcpy((double *) weightsConstIF.data(), weightsConstIF32.data(), weightsConstIF32.size() * sizeof(float)); } - return Weights{weightsConstIF, weightsIF}; + return Weights{.weightsConstIF=weightsConstIF, .weightsIF=weightsIF}; } string toLiteral(i32 value) { return to_string(value); } @@ -217,7 +219,7 @@ string toLiteral(const string& s) { return s; } [[maybe_unused]] string toLiteral(ulong2 cs) { return "U2("s + toLiteral(cs.first) + ',' + toLiteral(cs.second) + ')'; } template -string toDefine(const string& k, T v) { return " -D"s + k + '=' + toLiteral(v); } +string toDefine(const string& k, const T& v) { return " -D"s + k + '=' + toLiteral(v); } template string toDefine(const T& vect) { @@ -248,14 +250,14 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< } // Default value for -use options that must also be parsed in C++ code - tail_single_wide = 0, tail_single_kernel = 1; // Default tailSquare is double-wide in one kernel + tail_single_wide = false, tail_single_kernel = true; // Default tailSquare is double-wide in one kernel in_place = 0; // Default is not in-place wmul = 2; // Default is carryFused processes two lines at a time pad_size = isAmdGpu(id) ? 256 : 0; // Default is 256 bytes for AMD, 0 for others // Validate -use options for (const auto& [k, v] : config) { - bool isValid = isInList(k, { + bool const isValid = isInList(k, { "FAST_BARRIER", "STATS", "IN_SIZEX", @@ -295,10 +297,10 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Some -use options are needed in both OpenCL code and C++ initialization code if (k == "TAIL_KERNELS") { - if (atoi(v.c_str()) == 0) tail_single_wide = 1, tail_single_kernel = 1; - if (atoi(v.c_str()) == 1) tail_single_wide = 1, tail_single_kernel = 0; - if (atoi(v.c_str()) == 2) tail_single_wide = 0, tail_single_kernel = 1; - if (atoi(v.c_str()) == 3) tail_single_wide = 0, tail_single_kernel = 0; + if (atoi(v.c_str()) == 0) tail_single_wide = true, tail_single_kernel = true; + if (atoi(v.c_str()) == 1) tail_single_wide = true, tail_single_kernel = false; + if (atoi(v.c_str()) == 2) tail_single_wide = false, tail_single_kernel = true; + if (atoi(v.c_str()) == 3) tail_single_wide = false, tail_single_kernel = false; } if (k == "INPLACE") in_place = atoi(v.c_str()); if (k == "WMUL") wmul = atoi(v.c_str()); @@ -308,7 +310,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< // Maximum WMUL is 32KB / (WIDTH * SHUFL_BYTES_W). If using the 32KB maximum, LDS padding must be disabled. // Furthermore, I've seen the CUDA compiler refuse to create a kernel with 1024 threads. Thus, we limit WMUL to 2 for a 1K width and to 1 for a 4K width. { - u32 shufl_bytes_w = args.value("SHUFL_BYTES_W", 8); + u32 const shufl_bytes_w = args.value("SHUFL_BYTES_W", 8); u32 max_wmul = 32768 / (fft.shape.width * shufl_bytes_w); if (max_wmul > 2 && fft.shape.width >= 1024) max_wmul = 2; if (max_wmul > 1 && fft.shape.width >= 4096) max_wmul = 1; @@ -345,7 +347,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< defines += toDefine("CARRY64", 1); } - u32 N = fft.shape.size(); + u32 const N = fft.shape.size(); defines += toDefine("FFT_VARIANT", fft.variant); defines += toDefine("MAXBPW", (u32)(fft.maxBpw() * 100.0f)); @@ -355,7 +357,7 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< if (fft.FFT_FP64) defines += toDefine("TAILT", root1Fancy(fft.shape.height * 2, 1)); else defines += toDefine("TAILT", root1FancyFP32(fft.shape.height * 2, 1)); - TrigCoefs coefs = trigCoefs(fft.shape.size() / 4); + TrigCoefs const coefs = trigCoefs(fft.shape.size() / 4); defines += toDefine("TRIG_SCALE", int(coefs.scale)); defines += toDefine("TRIG_SIN", coefs.sinCoefs); defines += toDefine("TRIG_COS", coefs.cosCoefs); @@ -431,8 +433,8 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< } // Calculate fractional bits-per-word = (E % N) / N * 2^64 - u32 bpw_hi = (u64(E % N) << 32) / N; - u32 bpw_lo = (((u64(E % N) << 32) % N) << 32) / N; + u32 const bpw_hi = (u64(E % N) << 32) / N; + u32 const bpw_lo = (((u64(E % N) << 32) % N) << 32) / N; u64 bpw = (u64(bpw_hi) << 32) + bpw_lo; bpw--; // bpw must not be an exact value -- it must be less than exact value to get last biglit value right defines += toDefine("FRAC_BPW_HI", (u32) (bpw >> 32)); @@ -465,16 +467,16 @@ RoeInfo roeStat(const vector& roe) { double maxRoe = 0; for (auto xf : roe) { - double x = xf; + double const x = xf; assert(x >= 0); maxRoe = max(x, maxRoe); sumRoe += x; sum2Roe += x * x; } - u32 n = roe.size(); + u32 const n = roe.size(); - double sdRoe = sqrt(n * sum2Roe - sumRoe * sumRoe) / n; - double meanRoe = sumRoe / n; + double const sdRoe = sqrt(n * sum2Roe - sumRoe * sumRoe) / n; + double const meanRoe = sumRoe / n; return {n, maxRoe, meanRoe, sdRoe}; } @@ -487,9 +489,9 @@ class IterationTimer { explicit IterationTimer(u64 kStart) : kStart(kStart) { } float reset(u64 k) { - float secs = timer.reset(); + float const secs = timer.reset(); - u64 its = max(u64(1), k - kStart); + u64 const its = max(u64(1), k - kStart); kStart = k; return secs / its; } @@ -508,7 +510,7 @@ u32 baseCheckStep(u32 blockSize) { } u32 checkStepForErrors(u32 blockSize, u32 nErrors) { - u32 step = baseCheckStep(blockSize); + u32 const step = baseCheckStep(blockSize); return nErrors ? step / 2 : step; } @@ -528,7 +530,7 @@ string toHex(const vector& v) { string formatSecsPerIter(float secsPerIter) { char buf[64]; - float usecsPerIter = secsPerIter * 1.0e6f; // Convert to micro-seconds + float const usecsPerIter = secsPerIter * 1.0e6f; // Convert to micro-seconds if (usecsPerIter > 1000.0f) { snprintf(buf, sizeof(buf), "%4.0f", usecsPerIter); } else { @@ -686,7 +688,7 @@ string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { break; } // Get the optional override register count - int override_regs = args.value(use_override, 0); + int const override_regs = args.value(use_override, 0); // If a specified override is small, use the count as a CUDA launch_bounds rather than a maximum register count if (override_regs && (override_regs > 0 && override_regs <= 16)) return string("-DCUDA_MIN_BLOCKS=") + to_string(override_regs) + " "; // If specified, override the default maximum register count @@ -734,8 +736,10 @@ string Gpu::kernelDefines(enum WHICH_KERNEL_TYPE which_kernel) { return defines + " "; } -#define ROE_SIZE 100000 -#define CARRY_SIZE 100000 +enum { +ROE_SIZE = 100000, +CARRY_SIZE = 100000 +}; Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : shared(s), @@ -752,63 +756,63 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo nH(fft.shape.nH()), useLongCarry{args.carry == CARRY_64}, queue{*shared.context, args.profile}, - auxQueues{}, + compiler{args, shared.context, clDefines(args, shared.context->deviceId(), fft, extraConf, E, logFftSize, tail_single_wide, tail_single_kernel, in_place, pad_size, wmul)}, #define K(name, ...) name(#name, &compiler, profile.make(#name), &queue, __VA_ARGS__) - K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), (kernelDefines(KFP) + numCudaRegisters(MIDIN)).c_str()), - K(kfftHin, "ffthin.cl", "fftHin", hN / nH, kernelDefines(KFP).c_str()), - K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2, kernelDefines(KFP).c_str()), + K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numCudaRegisters(MIDIN)), + K(kfftHin, "ffthin.cl", "fftHin", hN / nH, kernelDefines(KFP)), + K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2, kernelDefines(KFP)), K(ktailSquare, "tailsquare.cl", "tailSquare", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, (kernelDefines(KFP) + numCudaRegisters(TAIL)).c_str()), // Single-wide tailSquare with one kernel - K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2, kernelDefines(KFP).c_str()), - K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, (kernelDefines(KFP) + "-DMUL_LOW=1").c_str()), - K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), (kernelDefines(KFP) + numCudaRegisters(MIDOUT)).c_str()), - K(kfftW, "fftw.cl", "fftW", hN / nW, kernelDefines(KFP).c_str()), - - K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), (kernelDefines(K31) + numCudaRegisters(MIDIN31)).c_str()), - K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH, kernelDefines(K31).c_str()), - K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2, kernelDefines(K31).c_str()), + hN / nH / 2, kernelDefines(KFP) + numCudaRegisters(TAIL)), // Single-wide tailSquare with one kernel + K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2, kernelDefines(KFP)), + K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, kernelDefines(KFP) + "-DMUL_LOW=1"), + K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numCudaRegisters(MIDOUT)), + K(kfftW, "fftw.cl", "fftW", hN / nW, kernelDefines(KFP)), + + K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numCudaRegisters(MIDIN31)), + K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH, kernelDefines(K31)), + K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2, kernelDefines(K31)), K(ktailSquareGF31, "tailsquare.cl", "tailSquareGF31", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, (kernelDefines(K31) + numCudaRegisters(TAIL31)).c_str()), // Single-wide tailSquare with one kernel - K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, kernelDefines(K31).c_str()), - K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, (kernelDefines(K31) + "-DMUL_LOW=1").c_str()), - K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), (kernelDefines(K31) + numCudaRegisters(MIDOUT31)).c_str()), - K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW, kernelDefines(K31).c_str()), - - K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), (kernelDefines(K61) + numCudaRegisters(MIDIN61)).c_str()), - K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH, kernelDefines(K61).c_str()), - K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2, kernelDefines(K61).c_str()), + hN / nH / 2, kernelDefines(K31) + numCudaRegisters(TAIL31)), // Single-wide tailSquare with one kernel + K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, kernelDefines(K31)), + K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, kernelDefines(K31) + "-DMUL_LOW=1"), + K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numCudaRegisters(MIDOUT31)), + K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW, kernelDefines(K31)), + + K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numCudaRegisters(MIDIN61)), + K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH, kernelDefines(K61)), + K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2, kernelDefines(K61)), K(ktailSquareGF61, "tailsquare.cl", "tailSquareGF61", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, (kernelDefines(K61) + numCudaRegisters(TAIL61)).c_str()), // Single-wide tailSquare with one kernel - K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, kernelDefines(K61).c_str()), - K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, (kernelDefines(K61) + "-DMUL_LOW=1").c_str()), - K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), (kernelDefines(K61) + numCudaRegisters(MIDOUT61)).c_str()), - K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW, kernelDefines(K61).c_str()), - - K(kfftP, "fftp.cl", "fftP", hN / nW, kernelDefines(KALL).c_str()), - K(kCarryA, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL).c_str()), - K(kCarryAROE, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DROE=1").c_str()), - K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DMUL3=1").c_str()), - K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DMUL3=1 -DROE=1").c_str()), - K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, (kernelDefines(KALL) + "-DLL=1").c_str()), - K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED)).c_str()), - K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DROE=1").c_str()), - K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1").c_str()), - K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1").c_str()), - K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, (kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DLL=1").c_str()), - - K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN, kernelDefines(KALL).c_str()), + hN / nH / 2, kernelDefines(K61) + numCudaRegisters(TAIL61)), // Single-wide tailSquare with one kernel + K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, kernelDefines(K61)), + K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, kernelDefines(K61) + "-DMUL_LOW=1"), + K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numCudaRegisters(MIDOUT61)), + K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW, kernelDefines(K61)), + + K(kfftP, "fftp.cl", "fftP", hN / nW, kernelDefines(KALL)), + K(kCarryA, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL)), + K(kCarryAROE, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DROE=1"), + K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DMUL3=1"), + K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DMUL3=1 -DROE=1"), + K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DLL=1"), + K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED)), + K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DROE=1"), + K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1"), + K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1"), + K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DLL=1"), + + K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN, kernelDefines(KALL)), // 64 K(transpIn, "transpose.cl", "transposeIn", hN / 64), @@ -859,13 +863,11 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo #undef BUF statsBits{u32(args.value("STATS", 0))}, - timeBufVect{profile.make("proofBufVect")}, - - recorded_kernels{}, - recorded_kernel_args{} + timeBufVect{profile.make("proofBufVect")} + { - float bitsPerWord = E / float(N); + float const bitsPerWord = E / float(N); if (logFftSize) { log("FFT: %s %s (%.2f bpw)\n", numberK(N).c_str(), fft.spec().c_str(), bitsPerWord); @@ -969,20 +971,20 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo // If the L2 cache is large enough so that all FFT data fits in the cache, this ought to be a win. // If the L2 cache is small enough such that L2 cache hits are very low anyway, this might be a win. -void Gpu::splitQueue(void) { +void Gpu::splitQueue() { // If MULTI_Q -use not set, return if (!args.value("MULTI_Q", 0)) return; // Create aux queues. For now, we only have one auxiliary queue. We could do more. - if (auxQueues.size() == 0) { - auxQueues.push_back(Queue{*shared.context, args.profile, true}); + if (auxQueues.empty()) { + auxQueues.emplace_back(*shared.context, args.profile, true); } // Queue a sync event in the main queue. Have all auxiliary queues wait on the event. EventHolder event = queue.createSyncEvent(); - for (size_t i = 0; i < auxQueues.size(); ++i) { - auxQueues[i].waitForSyncEvent(&event); + for (auto & auxQueue : auxQueues) { + auxQueue.waitForSyncEvent(&event); } // Assign kernels to running on the main queue or an auxiliary queue @@ -1036,14 +1038,14 @@ void Gpu::splitQueue(void) { } } -void Gpu::mergeQueue(void) { +void Gpu::mergeQueue() { // If MULTI_Q -use not set, return if (!args.value("MULTI_Q", 0)) return; // Queue a sync event in each auxiliary queue(s). Wait on the event(s) in the main queue. - for (size_t i = 0; i < auxQueues.size(); ++i) { - EventHolder event = auxQueues[i].createSyncEvent(); + for (auto & auxQueue : auxQueues) { + EventHolder event = auxQueue.createSyncEvent(); queue.waitForSyncEvent(&event); } @@ -1083,7 +1085,7 @@ void Gpu::mergeQueue(void) { // Replay the recorded bottom half kernels in a cache friendly order. We support several // options here using multiple openCl command queues. -void Gpu::replay(void) { +void Gpu::replay() { // If using multiple command queues, handle that now. splitQueue(); @@ -1102,28 +1104,28 @@ void Gpu::replay(void) { // Call the appropriate kernel if (kern == KMIDIN) { - Buffer *buf = recorded_kernel_args[arg++]; + Buffer const*buf = recorded_kernel_args[arg++]; // If not in place, the input is from the scratch buffer - Buffer *in = in_place ? buf : &buf3; - Buffer *out = buf; + Buffer const*in = in_place ? buf : &buf3; + Buffer const*out = buf; if (cache_group == 1) kfftMidIn(*out, *in); if (cache_group == 2) kfftMidInGF31(*out, *in); if (cache_group == 3) kfftMidInGF61(*out, *in); } if (kern == KFFTHIN) { - Buffer *out = recorded_kernel_args[arg++]; - Buffer *in = recorded_kernel_args[arg++]; + Buffer const*out = recorded_kernel_args[arg++]; + Buffer const*in = recorded_kernel_args[arg++]; if (cache_group == 1) kfftHin(*out, *in); if (cache_group == 2) kfftHinGF31(*out, *in); if (cache_group == 3) kfftHinGF61(*out, *in); } if (kern == KTAILSQUARE) { - Buffer *buf = recorded_kernel_args[arg++]; + Buffer const*buf = recorded_kernel_args[arg++]; // If not in place, the output is to the scratch buffer - Buffer *in = buf; - Buffer *out = in_place ? buf : &buf3; + Buffer const*in = buf; + Buffer const*out = in_place ? buf : &buf3; if (!tail_single_kernel) { if (cache_group == 1) ktailSquareZero(*out, *in); if (cache_group == 2) ktailSquareZeroGF31(*out, *in); @@ -1135,40 +1137,40 @@ void Gpu::replay(void) { } if (kern == KTAILMUL) { - Buffer *buf = recorded_kernel_args[arg++]; - Buffer *in2 = recorded_kernel_args[arg++]; + Buffer const*buf = recorded_kernel_args[arg++]; + Buffer const*in2 = recorded_kernel_args[arg++]; // If not in place, the output is to the scratch buffer - Buffer *in1 = buf; - Buffer *out = in_place ? buf : &buf3; + Buffer const*in1 = buf; + Buffer const*out = in_place ? buf : &buf3; if (cache_group == 1) ktailMul(*out, *in1, *in2); if (cache_group == 2) ktailMulGF31(*out, *in1, *in2); if (cache_group == 3) ktailMulGF61(*out, *in1, *in2); } if (kern == KTAILMULLOW) { - Buffer *buf = recorded_kernel_args[arg++]; - Buffer *in2 = recorded_kernel_args[arg++]; + Buffer const*buf = recorded_kernel_args[arg++]; + Buffer const*in2 = recorded_kernel_args[arg++]; // If not in place, the output is to the scratch buffer - Buffer *in1 = buf; - Buffer *out = in_place ? buf : &buf3; + Buffer const*in1 = buf; + Buffer const*out = in_place ? buf : &buf3; if (cache_group == 1) ktailMulLow(*out, *in1, *in2); if (cache_group == 2) ktailMulLowGF31(*out, *in1, *in2); if (cache_group == 3) ktailMulLowGF61(*out, *in1, *in2); } if (kern == KMIDOUT) { - Buffer *buf = recorded_kernel_args[arg++]; + Buffer const*buf = recorded_kernel_args[arg++]; // If not in place, the input is from the scratch buffer - Buffer *in = in_place ? buf : &buf3; - Buffer *out = buf; + Buffer const*in = in_place ? buf : &buf3; + Buffer const*out = buf; if (cache_group == 1) kfftMidOut(*out, *in); if (cache_group == 2) kfftMidOutGF31(*out, *in); if (cache_group == 3) kfftMidOutGF61(*out, *in); } if (kern == KFFTW) { - Buffer *out = recorded_kernel_args[arg++]; - Buffer *in = recorded_kernel_args[arg++]; + Buffer const*out = recorded_kernel_args[arg++]; + Buffer const*in = recorded_kernel_args[arg++]; if (cache_group == 1) kfftW(*out, *in); if (cache_group == 2) kfftWGF31(*out, *in); if (cache_group == 3) kfftWGF61(*out, *in); @@ -1188,7 +1190,7 @@ void Gpu::replay(void) { void Gpu::fftP(Buffer& buf, Buffer& in) { // If not in place, instead write the output to the scratch buffer - Buffer *out = in_place ? &buf : &buf3; + Buffer const*out = in_place ? &buf : &buf3; kfftP(*out, in); } @@ -1261,8 +1263,8 @@ void Gpu::carryFused(Buffer& buf) { replay(); assert(roePos <= ROE_SIZE); // Like fftP, if not in place write the output to the scratch buffer - Buffer *in = &buf; - Buffer *out = in_place ? &buf : &buf3; + Buffer const*in = &buf; + Buffer const*out = in_place ? &buf : &buf3; roePos < wantROE ? kCarryFusedROE(*out, *in, roePos++) : kCarryFused(*out, *in, updateCarryPos(1 << 0)); } @@ -1272,8 +1274,8 @@ void Gpu::carryFusedMul(Buffer& buf) { replay(); assert(roePos <= ROE_SIZE); // Like fftP, if not in place write the output to the scratch buffer - Buffer *in = &buf; - Buffer *out = in_place ? &buf : &buf3; + Buffer const*in = &buf; + Buffer const*out = in_place ? &buf : &buf3; roePos < wantROE ? kCarryFusedMulROE(*out, *in, roePos++) : kCarryFusedMul(*out, *in, updateCarryPos(1 << 1)); } @@ -1282,8 +1284,8 @@ void Gpu::carryFusedLL(Buffer& buf) { // This kernel always ends the "bottom half". Replay the recorded kernel calls. replay(); // Like fftP, if not in place write the output to the scratch buffer - Buffer *in = &buf; - Buffer *out = in_place ? &buf : &buf3; + Buffer const*in = &buf; + Buffer const*out = in_place ? &buf : &buf3; kCarryFusedLL(*out, *in, updateCarryPos(1 << 0)); } @@ -1316,29 +1318,30 @@ u32 Gpu::updateCarryPos(u32 bit) { vector> Gpu::makeBufVector(u32 size) { vector> r; - for (u32 i = 0; i < size; ++i) { r.emplace_back(timeBufVect, &queue, N); } + r.reserve(size); +for (u32 i = 0; i < size; ++i) { r.emplace_back(timeBufVect, &queue, N); } return r; } pair Gpu::readROE() { assert(roePos <= ROE_SIZE); if (roePos) { - vector roe = bufROE.read(roePos); + vector const roe = bufROE.read(roePos); assert(roe.size() == roePos); bufROE.zero(roePos); roePos = 0; auto [squareRoe, mulRoe] = split(roe, mulRoePos); mulRoePos.clear(); return {roeStat(squareRoe), roeStat(mulRoe)}; - } else { + } return {}; - } + } RoeInfo Gpu::readCarryStats() { assert(carryPos <= CARRY_SIZE); if (carryPos == 0) { return {}; } - vector carry = bufStatsCarry.read(carryPos); + vector const carry = bufStatsCarry.read(carryPos); assert(carry.size() == carryPos); bufStatsCarry.zero(carryPos); carryPos = 0; @@ -1384,7 +1387,7 @@ vector Gpu::readChecked(Buffer& buf) { u64 hostSum = 0; for (auto it = data.begin(), end = data.end(); it < end; ++it) hostSum += u64(*it); - u64 gpuSum = expectedVect[0]; + u64 const gpuSum = expectedVect[0]; if (hostSum == gpuSum) { // A buffer containing all-zero is exceptional, so mark that through the empty vector. if (gpuSum == 0 && isAllZero(data)) { @@ -1479,10 +1482,10 @@ void Gpu::logTimeKernels() { string s = "Profile:\n"; for (const TimeInfo* p : prof) { - u32 n = p->n; + u32 const n = p->n; assert(n); - double f = 1e-3 / n; - double percent = 100.0 / total * p->times[2]; + double const f = 1e-3 / n; + double const percent = 100.0 / total * p->times[2]; if (!args.verbose && percent < 0.2) { break; } snprintf(buf, sizeof(buf), args.verbose ? "%s %5.2f%% %-11s : %6.0f us/call x %5d calls (%.3f %.0f)\n" @@ -1512,7 +1515,7 @@ vector Gpu::readWords(Buffer &buf) { void Gpu::writeWords(Buffer& buf, vector &words) { // GPU is expecting either 4-byte or 8-byte integers. C++ code is using 8-byte integers. Handle the "no conversion" case. - if (fft.WordSize == 8) buf.write(std::move(words)); + if (fft.WordSize == 8) buf.write(words); // Convert 64-bit C++ Words into 32-bit GPU Words else { vector GPUdata; @@ -1521,7 +1524,7 @@ void Gpu::writeWords(Buffer& buf, vector &words) { for (u32 i = 0; i < words.size(); i += 2) { GPUdata[i/2] = ((i64) words[i+1] << 32) | (u32) words[i]; } - buf.write(std::move(GPUdata)); + buf.write(GPUdata); } } @@ -1538,19 +1541,19 @@ void Gpu::writeIn(Buffer& buf, vector&& words) { } Words Gpu::expExp2(const Words& A, u32 n) { - u32 logStep = 10000; - u32 blockSize = 100; + u32 const logStep = 10000; + u32 const blockSize = 100; - writeIn(bufData, std::move(A)); + writeIn(bufData, A); IterationTimer timer{0}; u32 k = 0; while (k < n) { - u32 its = std::min(blockSize, n - k); + u32 const its = std::min(blockSize, n - k); squareLoop(bufData, 0, its); k += its; queue.finish(); if (k % logStep == 0) { - float secsPerIt = timer.reset(k); + float const secsPerIt = timer.reset(k); log("%u / %u, %s us/it\n", k, n, formatSecsPerIter(secsPerIt).c_str()); } } @@ -1583,7 +1586,7 @@ void Gpu::exponentiate(Buffer& bufInOut, u64 exp) { fftP(buf1, bufInOut); fftMidIn(buf1); fftHin(buf2, buf1); // save fully FFTed "base" to buf2 - bool midInAlreadyDone = 1; + bool midInAlreadyDone = true; int p = 63; while (!testBit(exp, p)) { --p; } @@ -1592,7 +1595,7 @@ void Gpu::exponentiate(Buffer& bufInOut, u64 exp) { if (!midInAlreadyDone) fftMidIn(buf1); tailSquare(buf1); fftMidOut(buf1); - midInAlreadyDone = 0; + midInAlreadyDone = false; if (testBit(exp, p)) { doCarry(buf1, bufInOut); @@ -1670,7 +1673,7 @@ u32 Gpu::squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool assert(from < to); enum LEAD_TYPE leadIn = LEAD_NONE; for (u64 k = from; k < to; ++k) { - enum LEAD_TYPE leadOut = useLongCarry || (k == to - 1) ? LEAD_NONE : LEAD_WIDTH; + enum LEAD_TYPE const leadOut = useLongCarry || (k == to - 1) ? LEAD_NONE : LEAD_WIDTH; square(out, (k==from) ? in : out, leadIn, leadOut, doTailMul3 && (k == to - 1)); leadIn = leadOut; } @@ -1691,18 +1694,18 @@ u64 Gpu::bufResidue(Buffer &buf) { int carry = 0; for (int i = 0; i < 32; ++i) { - u32 len = bitlen(N, E, N - 32 + i); - i64 w = (i64) words[i] + carry; + u32 const len = bitlen(N, E, N - 32 + i); + i64 const w = (i64) words[i] + carry; carry = (int) (w >> len); } u64 res = 0; int hasBits = 0; for (int k = 0; k < 32 && hasBits < 64; ++k) { - u32 len = bitlen(N, E, k); - i64 tmp = (i64) words[32 + k] + carry; + u32 const len = bitlen(N, E, k); + i64 const tmp = (i64) words[32 + k] + carry; carry = (int) (tmp >> len); - u64 w = tmp - ((i64) carry << len); + u64 const w = tmp - ((i64) carry << len); assert(w < (1ULL << len)); res += w << hasBits; hasBits += len; @@ -1711,10 +1714,10 @@ u64 Gpu::bufResidue(Buffer &buf) { } static string formatETA(u32 secs) { - u32 etaMins = (secs + 30) / 60; - int days = etaMins / (24 * 60); - int hours = etaMins / 60 % 24; - int mins = etaMins % 60; + u32 const etaMins = (secs + 30) / 60; + int const days = etaMins / (24 * 60); + int const hours = etaMins / 60 % 24; + int const mins = etaMins % 60; char buf[64]; if (days) { snprintf(buf, sizeof(buf), "%dd %02d:%02d", days, hours, mins); @@ -1725,7 +1728,7 @@ static string formatETA(u32 secs) { } static string getETA(u32 step, u32 total, float secsPerStep) { - u32 etaSecs = max(0u, u32((total - step) * secsPerStep)); + u32 const etaSecs = max(0u, u32((total - step) * secsPerStep)); return formatETA(etaSecs); } @@ -1749,7 +1752,7 @@ static string makeLogStr(const string& status, u64 k, u64 res, float secsPerIt, void Gpu::doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors) { auto [roeSq, roeMul] = readROE(); - double z = roeSq.z(); + double const z = roeSq.z(); zAvg.update(z, roeSq.N); if (roeSq.max > 0.005) log("%sZ=%.0f (avg %.1f), ROEmax=%.3f, ROEavg=%.3f. %s\n", makeLogStr(checkOK ? "OK" : "EE", k, res, secsPerIt, nIters).c_str(), @@ -1765,10 +1768,10 @@ void Gpu::doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u3 // Unless ROE log is not explicitly requested, measure only a few iterations to minimize overhead wantROE = args.logROE ? ROE_SIZE : 400; - RoeInfo carryStats = readCarryStats(); + RoeInfo const carryStats = readCarryStats(); if (carryStats.N > 2) { - u32 m = ldexp(carryStats.max, 32); - double z = carryStats.z(); + u32 const m = ldexp(carryStats.max, 32); + double const z = carryStats.z(); log("Carry: %x Z(%u)=%.1f\n", m, carryStats.N, z); } } @@ -1779,20 +1782,20 @@ bool Gpu::equals9(const Words& a) { return true; } -int ulps(double a, double b) { +static int ulps(double a, double b) { if (a == 0 && b == 0) { return 0; } - u64 aa = as(a); - u64 bb = as(b); - bool sameSign = (aa >> 63) == (bb >> 63); - int delta = sameSign ? bb - aa : bb + aa; + u64 const aa = as(a); + u64 const bb = as(b); + bool const sameSign = (aa >> 63) == (bb >> 63); + int const delta = sameSign ? bb - aa : bb + aa; return delta; } [[maybe_unused]] static double trigNorm(double c, double s) { - double c2 = c * c; - double err = fma(c, c, -c2); - double norm = c2 + fma(s, s, err); + double const c2 = c * c; + double const err = fma(c, c, -c2); + double const norm = c2 + fma(s, s, err); return norm; } @@ -1837,22 +1840,22 @@ void Gpu::selftestTrig() { if (isAmdGpu(shared.context->deviceId())) { vector WHATS {"V_NOP", "V_ADD_I32", "V_FMA_F32", "V_ADD_F64", "V_FMA_F64", "V_MUL_F64", "V_MAD_U64_U32"}; - for (int w = 0; w < int(WHATS.size()); ++w) { + for (int w = 0; std::cmp_less(w, WHATS.size()); ++w) { const int what = w; testTime(what, bufCarry); - vector times = bufCarry.read(4096 * 2); - [[maybe_unused]] i64 prev = 0; + vector const times = bufCarry.read(4096 * 2); + [[maybe_unused]] i64 const prev = 0; u64 min = -1; u64 sum = 0; - for (int i = 0; i < int(times.size()); ++i) { - i64 x = times[i]; + for (long const x : times) { + #if 0 if (x != prev) { log("%4d : %ld\n", i, x); prev = x; } #endif - if (x > 0 && u64(x) < min) { min = x; } + if (x > 0 && std::cmp_less(x, min)) { min = x; } if (x > 0) { sum += x; } } log("%-15s : %.2f cycles latency; time min: %d; avg %.0f\n", @@ -1864,22 +1867,22 @@ void Gpu::selftestTrig() { static u32 mod3(const std::vector &words) { u32 r = 0; // uses the fact that 2**32 % 3 == 1. - for (u32 w : words) { r += w % 3; } + for (u32 const w : words) { r += w % 3; } return r % 3; } static void doDiv3(u64 E, Words& words) { u32 r = (3 - mod3(words)) % 3; assert(r < 3); - int topBits = E % 32; + int const topBits = E % 32; assert(topBits > 0 && topBits < 32); { - u64 w = (u64(r) << topBits) + words.back(); + u64 const w = (u64(r) << topBits) + words.back(); words.back() = w / 3; r = w % 3; } for (auto it = words.rbegin() + 1, end = words.rend(); it != end; ++it) { - u64 w = (u64(r) << 32) + *it; + u64 const w = (u64(r) << 32) + *it; *it = w / 3; r = w % 3; } @@ -1895,12 +1898,12 @@ fs::path Gpu::saveProof(const Args& args, ProofSet& proofSet) { for ( ; ; ) { for (int retry = 0; retry == 0 || (retry == 1 && !problem_proof); ++retry) { auto [proof, hashes] = proofSet.computeProof(this); - fs::path tmpFile = proof.file(args.proofToVerifyDir); + fs::path const tmpFile = proof.file(args.proofToVerifyDir); proof.save(tmpFile); fs::path proofFile = proof.file(args.proofResultDir); - bool ok = Proof::load(tmpFile).verify(this, hashes); + bool const ok = Proof::load(tmpFile).verify(this, hashes); log("Proof '%s' verification %s\n", tmpFile.string().c_str(), ok ? "OK" : "FAILED"); if (ok) { fancyRename(tmpFile, proofFile); @@ -1923,7 +1926,7 @@ PRPState Gpu::loadPRP(Saver& saver) { PRPState state = saver.load(); writeState(state.k, state.check, state.blockSize); - u64 res = dataResidue(); + u64 const res = dataResidue(); if (res == state.res64) { log("OK %9" PRIu64 " on-load: blockSize %d, %016" PRIx64 "\n", state.k, state.blockSize, res); @@ -1940,7 +1943,7 @@ PRPState Gpu::loadPRP(Saver& saver) { } u32 Gpu::getProofPower(u64 k) { - u32 power = ProofSet::effectivePower(E, args.getProofPow(E), k); + u32 const power = ProofSet::effectivePower(E, args.getProofPow(E), k); if (power != args.getProofPow(E)) { log("Proof using power %u (vs %u)\n", power, args.getProofPow(E)); @@ -1964,10 +1967,10 @@ tuple Gpu::measureCarry() { assert(iters % blockSize == 0); u32 k = 0; - PRPState state{E, 0, blockSize, 3, makeWords(E, 1), 0}; + PRPState const state{.exponent=E, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(E, 1), .nErrors=0}; writeState(state.k, state.check, state.blockSize); { - u64 res = dataResidue(); + u64 const res = dataResidue(); if (res != state.res64) { log("residue expected %016" PRIx64 " found %016" PRIx64 "\n", state.res64, res); } @@ -1978,7 +1981,7 @@ tuple Gpu::measureCarry() { modMul(bufCheck, bufData, leadIn); leadIn = LEAD_MIDDLE; - enum LEAD_TYPE leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; + enum LEAD_TYPE const leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; square(bufData, bufData, leadIn, leadOut); leadIn = leadOut; ++k; @@ -2010,27 +2013,23 @@ tuple Gpu::measureCarry() { if (Signal::stopRequested()) { throw "stop requested"; } } - [[maybe_unused]] u64 res = dataResidue(); + [[maybe_unused]] u64 const res = dataResidue(); if (Signal::stopRequested()) { throw "stop requested"; } - bool ok = doCheck(blockSize); + bool const ok = doCheck(blockSize); auto stats = readCarryStats(); // log("%s %016" PRIx64 " %s\n", ok ? "OK" : "EE", res, roe.toString(statsBits).c_str()); return {ok, stats}; } -tuple Gpu::measureROE(bool quick) { +tuple Gpu::measureROE(bool /*quick*/) { u32 blockSize{}, iters{}, warmup{}; - if (true) { + { blockSize = 200; iters = 2000; warmup = 50; - } else { - blockSize = 500; - iters = 10'000; - warmup = 100; } assert(iters % blockSize == 0); @@ -2038,10 +2037,10 @@ tuple Gpu::measureROE(bool quick) { wantROE = ROE_SIZE; // should be large enough to capture fully this measureROE() u32 k = 0; - PRPState state{E, 0, blockSize, 3, makeWords(E, 1), 0}; + PRPState const state{.exponent=E, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(E, 1), .nErrors=0}; writeState(state.k, state.check, state.blockSize); { - u64 res = dataResidue(); + u64 const res = dataResidue(); if (res != state.res64) { log("residue expected %016" PRIx64 " found %016" PRIx64 "\n", state.res64, res); } @@ -2052,7 +2051,7 @@ tuple Gpu::measureROE(bool quick) { modMul(bufCheck, bufData, leadIn); leadIn = LEAD_MIDDLE; - enum LEAD_TYPE leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; + enum LEAD_TYPE const leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; square(bufData, bufData, leadIn, leadOut); leadIn = leadOut; ++k; @@ -2084,10 +2083,10 @@ tuple Gpu::measureROE(bool quick) { if (Signal::stopRequested()) { throw "stop requested"; } } - [[maybe_unused]] u64 res = dataResidue(); + [[maybe_unused]] u64 const res = dataResidue(); if (Signal::stopRequested()) { throw "stop requested"; } - bool ok = doCheck(blockSize); + bool const ok = doCheck(blockSize); auto roes = readROE(); wantROE = 0; @@ -2113,7 +2112,7 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest assert(iters % blockSize == 0); u32 k = 0; - PRPState state{E, 0, blockSize, 3, makeWords(E, 1), 0}; + PRPState const state{.exponent=E, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(E, 1), .nErrors=0}; writeState(state.k, state.check, state.blockSize); assert(dataResidue() == state.res64); @@ -2121,7 +2120,7 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest modMul(bufCheck, bufData, leadIn); leadIn = LEAD_MIDDLE; - enum LEAD_TYPE leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; + enum LEAD_TYPE const leadOut = useLongCarry ? LEAD_NONE : LEAD_WIDTH; square(bufData, bufData, leadIn, leadOut); leadIn = leadOut; ++k; @@ -2157,8 +2156,8 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest if (Signal::stopRequested()) { throw "stop requested"; } - u64 res = dataResidue(); - bool ok = doCheck(blockSize); + u64 const res = dataResidue(); + bool const ok = doCheck(blockSize); if (!ok) { log("Error %016" PRIx64 "\n", res); secsPerIt = 0.1; // a large value to mark the error @@ -2175,7 +2174,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { u32 nErrors = 0; int nSeqErrors = 0; u64 lastFailedRes64 = 0; - u32 logStep = args.logStep; + u32 const logStep = args.logStep; reload: elapsedTimer.reset(); @@ -2184,7 +2183,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { double elapsedBefore = 0; { - PRPState state = loadPRP(*getSaver()); + PRPState const state = loadPRP(*getSaver()); nErrors = std::max(nErrors, state.nErrors); blockSize = state.blockSize; k = state.k; @@ -2193,10 +2192,10 @@ PRPResult Gpu::isPrimePRP(const Task& task) { assert(blockSize > 0 && logStep % blockSize == 0); - u32 checkStep = checkStepForErrors(blockSize, nErrors); + u32 const checkStep = checkStepForErrors(blockSize, nErrors); assert(checkStep % logStep == 0); - u32 power = getProofPower(k); + u32 const power = getProofPower(k); ProofSet proofSet{E, power}; @@ -2213,7 +2212,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { assert(k < kEnd); // We continue beyound kEnd: to the next multiple of blockSize, to do a check there - u64 kEndEnd = roundUp(kEnd, blockSize); + u64 const kEndEnd = roundUp(kEnd, blockSize); bool skipNextCheckUpdate = false; @@ -2242,10 +2241,10 @@ PRPResult Gpu::isPrimePRP(const Task& task) { ++k; // !! early inc - bool doStop = (k % blockSize == 0) && (Signal::stopRequested() || (args.iters && k - startK >= args.iters)); - bool doCheck = doStop || (k % checkStep == 0) || (k >= kEndEnd) || (k - startK == 2 * blockSize); - bool doLog = k % logStep == 0; - enum LEAD_TYPE leadOut = doCheck || doLog || k == persistK || k == kEnd || useLongCarry ? LEAD_NONE : LEAD_WIDTH; + bool const doStop = (k % blockSize == 0) && (Signal::stopRequested() || (args.iters && k - startK >= args.iters)); + bool const doCheck = doStop || (k % checkStep == 0) || (k >= kEndEnd) || (k - startK == 2 * blockSize); + bool const doLog = k % logStep == 0; + enum LEAD_TYPE const leadOut = doCheck || doLog || k == persistK || k == kEnd || useLongCarry ? LEAD_NONE : LEAD_WIDTH; if (doStop) { log("Stopping, please wait..\n"); } @@ -2253,7 +2252,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { leadIn = leadOut; if (k == persistK) { - vector rawData = readChecked(bufData); + vector const rawData = readChecked(bufData); if (rawData.empty()) { log("Data error ZERO\n"); ++nErrors; @@ -2276,8 +2275,8 @@ PRPResult Gpu::isPrimePRP(const Task& task) { if (!doCheck && !doLog) continue; - u64 res = dataResidue(); - float secsPerIt = iterationTimer.reset(k); + u64 const res = dataResidue(); + float const secsPerIt = iterationTimer.reset(k); queue.setSquareTime((int) (secsPerIt * 1'000'000)); vector rawCheck = readChecked(bufCheck); @@ -2290,20 +2289,20 @@ PRPResult Gpu::isPrimePRP(const Task& task) { if (!doCheck) { (*background)([=, this] { - getSaver()->saveUnverified({E, k, blockSize, res, compactBits(rawCheck, E), nErrors, - elapsedBefore + elapsedTimer.at()}); + getSaver()->saveUnverified({.exponent=E, .k=k, .blockSize=blockSize, .res64=res, .check=compactBits(rawCheck, E), .nErrors=nErrors, + .elapsed=elapsedBefore + elapsedTimer.at()}); }); log(" %9" PRIu64 " %016" PRIx64 " %s\n", k, res, formatSecsPerIter(secsPerIt).c_str()); - RoeInfo carryStats = readCarryStats(); + RoeInfo const carryStats = readCarryStats(); if (carryStats.N) { - u32 m = ldexp(carryStats.max, 32); - double z = carryStats.z(); + u32 const m = ldexp(carryStats.max, 32); + double const z = carryStats.z(); log("Carry: %x Z(%u)=%.1f\n", m, carryStats.N, z); } } else { - bool ok = this->doCheck(blockSize); - [[maybe_unused]] float secsCheck = iterationTimer.reset(k); + bool const ok = this->doCheck(blockSize); + [[maybe_unused]] float const secsCheck = iterationTimer.reset(k); if (ok) { nSeqErrors = 0; @@ -2312,15 +2311,15 @@ PRPResult Gpu::isPrimePRP(const Task& task) { if (k < kEnd) { (*background)([=, this, rawCheck = std::move(rawCheck)] { - getSaver()->save({E, k, blockSize, res, compactBits(rawCheck, E), nErrors, elapsedBefore + elapsedTimer.at()}); + getSaver()->save({.exponent=E, .k=k, .blockSize=blockSize, .res64=res, .check=compactBits(rawCheck, E), .nErrors=nErrors, .elapsed=elapsedBefore + elapsedTimer.at()}); }); } doBigLog(k, res, ok, secsPerIt, kEndEnd, nErrors); if (k >= kEndEnd) { - fs::path proofFile = saveProof(args, proofSet); - return {isPrime, finalRes64, nErrors, proofFile.string(), toHex(res2048)}; + fs::path const proofFile = saveProof(args, proofSet); + return {.isPrime=isPrime, .res64=finalRes64, .nErrors=nErrors, .proofPath=proofFile.string(), .res2048=toHex(res2048)}; } } else { ++nErrors; @@ -2367,9 +2366,9 @@ LLResult Gpu::isPrimeLL(const Task& task) { elapsedBefore = state.elapsed; startK = state.k; - u64 expectedRes = (u64(state.data[1]) << 32) | state.data[0]; - writeIn(bufData, std::move(state.data)); - u64 res = dataResidue(); + u64 const expectedRes = (u64(state.data[1]) << 32) | state.data[0]; + writeIn(bufData, state.data); + u64 const res = dataResidue(); if (res != expectedRes) { throw "Invalid savefile (res64)"; } assert(res == expectedRes); log("LL loaded @ %" PRIu64 " : %016" PRIx64 "\n", startK, res); @@ -2378,7 +2377,7 @@ LLResult Gpu::isPrimeLL(const Task& task) { IterationTimer iterationTimer{startK}; u64 k = startK; - u64 kEnd = E - 2; + u64 const kEnd = E - 2; enum LEAD_TYPE leadIn = LEAD_NONE; while (true) { @@ -2390,8 +2389,8 @@ LLResult Gpu::isPrimeLL(const Task& task) { log("Stopping, please wait..\n"); } - bool doLog = (k % args.logStep == 0) || doStop; - enum LEAD_TYPE leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; + bool const doLog = (k % args.logStep == 0) || doStop; + enum LEAD_TYPE const leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; squareLL(bufData, leadIn, leadOut); leadIn = leadOut; @@ -2400,29 +2399,29 @@ LLResult Gpu::isPrimeLL(const Task& task) { u64 res64 = 0; auto data = readData(); - bool isAllZero = data.empty(); + bool const isAllZero = data.empty(); if (isAllZero) { if (k < kEnd) { log("Error: early ZERO @ %" PRIu64 "\n", k); if (doStop) { throw "stop requested"; - } else { + } goto reload; - } + } res64 = 0; } else { assert(data.size() >= 2); res64 = (u64(data[1]) << 32) | data[0]; - saver.save({E, k, std::move(data), elapsedBefore + elapsedTimer.at()}); + saver.save({.exponent=E, .k=k, .data=std::move(data), .elapsed=elapsedBefore + elapsedTimer.at()}); } - float secsPerIt = iterationTimer.reset(k); + float const secsPerIt = iterationTimer.reset(k); queue.setSquareTime((int) (secsPerIt * 1'000'000)); log("%9" PRIu64 " %016" PRIx64 " %s ETA %s\n", k, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); - if (k >= kEnd) { return {isAllZero, res64}; } + if (k >= kEnd) { return {.isPrime=isAllZero, .res64=res64}; } if (doStop) { throw "stop requested"; } } @@ -2440,21 +2439,21 @@ array Gpu::isCERT(const Task& task) { { // Enclosing this code in braces ensures the file will be closed by the File destructor. The later file deletion requires the file be closed in Windows. File fi = File::openReadThrow(fname); - u32 nBytes = u32((E - 1) / 8 + 1); - Words B = fi.readBytesLE(nBytes); - writeIn(bufData, std::move(B)); + u32 const nBytes = u32((E - 1) / 8 + 1); + Words const B = fi.readBytesLE(nBytes); + writeIn(bufData, B); } Timer elapsedTimer; elapsedTimer.reset(); - u32 startK = 0; + u32 const startK = 0; IterationTimer iterationTimer{startK}; u32 k = 0; - u32 kEnd = task.squarings; + u32 const kEnd = task.squarings; enum LEAD_TYPE leadIn = LEAD_NONE; while (true) { @@ -2466,8 +2465,8 @@ array Gpu::isCERT(const Task& task) { log("Stopping, please wait..\n"); } - bool doLog = (k % 100'000 == 0) || doStop; - enum LEAD_TYPE leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; + bool const doLog = (k % 100'000 == 0) || doStop; + enum LEAD_TYPE const leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; squareCERT(bufData, leadIn, leadOut); leadIn = leadOut; @@ -2476,9 +2475,9 @@ array Gpu::isCERT(const Task& task) { Words data = readData(); assert(data.size() >= 2); - u64 res64 = (u64(data[1]) << 32) | data[0]; + u64 const res64 = (u64(data[1]) << 32) | data[0]; - float secsPerIt = iterationTimer.reset(k); + float const secsPerIt = iterationTimer.reset(k); queue.setSquareTime((int) (secsPerIt * 1'000'000)); log("%7u / %7u %016" PRIx64 " %s ETA %s\n", k, kEnd, res64, formatSecsPerIter(secsPerIt).c_str(), getETA(k, kEnd, secsPerIt).c_str()); diff --git a/src/Gpu.h b/src/Gpu.h index e7a2b810..7e91c1bd 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -15,6 +15,7 @@ #include "GpuCommon.h" #include "FFTConfig.h" +#include #include #include #include @@ -35,7 +36,7 @@ struct PRPResult { bool isPrime{}; u64 res64 = 0; u32 nErrors = 0; - fs::path proofPath{}; + fs::path proofPath; std::string res2048; }; @@ -62,15 +63,15 @@ class RoeInfo { RoeInfo(u32 n, double max, double mean, double sd) : N{n}, max{max}, mean{mean}, sd{sd} { // https://en.wikipedia.org/wiki/Gumbel_distribution gumbelBeta = sd * 0.779696801233676; // sqrt(6)/pi - gumbelMiu = mean - gumbelBeta * 0.577215664901533; // Euler-Mascheroni + gumbelMiu = mean - gumbelBeta * std::numbers::egamma; // Euler-Mascheroni } - double z(double x = 0.5) const { return N ? (x - gumbelMiu) / gumbelBeta : 0.0; } + [[nodiscard]] double z(double x = 0.5) const { return N ? (x - gumbelMiu) / gumbelBeta : 0.0; } - double gumbelCDF(double x) const { return exp(-exp(-z(x))); } - double gumbelRightCDF(double x) const { return -expm1(-exp(-z(x))); } + [[nodiscard]] double gumbelCDF(double x) const { return exp(-exp(-z(x))); } + [[nodiscard]] double gumbelRightCDF(double x) const { return -expm1(-exp(-z(x))); } - std::string toString() const; + [[nodiscard]] std::string toString() const; u32 N{}; double max{}, mean{}, sd{}; @@ -224,9 +225,9 @@ class Gpu { vector *> recorded_kernel_args; const int NUM_CACHE_GROUPS = 3; - void splitQueue(void); - void mergeQueue(void); - void replay(void); + void splitQueue(); + void mergeQueue(); + void replay(); void fftP(Buffer& out, Buffer& in) { fftP(out, reinterpret_cast&>(in)); } void fftP(Buffer& out, Buffer& in); @@ -354,11 +355,11 @@ class Gpu { // Data buffers require extra space for padding. We can probably tighten up the amount of extra memory allocated. // The worst case seems to be !INPLACE, MIDDLE=4, PAD_SIZE=512. -#define MID_ADJUST(size,M,pad) ((pad == 0 || M != 4) ? (size) : (size) * 5/4) -#define PAD_ADJUST(N,M,inplace,pad) (inplace ? 3*N/2 : MID_ADJUST(pad == 0 ? N : pad <= 128 ? 9*N/8 : pad <= 256 ? 5*N/4 : 3*N/2, M, pad)) -#define FP64_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) -#define FP32_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) * sizeof(float) / sizeof(double) -#define GF31_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) * sizeof(uint) / sizeof(double) -#define GF61_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST(W*M*H*2, M, inplace, pad) * sizeof(ulong) / sizeof(double) -#define TOTAL_DATA_SIZE(fft,W,M,H,inplace,pad) (int)fft.FFT_FP64 * FP64_DATA_SIZE(W,M,H,inplace,pad) + (int)fft.FFT_FP32 * FP32_DATA_SIZE(W,M,H,inplace,pad) + \ - (int)fft.NTT_GF31 * GF31_DATA_SIZE(W,M,H,inplace,pad) + (int)fft.NTT_GF61 * GF61_DATA_SIZE(W,M,H,inplace,pad) +#define MID_ADJUST(size,M,pad) (((pad) == 0 || (M) != 4) ? (size) : (size) * 5/4) +#define PAD_ADJUST(N,M,inplace,pad) ((inplace) ? 3*(N)/2 : MID_ADJUST((pad) == 0 ? (N) : (pad) <= 128 ? 9*(N)/8 : (pad) <= 256 ? 5*(N)/4 : 3*(N)/2, M, pad)) +#define FP64_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) +#define FP32_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) * sizeof(float) / sizeof(double) +#define GF31_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) * sizeof(uint) / sizeof(double) +#define GF61_DATA_SIZE(W,M,H,inplace,pad) PAD_ADJUST((W)*(M)*(H)*2, M, inplace, pad) * sizeof(ulong) / sizeof(double) +#define TOTAL_DATA_SIZE(fft,W,M,H,inplace,pad) ((int)(fft).FFT_FP64 * FP64_DATA_SIZE(W,M,H,inplace,pad) + (int)(fft).FFT_FP32 * FP32_DATA_SIZE(W,M,H,inplace,pad) + \ + (int)(fft).NTT_GF31 * GF31_DATA_SIZE(W,M,H,inplace,pad) + (int)(fft).NTT_GF61 * GF61_DATA_SIZE(W,M,H,inplace,pad)) diff --git a/src/Hash.h b/src/Hash.h index ddeb9c2e..5144db17 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -11,7 +11,7 @@ class Hash { public: template - static auto hash(Ts... data) { + static auto hash(const Ts&... data) { Hash hash; (hash.update(data),...); return std::move(hash).finish(); diff --git a/src/Kernel.cpp b/src/Kernel.cpp index 5da47088..5807534d 100644 --- a/src/Kernel.cpp +++ b/src/Kernel.cpp @@ -3,7 +3,6 @@ #include "Kernel.h" #include "KernelCompiler.h" -#include Kernel::Kernel(string_view name, KernelCompiler* compiler, TimeInfo* timeInfo, Queue* queue, string_view fileName, string_view nameInFile, diff --git a/src/Kernel.h b/src/Kernel.h index 67a7bf50..557dd7e6 100644 --- a/src/Kernel.h +++ b/src/Kernel.h @@ -27,7 +27,7 @@ class Kernel { size_t workSize; u32 groupSize = 0; - KernelHolder kernel{}; + KernelHolder kernel; std::future pendingKernel; cl_device_id deviceId; std::vector> pendingArgs; @@ -67,7 +67,7 @@ class Kernel { if (kernel) { ::setArg(kernel.get(), pos, arg, name); } else { - pendingArgs.push_back({pos, arg}); + pendingArgs.emplace_back(pos, arg); } } diff --git a/src/KernelCompiler.cpp b/src/KernelCompiler.cpp index 59165081..a5885227 100644 --- a/src/KernelCompiler.cpp +++ b/src/KernelCompiler.cpp @@ -8,6 +8,7 @@ #include #include #include +#include using namespace std; @@ -34,7 +35,7 @@ KernelCompiler::KernelCompiler(const Args& args, const Context* context, const s deviceId{context->deviceId()} { - string hw = getDriverVersion(deviceId) + ':' + getDeviceName(deviceId); + string const hw = getDriverVersion(deviceId) + ':' + getDeviceName(deviceId); if (args.verbose) { log("OpenCL: %s, args %s\n", hw.c_str(), baseArgs.c_str()); } SHA3 hasher; @@ -44,10 +45,10 @@ KernelCompiler::KernelCompiler(const Args& args, const Context* context, const s auto& clNames = getClFileNames(); auto& clFiles = getClFiles(); assert(clNames.size() == clFiles.size()); - int n = clNames.size(); + int const n = clNames.size(); for (int i = 0; i < n; ++i) { auto &src = clFiles[i]; - files.push_back({clNames[i], src}); + files.emplace_back(clNames[i], src); clSources.push_back(loadSource(context->get(), src)); hasher.update(clNames[i]); @@ -75,7 +76,7 @@ Program KernelCompiler::compile(const string& fileName, const string& extraArgs) clSources.size()-1, (const cl_program*) (clSources.data()+1), getClFileNames().data()+1, nullptr, nullptr); #endif - if (string mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } + if (string const mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } if (err != CL_SUCCESS) { log("Compiling '%s' error %s (args %s)\n", fileName.c_str(), errMes(err).c_str(), args.c_str()); return {}; @@ -83,7 +84,7 @@ Program KernelCompiler::compile(const string& fileName, const string& extraArgs) Program p2{clLinkProgram(context, 1, &deviceId, linkArgs.c_str(), 1, (cl_program *) &p1, nullptr, nullptr, &err)}; - if (string mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } + if (string const mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } if (err != CL_SUCCESS) { log("Linking '%s' error %s (args %s)\n", fileName.c_str(), errMes(err).c_str(), linkArgs.c_str()); } @@ -97,14 +98,14 @@ static string to_hex(u64 d) { } KernelHolder KernelCompiler::loadAux(const string& fileName, const string& kernelName, const string& args) const { - Timer timer; + Timer const timer; bool fromCache = true; Program program; string cacheFile; if (useCache) { - string f = kernelName + '-' + to_hex(SHA3::hash(contextHash, fileName, kernelName, args)[0]); + string const f = kernelName + '-' + to_hex(SHA3::hash(contextHash, fileName, kernelName, args)[0]); cacheFile = cacheDir + '/' + f; program = loadBinary(context, deviceId, cacheFile); } diff --git a/src/KernelCompiler.h b/src/KernelCompiler.h index 0d07a7ee..19cd9d60 100644 --- a/src/KernelCompiler.h +++ b/src/KernelCompiler.h @@ -25,13 +25,13 @@ class KernelCompiler { u64 contextHash{}; - Program compile(const string& fileName, const string& args) const; - KernelHolder loadAux(const string& fileName, const string& kernelName, const string& args) const; + [[nodiscard]] Program compile(const string& fileName, const string& args) const; + [[nodiscard]] KernelHolder loadAux(const string& fileName, const string& kernelName, const string& args) const; public: const cl_device_id deviceId; KernelCompiler(const Args& args, const Context* context, const string& clArgs); - std::future load(const string& fileName, const string& kernelName, const string& args) const; + [[nodiscard]] std::future load(const string& fileName, const string& kernelName, const string& args) const; }; diff --git a/src/MD5.h b/src/MD5.h index 433d1b64..c0b19437 100644 --- a/src/MD5.h +++ b/src/MD5.h @@ -22,10 +22,10 @@ class MD5Hash { unsigned char digest[16]; MD5Final(digest, &context); string s; - char hex[] = "0123456789abcdef"; - for (int i = 0; i < 16; ++i) { - s.push_back(hex[digest[i] >> 4]); - s.push_back(hex[digest[i] & 0xf]); + char const hex[] = "0123456789abcdef"; + for (unsigned char const i : digest) { + s.push_back(hex[i >> 4]); + s.push_back(hex[i & 0xf]); } return s; } diff --git a/src/Primes.cpp b/src/Primes.cpp index 6e3ceb36..665ee251 100644 --- a/src/Primes.cpp +++ b/src/Primes.cpp @@ -9,7 +9,7 @@ Primes::Primes() { for (u32 i = 0; i < sieve.size(); ++i) { if (sieve[i]) { - u32 n = 2 * i + 3; + u32 const n = 2 * i + 3; for (u32 k = i + n; k < sieve.size(); k += n) { sieve.reset(k); } } } @@ -21,7 +21,7 @@ bool Primes::isPrimeOdd(u64 n) const { if (n < 3) { return false; } for (u32 k = 0; k < sieve.size(); ++k) { if (sieve[k]) { - u32 p = k * 2 + 3; + u32 const p = k * 2 + 3; if (u64(p) * u64(p) > n) { return true; } if (n % p == 0) { return false; } } @@ -53,8 +53,8 @@ u64 Primes::nextPrime(u64 n) const { u64 Primes::nearestPrime(u64 n) const { if (isPrime(n)) { return n; } - u64 a = prevPrime(n); - u64 b = nextPrime(n); + u64 const a = prevPrime(n); + u64 const b = nextPrime(n); assert(a < n && n < b); return n-a < b-n ? a : b; } diff --git a/src/Primes.h b/src/Primes.h index 7f1f16fb..b5dbd4e4 100644 --- a/src/Primes.h +++ b/src/Primes.h @@ -7,13 +7,13 @@ class Primes { std::bitset<50000> sieve; // Allows for testing prims up to 10 billion - bool isPrimeOdd(u64 n) const; + [[nodiscard]] bool isPrimeOdd(u64 n) const; public: Primes(); - bool isPrime(u64 n) const; - u64 prevPrime(u64 n) const; - u64 nextPrime(u64 n) const; - u64 nearestPrime(u64 n) const; + [[nodiscard]] bool isPrime(u64 n) const; + [[nodiscard]] u64 prevPrime(u64 n) const; + [[nodiscard]] u64 nextPrime(u64 n) const; + [[nodiscard]] u64 nearestPrime(u64 n) const; }; diff --git a/src/Profile.cpp b/src/Profile.cpp index dcb543d2..4437abde 100644 --- a/src/Profile.cpp +++ b/src/Profile.cpp @@ -13,7 +13,7 @@ TimeInfo* Profile::make(string_view s) { vector Profile::get() const { vector ret; for (auto& t : entries) { if (t->n) { ret.push_back(t.get()); } } - std::sort(ret.begin(), ret.end(), [](auto p1, auto p2) { return p1->times[2] > p2->times[2]; }); + std::ranges::sort(ret, [](auto p1, auto p2) { return p1->times[2] > p2->times[2]; }); return ret; } diff --git a/src/Profile.h b/src/Profile.h index 3c919142..46040201 100644 --- a/src/Profile.h +++ b/src/Profile.h @@ -16,7 +16,7 @@ class Profile { public: TimeInfo *make(std::string_view s); - std::vector get() const; + [[nodiscard]] std::vector get() const; void reset(); }; diff --git a/src/Proof.cpp b/src/Proof.cpp index bd8d4be6..1d395954 100644 --- a/src/Proof.cpp +++ b/src/Proof.cpp @@ -6,6 +6,7 @@ #include "Gpu.h" #include +#include #include #include #include @@ -37,7 +38,7 @@ string fileHash(const fs::path& filePath) { } ProofInfo getInfo(const fs::path& proofFile) { - string hash = proof::fileHash(proofFile); + string const hash = proof::fileHash(proofFile); File fi = File::openReadThrow(proofFile); u64 E = 0; u32 power = 0; @@ -46,7 +47,7 @@ ProofInfo getInfo(const fs::path& proofFile) { log("Proof file '%s' has invalid header\n", proofFile.string().c_str()); throw "Invalid proof header"; } - return {power, E, hash}; + return {.power=power, .exp=E, .md5=hash}; } } @@ -54,14 +55,14 @@ ProofInfo getInfo(const fs::path& proofFile) { // ---- Proof ---- fs::path Proof::file(const fs::path& proofDir) const { - string strE = to_string(E); - u32 power = middles.size(); + string const strE = to_string(E); + u32 const power = middles.size(); return proofDir / (strE + '-' + to_string(power) + ".proof"); } void Proof::save(const fs::path& proofFile) const { - File fo = File::openWrite(proofFile); - u32 power = middles.size(); + File const fo = File::openWrite(proofFile); + u32 const power = middles.size(); fo.printf(HEADER_v2, power, E, '\n'); fo.write(B.data(), (E-1)/8+1); for (const Words& w : middles) { fo.write(w.data(), (E-1)/8+1); } @@ -76,21 +77,22 @@ Proof Proof::load(const fs::path& path) { log("Proof file '%s' has invalid header\n", path.string().c_str()); throw "Invalid proof header"; } - u32 nBytes = (E - 1) / 8 + 1; - Words B = fi.readBytesLE(nBytes); + u32 const nBytes = (E - 1) / 8 + 1; + Words const B = fi.readBytesLE(nBytes); vector middles; - for (u32 i = 0; i < power; ++i) { middles.push_back(fi.readBytesLE(nBytes)); } - return {E, B, middles}; + middles.reserve(power); +for (u32 i = 0; i < power; ++i) { middles.push_back(fi.readBytesLE(nBytes)); } + return {.E=E, .B=B, .middles=middles}; } bool Proof::verify(Gpu *gpu, const vector& hashes) const { // log("B %016" PRIx64 "\n", res64(B)); // for (u32 i = 0; i < middles.size(); ++i) { log("Middle[%u] %016" PRIx64 "\n", i, res64(middles[i])); } - u32 power = middles.size(); + u32 const power = middles.size(); assert(power > 0); - bool isPrime = (B == makeWords(E, 9)); + bool const isPrime = (B == makeWords(E, 9)); Words A{makeWords(E, 3)}; Words B{this->B}; @@ -101,14 +103,14 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { for (u32 i = 0; i < power; ++i, span = (span + 1) / 2) { const Words& M = middles[i]; hash = proof::hashWords(E, hash, M); - u64 h = hash[0]; + u64 const h = hash[0]; if (hashes.size() > i && h != hashes.at(i)) { log("proof [%u] : hash expected %016" PRIx64 " != %016" PRIx64 "\n", i, hashes[i], h); return false; } - bool doSquareB = span % 2; + bool const doSquareB = span % 2; B = gpu->expMul(M, h, B, doSquareB); A = gpu->expMul(A, h, M, false); @@ -118,7 +120,7 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { log("proof verification: doing %" PRIu64 " iterations\n", span); A = gpu->expExp2(A, span); - bool ok = (A == B); + bool const ok = (A == B); if (ok) { log("proof: %" PRIu64 " proved %s\n", E, isPrime ? "probable prime" : "composite"); } else { @@ -156,7 +158,7 @@ ProofSet::ProofSet(u64 E, u32 power) assert(points.front() == 0); points.front() = E; - std::sort(points.begin(), points.end()); + std::ranges::sort(points); assert(points.size() == (1u << power)); assert(points.back() == E); @@ -164,7 +166,7 @@ ProofSet::ProofSet(u64 E, u32 power) points.push_back(u32(-1)); // guard element cacheIt = points.begin(); - for ([[maybe_unused]] u64 p : points) { + for ([[maybe_unused]] u64 const p : points) { assert(p > E || isInPoints(E, power, p)); } } @@ -198,7 +200,7 @@ u32 ProofSet::bestPower(u64 E) { assert(E > 0); // log2(x)/2 is log4(x) - int power = 10 + floor(log2(E / 60e6) / 2); + int const power = 10 + floor(log2(E / 60e6) / 2); assert(power >= 2); return power; } @@ -224,7 +226,7 @@ bool ProofSet::fileExists(u64 k) const { } bool ProofSet::isValidTo(u64 limitK) const { - auto it = upper_bound(points.begin(), points.end(), limitK); + auto it = std::ranges::upper_bound(points, limitK); if (it == points.begin()) { return true; @@ -246,7 +248,7 @@ bool ProofSet::isValidTo(u64 limitK) const { u64 ProofSet::next(u64 k) const { if (*cacheIt <= k || (cacheIt > points.begin() && *prev(cacheIt) > k)) { - cacheIt = upper_bound(points.begin(), points.end(), k); + cacheIt = std::ranges::upper_bound(points, k); } return *cacheIt; } @@ -267,7 +269,7 @@ Words ProofSet::load(u64 E, u32 power, u64 k) { std::pair> ProofSet::computeProof(Gpu *gpu) const { Words B = load(E); - Words A = makeWords(E, 3); + Words const A = makeWords(E, 3); vector middles; vector hashes; @@ -279,19 +281,19 @@ std::pair> ProofSet::computeProof(Gpu *gpu) const { for (u32 p = 0; p < power; ++p) { auto bufIt = bufVect.begin(); assert(p == hashes.size()); - u32 s = (1u << (power - p - 1)); + u32 const s = (1u << (power - p - 1)); for (u32 i = 0; i < (1u << p); ++i) { - Words w = load(points[s * (i * 2 + 1) - 1]); + Words const w = load(points[s * (i * 2 + 1) - 1]); gpu->writeIn(*bufIt++, w); for (u32 k = 0; i & (1u << k); ++k) { assert(k <= p - 1); --bufIt; - u64 h = hashes[p - 1 - k]; + u64 const h = hashes[p - 1 - k]; gpu->expMul(*(bufIt - 1), h, *bufIt); } } assert(bufIt == bufVect.begin() + 1); - Words w = gpu->readAndCompress(bufVect.front()); + Words const w = gpu->readAndCompress(bufVect.front()); if (w.empty()) { throw "Read ZERO during proof generation"; } middles.push_back(w); hash = proof::hashWords(E, hash, middles.back()); @@ -299,5 +301,5 @@ std::pair> ProofSet::computeProof(Gpu *gpu) const { log("proof [%u] : M %016" PRIx64 ", h %016" PRIx64 "\n", p, res64(middles.back()), hashes.back()); } - return {Proof{E, std::move(B), std::move(middles)}, hashes}; + return {Proof{.E=E, .B=std::move(B), .middles=std::move(middles)}, hashes}; } diff --git a/src/Proof.h b/src/Proof.h index e2df024c..bac5d295 100644 --- a/src/Proof.h +++ b/src/Proof.h @@ -47,7 +47,7 @@ class Proof { void save(const fs::path& proofResultDir) const; - fs::path file(const fs::path& proofDir) const; + [[nodiscard]] fs::path file(const fs::path& proofDir) const; bool verify(Gpu *gpu, const vector& hashes = {}) const; }; @@ -64,7 +64,7 @@ class ProofSet { static bool canDo(u64 E, u32 power, u64 currentK); - mutable decltype(points)::const_iterator cacheIt{}; + mutable decltype(points)::const_iterator cacheIt; bool fileExists(u64 k) const; @@ -86,6 +86,6 @@ class ProofSet { void save(u64 k, const Words& words) const { return save(E, power, k, words); } Words load(u64 k) const { return load(E, power, k); } - void reducePower(void) { power--; } + void reducePower() { power--; } std::pair> computeProof(Gpu *gpu) const; }; diff --git a/src/Queue.cpp b/src/Queue.cpp index 0b2d0baf..b13433a7 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -1,14 +1,13 @@ // Copyright (C) Mihai Preda #include "Queue.h" -#include "Args.h" #include "TimeInfo.h" -#include "timeutil.h" #include "log.h" #include #include #include +#include void Events::clearCompleted() { while (!empty() && front().isComplete()) { pop_front(); } } @@ -21,13 +20,8 @@ Queue::Queue(const Context& context, bool profile, bool auxQueue) : QueueHolder{makeQueue(context.deviceId(), context.get(), profile)}, hasEvents{profile}, isAuxQueue(auxQueue), - context{&context}, - markerEvent{}, - markerQueued(false), - queueCount(0), - squareTime(50), - squareKernels(4), - firstSetTime(true) + context{&context} + { // Formerly a constant (thus the CAPS). nVidia is 3% CPU load at 400 or 500, and 35% load at 800 on my Linux machine. // AMD is just over 2% load at 1600 and 3200 on the same Linux machine. Marginally better timings(?) at 3200. @@ -43,7 +37,7 @@ void Queue::fillBufTE(cl_mem buf, u32 patSize, const void* pattern, u64 size, Ti add(::fillBuf(get(), {}, buf, pattern, patSize, size, hasEvents), tInfo); } -string status(Events& events) { +static string status(Events& events) { if (events.empty()) { return ""; } Event& f = events.front(); return f.isComplete() ? "C" : f.isQueued() ? "Q" : f.isRunning() ? "R" : f.isSubmitted() ? "S" : "?"; @@ -121,7 +115,7 @@ void Queue::setSquareTime(int time) { firstSetTime = false; return; } - if (time < 30) time = 30; // Assume a minimum square time of 30us - if (time > 3000) time = 3000; // Assume a maximum square time of 3000us + time = std::max(time, 30); // Assume a minimum square time of 30us + time = std::min(time, 3000); // Assume a maximum square time of 3000us squareTime = time; } diff --git a/src/Queue.h b/src/Queue.h index 3e0b3ab0..6c5f3b79 100644 --- a/src/Queue.h +++ b/src/Queue.h @@ -51,7 +51,7 @@ class Queue : public QueueHolder { void copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo); void finish(); - EventHolder createSyncEvent(void) { return enqueueMarker(get()); } // Enqueue a synchronization event. Used to sync work among multiple queues. + EventHolder createSyncEvent() { return enqueueMarker(get()); } // Enqueue a synchronization event. Used to sync work among multiple queues. void waitForSyncEvent(EventHolder* e) { enqueueMarkerWithWaits(get(), {e->get()}); } // Wait for a synchronization event to complete. void setSquareTime(int); // Update the time to do one squaring (in microseconds) @@ -60,11 +60,11 @@ class Queue : public QueueHolder { private: // This replaces the "call queue->finish every 400 squarings" code in Gpu.cpp. Solves the busy wait on nVidia GPUs. int MAX_QUEUE_COUNT; // Queue size before a marker will be enqueued. Typically, 100 to 1000 squarings. EventHolder markerEvent; // Event associated with an enqueued marker placed in the queue every MAX_QUEUE_COUNT entries and before r/w operations. - bool markerQueued; // TRUE if a marker and event have been queued - int queueCount; // Count of items added to the queue since last marker - int squareTime; // Time to do one squaring (in microseconds) - int squareKernels; // Number of kernels in one squaring - bool firstSetTime; // Flag so we can ignore first setSquareTime call (which is inaccurate because of all the initial openCL compiles) + bool markerQueued{false}; // TRUE if a marker and event have been queued + int queueCount{0}; // Count of items added to the queue since last marker + int squareTime{50}; // Time to do one squaring (in microseconds) + int squareKernels{4}; // Number of kernels in one squaring + bool firstSetTime{true}; // Flag so we can ignore first setSquareTime call (which is inaccurate because of all the initial openCL compiles) void queueMarkerEvent(); // Queue the marker event void waitForMarkerEvent(); // Wait for marker event to complete }; diff --git a/src/Saver.cpp b/src/Saver.cpp index 68caba41..c9663fb5 100644 --- a/src/Saver.cpp +++ b/src/Saver.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace { @@ -32,14 +33,14 @@ static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 struct BadHeaderError { string name; }; bool startsWith(const string& s, const string& prefix) { - return s.rfind(prefix, 0) == 0; + return s.starts_with(prefix); } -vector savefiles(fs::path dir, const string& prefix, const string& kind) { +vector savefiles(const fs::path& dir, const string& prefix, const string& kind) { vector v; for (const auto& entry: fs::directory_iterator(dir)) { if (entry.is_regular_file()) { - string filename = entry.path().filename().string(); + string const filename = entry.path().filename().string(); auto dot = filename.find('.'); if (dot != string::npos && startsWith(filename, prefix) && filename.substr(dot + 1) == kind) { assert(dot > prefix.size()); @@ -57,7 +58,7 @@ vector savefiles(fs::path dir, const string& prefix, const string& kind) { } } } - std::sort(v.begin(), v.end()); + std::ranges::sort(v); return v; } @@ -67,21 +68,21 @@ string str9(u64 k) { return buf; } -fs::path pathFor(fs::path base, const string& prefix, const string& kind, u64 k) { +fs::path pathFor(const fs::path& base, const string& prefix, const string& kind, u64 k) { return base / (prefix + str9(k) + '.' + kind); } -fs::path pathUnverified(fs::path base, const string& prefix) { +fs::path pathUnverified(const fs::path& base, const string& prefix) { return base / (prefix + "unverified.prp"); } // find the "most advanced" file in dir with a name of the form // . // e.g.: 125784077-010000000.prp -fs::path findLast(fs::path dir, const string& prefix, const string& kind) { +fs::path findLast(const fs::path& dir, const string& prefix, const string& kind) { vector v = savefiles(dir, prefix, kind); if (v.empty()) { return {}; } - u64 lastK = v.back(); + u64 const lastK = v.back(); fs::path path = pathFor(dir, prefix, kind, lastK); assert(is_regular_file(path)); return path; @@ -93,15 +94,15 @@ PRPState readState(const PRPState& dummy, File fi) { u64 res64{}; double elapsed{}; - string header = fi.readLine(); + string const header = fi.readLine(); if (sscanf(header.c_str(), PRP_v13, &exponent, &k, &blockSize, &res64, &nErrors, &elapsed) == 6) { - return {exponent, k, blockSize, res64, fi.readChecked(nWords(exponent)), nErrors, elapsed}; + return {.exponent=exponent, .k=k, .blockSize=blockSize, .res64=res64, .check=fi.readChecked(nWords(exponent)), .nErrors=nErrors, .elapsed=elapsed}; } u32 crc{}; if (sscanf(header.c_str(), PRP_v12, &exponent, &k, &blockSize, &res64, &nErrors, &crc) == 6) { - return {exponent, k, blockSize, res64, fi.readWithCRC(nWords(exponent), crc), nErrors, 0}; + return {.exponent=exponent, .k=k, .blockSize=blockSize, .res64=res64, .check=fi.readWithCRC(nWords(exponent), crc), .nErrors=nErrors, .elapsed=0}; } log("Loading PRP from '%s': bad header '%s'\n", fi.name.c_str(), header.c_str()); @@ -112,15 +113,15 @@ LLState readState(const LLState& dummy, File fi) { u64 exponent{}, k{}; double elapsed{}; - string header = fi.readLine(); + string const header = fi.readLine(); if (sscanf(header.c_str(), LL_v13, &exponent, &k, &elapsed) == 3) { - return {exponent, k, fi.readChecked(nWords(exponent)), elapsed}; + return {.exponent=exponent, .k=k, .data=fi.readChecked(nWords(exponent)), .elapsed=elapsed}; } u32 crc{}; if (sscanf(header.c_str(), LL_v1, &exponent, &k, &crc) == 3) { - return {exponent, k, fi.readWithCRC(nWords(exponent), crc), 0}; + return {.exponent=exponent, .k=k, .data=fi.readWithCRC(nWords(exponent), crc), .elapsed=0}; } log("Loading LL from '%s': bad header '%s'\n", fi.name.c_str(), header.c_str()); @@ -159,11 +160,11 @@ double roundNumberScore(u64 x) { } // namespace template<> PRPState Saver::initState() { - return {exponent, 0, blockSize, 3, makeWords(exponent, 1), 0, 0}; + return {.exponent=exponent, .k=0, .blockSize=blockSize, .res64=3, .check=makeWords(exponent, 1), .nErrors=0, .elapsed=0}; } template<> LLState Saver::initState() { - return {exponent, 0, makeWords(exponent, 4), 0}; + return {.exponent=exponent, .k=0, .data=makeWords(exponent, 4), .elapsed=0}; } @@ -191,14 +192,14 @@ Saver::~Saver() = default; template void Saver::clear(u64 exponent) { error_code dummy; - fs::path base = std::is_same_v ? + fs::path const base = std::is_same_v ? fs::current_path() / to_string(exponent) : fs::current_path() / (string(State::KIND) + '-' + to_string(exponent)); fs::remove_all(base, dummy); } template -void Saver::moveToTrash(fs::path src) { +void Saver::moveToTrash(const fs::path& src) { log("Removing bad savefile '%s'\n", src.string().c_str()); fancyRename(src, src + ".bad"s); } @@ -216,7 +217,7 @@ fs::path Saver::mostRecentSavefile() { template State Saver::load() { for (int i = 0; i < 2; ++i) { - fs::path path = mostRecentSavefile(); + fs::path const path = mostRecentSavefile(); if (path.empty()) { // no savefiles at all @@ -254,9 +255,9 @@ void Saver::trimFiles() { u64 prevK = 0; for (u32 i = 0; i < v.size() - 1; ++i) { - u64 k = v[i]; - double niceBias = std::min(1.0, roundNumberScore(k) - 4); - double span = (v[i + 1] - prevK) * niceBias; + u64 const k = v[i]; + double const niceBias = std::min(1.0, roundNumberScore(k) - 4); + double const span = (v[i + 1] - prevK) * niceBias; prevK = k; if (span < bestSpan) { bestSpan = span; @@ -264,9 +265,9 @@ void Saver::trimFiles() { } } assert(bestIdx >= 0); - u64 k = v[bestIdx]; + u64 const k = v[bestIdx]; // log("Deleting savefile %" PRIu64 "\n", k); - fs::path path = pathFor(base, prefix, State::KIND, k); + fs::path const path = pathFor(base, prefix, State::KIND, k); fs::remove(path); v.erase(v.begin() + bestIdx); } @@ -274,7 +275,7 @@ void Saver::trimFiles() { template void Saver::save(const State& state) { - fs::path path = pathFor(base, to_string(exponent) + '-', State::KIND, state.k); + fs::path const path = pathFor(base, to_string(exponent) + '-', State::KIND, state.k); ::writeState(*CycleFile{path}, state); trimFiles(); // log("rm '%s'\n", pathUnverified(base, prefix).string().c_str()); @@ -288,7 +289,7 @@ void Saver::saveUnverified(const PRPState& state) const { template void Saver::dropMostRecent() { - fs::path path = mostRecentSavefile(); + fs::path const path = mostRecentSavefile(); assert(!path.empty()); if (!path.empty()) { moveToTrash(path); } } diff --git a/src/Saver.h b/src/Saver.h index 5a3a7ba6..671b7025 100644 --- a/src/Saver.h +++ b/src/Saver.h @@ -40,7 +40,7 @@ class Saver { u32 nSavefiles; State initState(); - void moveToTrash(fs::path file); + void moveToTrash(const fs::path& file); void trimFiles(); fs::path mostRecentSavefile(); diff --git a/src/Sha3Hash.h b/src/Sha3Hash.h index f47fc74b..f02b7707 100644 --- a/src/Sha3Hash.h +++ b/src/Sha3Hash.h @@ -18,7 +18,7 @@ class Sha3Hash { void update(const void* data, u32 size) { SHA3Update(&context, reinterpret_cast(data), size); } array finish() && { - u64 *p = reinterpret_cast(SHA3Final(&context)); + u64 const*p = reinterpret_cast(SHA3Final(&context)); return {p[0], p[1], p[2], p[3]}; } }; diff --git a/src/Signal.cpp b/src/Signal.cpp index 77192bd2..49b3bed3 100644 --- a/src/Signal.cpp +++ b/src/Signal.cpp @@ -8,7 +8,7 @@ using namespace std; static volatile sig_atomic_t signalled = 0; -static void (* volatile oldHandler)(int) = 0; +static void (* volatile oldHandler)(int) = nullptr; static void signalHandler(int signal) { signalled = signal; } @@ -27,6 +27,6 @@ void Signal::release() { if (isOwner) { isOwner = false; signal(SIGINT, oldHandler); - oldHandler = 0; + oldHandler = nullptr; } } diff --git a/src/Task.cpp b/src/Task.cpp index 28794956..9d8d3648 100644 --- a/src/Task.cpp +++ b/src/Task.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace { @@ -63,9 +64,9 @@ struct OsInfo { }; [[maybe_unused]] OsInfo getOsInfoMinimum() { - int plat = platform(); - string os = plat == LINUX_64 || plat == LINUX_32 ? "Linux" : plat == 4 ? "Windows" : plat == MACOSX_64 ? "MacOS" : ""; - return {os, "", ""}; + int const plat = platform(); + string const os = plat == LINUX_64 || plat == LINUX_32 ? "Linux" : plat == 4 ? "Windows" : plat == MACOSX_64 ? "MacOS" : ""; + return {.os=os, .release="", .arch=""}; } #if __has_include() @@ -75,7 +76,7 @@ struct OsInfo { OsInfo getOsInfo() { utsname buf{}; uname(&buf); - return OsInfo{buf.sysname, buf.release, buf.machine}; + return OsInfo{.os=buf.sysname, .release=buf.release, .arch=buf.machine}; } #else @@ -124,7 +125,7 @@ vector commonFields(u64 E, const char *worktype, const string &status) { vector tailFields(const std::string &AID, const Args &args) { assert(*VERSION); // version string isn't empty - OsInfo os = getOsInfo(); + OsInfo const os = getOsInfo(); return {json("program", vector{ json("name", "prpll"), json("version", (VERSION[0] == 'v') ? VERSION + 1 : VERSION), // skip leading "v" from version @@ -144,11 +145,11 @@ vector tailFields(const std::string &AID, const Args &args) { void writeResult(u32 instance, u64 E, const char *workType, const string &status, const std::string &AID, const Args &args, const vector& extras) { - fs::path resultsFile = "results-" + to_string(instance) + ".txt"; + fs::path const resultsFile = "results-" + to_string(instance) + ".txt"; vector fields = commonFields(E, workType, status); fields += extras; fields += tailFields(AID, args); - string s = json(std::move(fields)); + string const s = json(fields); log("%s\n", s.c_str()); File::append(resultsFile, s + '\n'); } @@ -166,7 +167,7 @@ void Task::writeResultPRP(FFTConfig fft, const Args &args, u32 instance, bool is // "proof":{"version":1, "power":6, "hashsize":64, "md5":"0123456789ABCDEF"}, if (!proofPath.empty()) { - ProofInfo info = proof::getInfo(proofPath); + ProofInfo const info = proof::getInfo(proofPath); if (info.power > 0) { fields.push_back(json("proof", vector{ json("version", 1), @@ -181,7 +182,7 @@ void Task::writeResultPRP(FFTConfig fft, const Args &args, u32 instance, bool is } void Task::writeResultLL(FFTConfig fft, const Args &args, u32 instance, bool isPrime, u64 res64) const { - vector fields{json("res64", hex(res64)), + vector const fields{json("res64", hex(res64)), json("fft-type", ffttype(fft)), json("fft-length", fft.size()), json("shift-count", 0), @@ -192,7 +193,7 @@ void Task::writeResultLL(FFTConfig fft, const Args &args, u32 instance, bool isP } void Task::writeResultCERT(FFTConfig fft, const Args &args, u32 instance, array hash, u32 squarings) const { - string hexhash = hex(hash[3]) + hex(hash[2]) + hex(hash[1]) + hex(hash[0]); + string const hexhash = hex(hash[3]) + hex(hash[2]) + hex(hash[1]) + hex(hash[0]); vector fields{json("worktype", "Cert"), json("exponent", exponent), json("sha3-hash", hexhash.c_str()), @@ -203,9 +204,9 @@ void Task::writeResultCERT(FFTConfig fft, const Args &args, u32 instance, array json("error-code", "00000000"), // I don't know the meaning of this }; fields += tailFields(AID, args); - string s = json(std::move(fields)); + string const s = json(fields); log("%s\n", s.c_str()); - fs::path resultsFile = "results-" + to_string(instance) + ".txt"; + fs::path const resultsFile = "results-" + to_string(instance) + ".txt"; File::append(resultsFile, s + '\n'); } @@ -220,24 +221,24 @@ void Task::execute(GpuCommon shared, u32 instance) { // is very common to use command line argument "-prp some-random-exponent" to get a quick // timing. Instead, we output a warning and test a smaller prime exponent. { - Primes primes; + Primes const primes; if (!primes.isPrime(exponent)) { - u64 new_exponent = primes.prevPrime(exponent); + u64 const new_exponent = primes.prevPrime(exponent); log("Warning: Exponent %" PRIu64 " is not prime. Using exponent %" PRIu64 " instead.\n", exponent, new_exponent); exponent = new_exponent; } } - LogContext pushContext(std::to_string(exponent)); + LogContext const pushContext(std::to_string(exponent)); - FFTConfig fft = FFTConfig::bestFit(*shared.args, exponent, shared.args->fftSpec); + FFTConfig const fft = FFTConfig::bestFit(*shared.args, exponent, shared.args->fftSpec); auto gpu = Gpu::make(exponent, shared, fft); if (kind == VERIFY) { - Proof proof{Proof::load(verifyPath)}; + Proof const proof{Proof::load(verifyPath)}; assert(proof.E == exponent); - bool ok = proof.verify(gpu.get()); + bool const ok = proof.verify(gpu.get()); log("proof '%s' %s\n", verifyPath.c_str(), ok ? "verified" : "failed"); } else if (kind == PRP || kind == LL) { diff --git a/src/TimeInfo.h b/src/TimeInfo.h index 872d1945..b99ad4b1 100644 --- a/src/TimeInfo.h +++ b/src/TimeInfo.h @@ -29,7 +29,7 @@ class TimeInfo { bool operator<(const TimeInfo& rhs) const { return times[2] > rhs.times[2]; } - auto secs() const { + [[nodiscard]] auto secs() const { std::array ret{}; for (int i = 0; i < 3; ++i) { ret[i] = times[i] * 1e-9; } return ret; diff --git a/src/Trig.cpp b/src/Trig.cpp index 917d748f..1359d5c4 100644 --- a/src/Trig.cpp +++ b/src/Trig.cpp @@ -80,10 +80,10 @@ TrigCoefs trigCoefs(u32 n) { assert(mid % 2 == 1); assert(mid <= 15 || (mid % 625 == 0 && mid / 625 <= 13)); - double scale = 1.0 / (twos / 4); + double const scale = 1.0 / (twos / 4); for (u32 i = 0; i < MUL_TAB.size(); ++i) { if (MUL_TAB[i] % mid == 0) { - return {MUL_TAB[i] / mid, scaleSin(SIN[i], scale), scaleCos(COS[i], scale)}; + return {.scale=MUL_TAB[i] / mid, .sinCoefs=scaleSin(SIN[i], scale), .cosCoefs=scaleCos(COS[i], scale)}; } } log("Trig tab not found for %u (%u * %u)\n", n, mid, twos); diff --git a/src/TrigBufCache.cpp b/src/TrigBufCache.cpp index 67c53d73..933e3116 100644 --- a/src/TrigBufCache.cpp +++ b/src/TrigBufCache.cpp @@ -3,8 +3,10 @@ #include #include "TrigBufCache.h" -#define SAVE_ONE_MORE_WIDTH_MUL 0 // I want to make saving the only option -- but rocm optimizer is inexplicably making it slower in carryfused -#define SAVE_ONE_MORE_HEIGHT_MUL 1 // In tailSquare this is the fastest option +enum { +SAVE_ONE_MORE_WIDTH_MUL = 0, // I want to make saving the only option -- but rocm optimizer is inexplicably making it slower in carryfused +SAVE_ONE_MORE_HEIGHT_MUL = 1 // In tailSquare this is the fastest option +}; #define _USE_MATH_DEFINES #include @@ -25,7 +27,7 @@ double2 root1Fancy(u32 N, u32 k) { assert(k < N); assert(k < N/4); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); return {double(cosl(angle) - 1), double(sinl(angle))}; } @@ -34,15 +36,15 @@ static double trigError(double c, double s) { return abs(trigNorm(c, s) - 1.0); // Round trig long double to double as to satisfy c^2 + s^2 == 1 as best as possible static double2 roundTrig(long double lc, long double ls) { - double c1 = lc; - double c2 = nexttoward(c1, lc); - double s1 = ls; - double s2 = nexttoward(s1, ls); + double const c1 = lc; + double const c2 = nexttoward(c1, lc); + double const s1 = ls; + double const s2 = nexttoward(s1, ls); double c = c1; double s = s1; - for (double tryC : {c1, c2}) { - for (double tryS : {s1, s2}) { + for (double const tryC : {c1, c2}) { + for (double const tryS : {s1, s2}) { if (trigError(tryC, tryS) < trigError(c, s)) { c = tryC; s = tryS; @@ -58,13 +60,13 @@ double2 root1(u32 N, u32 k) { if (k >= N/2) { auto [c, s] = root1(N, k - N/2); return {-c, -s}; - } else if (k > N/4) { + } if (k > N/4) { auto [c, s] = root1(N, N/2 - k); return {-c, s}; - } else if (k > N/8) { + } if (k > N/8) { auto [c, s] = root1(N, N/4 - k); return {s, c}; - } else { + } assert(k <= N/8); long double angle = M_PIl * k / (N / 2); @@ -79,17 +81,17 @@ double2 root1(u32 N, u32 k) { return {double(cosl(angle)), double(sinl(angle))}; } #endif - } + } // Epsilon value, 2^-250, should have an exact representation as a double. Used to avoid divide-by-zero in root1over. const double epsilon = 5.5271478752604445602472651921923E-76; // Protect against divide by zero // Returns the primitive root of unity of order N, to the power k. Returned format is cosine, sine/cosine. -double2 root1over(u32 N, u32 k) { +static double2 root1over(u32 N, u32 k) { assert(k < N); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); double c = cos(angle); long double s = sinl(angle); @@ -99,10 +101,10 @@ double2 root1over(u32 N, u32 k) { } // Returns the primitive root of unity of order N, to the power k. Returns only the cosine value. -double root1cos(u32 N, u32 k) { +static double root1cos(u32 N, u32 k) { assert(k < N); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); double c = cos(angle); if (c > -1.0e-15 && c < 1.0e-15) c = epsilon; @@ -110,10 +112,10 @@ double root1cos(u32 N, u32 k) { } // Returns the primitive root of unity of order N, to the power k. Returns only the cosine value divided by another cosine value. -double root1cosover(u32 N, u32 k, double over) { +static double root1cosover(u32 N, u32 k, double over) { assert(k < N); - long double angle = M_PIl * k / (N / 2); + long double const angle = M_PIl * k / (N / 2); long double c = cosl(angle); if (c > -1.0e-15 && c < 1.0e-15) c = epsilon; @@ -123,9 +125,9 @@ double root1cosover(u32 N, u32 k, double over) { static const constexpr bool LOG_TRIG_ALLOC = false; // Interleave two lines of trig values so that AMD GPUs can use global_load_dwordx4 instructions -void T2shuffle(u32 size, u32 radix, u32 line, vector &tab) { +static void T2shuffle(u32 size, u32 radix, u32 line, vector &tab) { vector line1, line2; - u32 line_size = size / radix; + u32 const line_size = size / radix; for (u32 col = 0; col < line_size; ++col) { line1.push_back(tab[line*line_size + col]); line2.push_back(tab[(line+1)*line_size + col]); @@ -136,9 +138,9 @@ void T2shuffle(u32 size, u32 radix, u32 line, vector &tab) { } } -vector genSmallTrigFP64(u32 size, u32 radix) { +static vector genSmallTrigFP64(u32 size, u32 radix) { if (LOG_TRIG_ALLOC) { log("genSmallTrigFP64(%u, %u)\n", size, radix); } - u32 WG = size / radix; + u32 const WG = size / radix; vector tab; // old fft_WIDTH and fft_HEIGHT @@ -160,7 +162,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Sine/cosine values for first fft4 or fft8 for (u32 line = 1; line < radix; ++line) { for (u32 col = 0; col < WG; ++col) { - double2 root = root1over(size, col * line); + double2 const root = root1over(size, col * line); tab1.push_back(root.second); } } @@ -168,7 +170,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Sine/cosine values for later fft4 or fft8 for (u32 line = 0; line < radix; ++line) { for (u32 col = 0; col < WG; col += radix) { - double2 root = root1over(size, col * line); + double2 const root = root1over(size, col * line); tab1.push_back(root.second); } } @@ -176,7 +178,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Cosine values for first fft4 or fft8 (output in post-shufl order) //TODO: Examine why when sine is 0.0 cosine is not 1.0 or -1.0 (printf is outputting 0.999... and -0.999...) for (u32 grp = 0; grp < WG; ++grp) { - u32 line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. + u32 const line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. for (u32 col = 0; col < radix; ++col) { double divide_by = 1.0; // Compute cosine3 / cosine1 @@ -194,7 +196,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { // Cosine values for later fft4 or fft8 (output in post-shufl order). Similar to cosines above but output every radix-th value. for (u32 grp = 0; grp < radix; ++grp) { for (u32 col = 0; col < WG; col += radix) { - u32 line = col / (WG/radix); + u32 const line = col / (WG/radix); double divide_by = 1.0; // Compute cosine3 / cosine1 if ((radix == 4 && line == 3) || (radix == 8 && save_one_more_mul && line == 3)) { @@ -213,7 +215,7 @@ vector genSmallTrigFP64(u32 size, u32 radix) { for (u32 i = radix; i < 2*radix; i += 2) T2shuffle(size, radix, i, tab1); // Convert to a vector of double2 - for (u32 i = 0; i < tab1.size(); i += 2) tab.push_back({tab1[i], tab1[i+1]}); + for (u32 i = 0; i < tab1.size(); i += 2) tab.emplace_back(tab1[i], tab1[i+1]); } tab.resize(5*size); @@ -221,16 +223,16 @@ vector genSmallTrigFP64(u32 size, u32 radix) { } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { if (LOG_TRIG_ALLOC) { log("genSmallTrigComboFP64(%u, %u)\n", size, radix); } vector tab = genSmallTrigFP64(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating from scratch, no memory accesses + u32 const tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating from scratch, no memory accesses // From tailSquare pre-calculate some or all of these: T2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); if (tail_trigs == 1) { // Some trig values in memory, some are computed with a complex multiply. Best option on a Radeon VII. - u32 height = size; + u32 const height = size; // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1(width * middle * height, width * middle * me)); @@ -242,10 +244,10 @@ vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 siz } } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with lousy DP performance. - u32 height = size; + u32 const height = size; for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1(width * middle * height, line + width * middle * me)); } @@ -258,9 +260,11 @@ vector genSmallTrigComboFP64(Args *args, u32 width, u32 middle, u32 siz // starting from a MIDDLE of 5 we consider angles in [0, 2Pi/MIDDLE] as worth storing with the // cos-1 "fancy" trick. -#define SHARP_MIDDLE 5 +enum { +SHARP_MIDDLE = 5 +}; -vector genMiddleTrigFP64(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigFP64(u32 smallH, u32 middle, u32 width) { if (LOG_TRIG_ALLOC) { log("genMiddleTrigFP64(%u, %u, %u)\n", smallH, middle, width); } vector tab; if (middle == 1) { @@ -290,7 +294,7 @@ float2 root1FancyFP32(u32 N, u32 k) { assert(k < N); assert(k < N/4); - double angle = M_PI * k / (N / 2); + double const angle = M_PI * k / (N / 2); return {float(cos(angle) - 1), float(sin(angle))}; } @@ -299,15 +303,15 @@ static float trigError(float c, float s) { return abs(trigNorm(c, s) - 1.0f); } // Round trig double to float as to satisfy c^2 + s^2 == 1 as best as possible static float2 roundTrig(double lc, double ls) { - float c1 = lc; - float c2 = nexttoward(c1, lc); - float s1 = ls; - float s2 = nexttoward(s1, ls); + float const c1 = lc; + float const c2 = nexttoward(c1, lc); + float const s1 = ls; + float const s2 = nexttoward(s1, ls); float c = c1; float s = s1; - for (float tryC : {c1, c2}) { - for (float tryS : {s1, s2}) { + for (float const tryC : {c1, c2}) { + for (float const tryS : {s1, s2}) { if (trigError(tryC, tryS) < trigError(c, s)) { c = tryC; s = tryS; @@ -323,28 +327,28 @@ float2 root1FP32(u32 N, u32 k) { if (k >= N/2) { auto [c, s] = root1FP32(N, k - N/2); return {-c, -s}; - } else if (k > N/4) { + } if (k > N/4) { auto [c, s] = root1FP32(N, N/2 - k); return {-c, s}; - } else if (k > N/8) { + } if (k > N/8) { auto [c, s] = root1FP32(N, N/4 - k); return {s, c}; - } else { + } assert(k <= N/8); double angle = M_PI * k / (N / 2); return roundTrig(cos(angle), sin(angle)); - } + } // Epsilon value, 2^-50, should have an exact representation as a float. Used to avoid divide-by-zero in root1overFP32. const double epsilonFP32 = 8.8817841970012523233890533447266e-16; // Protect against divide by zero // Returns the primitive root of unity of order N, to the power k. Returned format is cosine, sine/cosine. -float2 root1overFP32(u32 N, u32 k) { +static float2 root1overFP32(u32 N, u32 k) { assert(k < N); - double angle = M_PI * k / (N / 2); + double const angle = M_PI * k / (N / 2); double c = cos(angle); double s = sin(angle); @@ -354,10 +358,10 @@ float2 root1overFP32(u32 N, u32 k) { } // Returns the primitive root of unity of order N, to the power k. Returns only the cosine value. -float root1cosFP32(u32 N, u32 k) { +static float root1cosFP32(u32 N, u32 k) { assert(k < N); - double angle = M_PI * k / (N / 2); + double const angle = M_PI * k / (N / 2); double c = cos(angle); if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; @@ -365,10 +369,10 @@ float root1cosFP32(u32 N, u32 k) { } // Returns the primitive root of unity of order N, to the power k. Returns only the cosine value divided by another cosine value. -float root1cosoverFP32(u32 N, u32 k, double over) { +static float root1cosoverFP32(u32 N, u32 k, double over) { assert(k < N); - double angle = M_PI * k / (N / 2); + double const angle = M_PI * k / (N / 2); double c = cos(angle); if (c > -1.0e-15 && c < 1.0e-15) c = epsilonFP32; @@ -376,9 +380,9 @@ float root1cosoverFP32(u32 N, u32 k, double over) { } // Interleave two lines of trig values so that AMD GPUs can use global_load_dwordx4 instructions -void F2shuffle(u32 size, u32 radix, u32 line, vector &tab) { +static void F2shuffle(u32 size, u32 radix, u32 line, vector &tab) { vector line1, line2; - u32 line_size = size / radix; + u32 const line_size = size / radix; for (u32 col = 0; col < line_size; ++col) { line1.push_back(tab[line*line_size + col]); line2.push_back(tab[(line+1)*line_size + col]); @@ -389,8 +393,8 @@ void F2shuffle(u32 size, u32 radix, u32 line, vector &tab) { } } -vector genSmallTrigFP32(u32 size, u32 radix) { - u32 WG = size / radix; +static vector genSmallTrigFP32(u32 size, u32 radix) { + u32 const WG = size / radix; vector tab; // old fft_WIDTH and fft_HEIGHT @@ -412,7 +416,7 @@ vector genSmallTrigFP32(u32 size, u32 radix) { // Sine/cosine values for first fft4 or fft8 for (u32 line = 1; line < radix; ++line) { for (u32 col = 0; col < WG; ++col) { - float2 root = root1overFP32(size, col * line); + float2 const root = root1overFP32(size, col * line); tab1.push_back(root.second); } } @@ -420,14 +424,14 @@ vector genSmallTrigFP32(u32 size, u32 radix) { // Sine/cosine values for later fft4 or fft8 for (u32 line = 0; line < radix; ++line) { for (u32 col = 0; col < WG; col += radix) { - float2 root = root1overFP32(size, col * line); + float2 const root = root1overFP32(size, col * line); tab1.push_back(root.second); } } // Cosine values for first fft4 or fft8 (output in post-shufl order) for (u32 grp = 0; grp < WG; ++grp) { - u32 line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. + u32 const line = grp / (WG/radix); // Output "line" number, where each line multiplies a different u[i]. There are radix lines. Each line has WG values. for (u32 col = 0; col < radix; ++col) { float divide_by = 1.0; // Compute cosine3 / cosine1 @@ -445,7 +449,7 @@ vector genSmallTrigFP32(u32 size, u32 radix) { // Cosine values for later fft4 or fft8 (output in post-shufl order). Similar to cosines above but output every radix-th value. for (u32 grp = 0; grp < radix; ++grp) { for (u32 col = 0; col < WG; col += radix) { - u32 line = col / (WG/radix); + u32 const line = col / (WG/radix); double divide_by = 1.0; // Compute cosine3 / cosine1 if ((radix == 4 && line == 3) || (radix == 8 && save_one_more_mul && line == 3)) { @@ -464,7 +468,7 @@ vector genSmallTrigFP32(u32 size, u32 radix) { for (u32 i = radix; i < 2*radix; i += 2) F2shuffle(size, radix, i, tab1); // Convert to a vector of float2 - for (u32 i = 0; i < tab1.size(); i += 2) tab.push_back({tab1[i], tab1[i+1]}); + for (u32 i = 0; i < tab1.size(); i += 2) tab.emplace_back(tab1[i], tab1[i+1]); } tab.resize(5*size); @@ -472,14 +476,14 @@ vector genSmallTrigFP32(u32 size, u32 radix) { } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab = genSmallTrigFP32(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS32", 2); // Default is calculating from scratch, no memory accesses + u32 const tail_trigs = args->value("TAIL_TRIGS32", 2); // Default is calculating from scratch, no memory accesses // From tailSquare pre-calculate some or all of these: F2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); if (tail_trigs == 1) { // Some trig values in memory, some are computed with a complex multiply. - u32 height = size; + u32 const height = size; // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1FP32(width * middle * height, width * middle * me)); @@ -491,10 +495,10 @@ vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size } } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with lousy FP performance? - u32 height = size; + u32 const height = size; for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1FP32(width * middle * height, line + width * middle * me)); } @@ -505,7 +509,7 @@ vector genSmallTrigComboFP32(Args *args, u32 width, u32 middle, u32 size return tab; } -vector genMiddleTrigFP32(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigFP32(u32 smallH, u32 middle, u32 width) { vector tab; if (middle == 1) { tab.resize(1); @@ -556,10 +560,10 @@ class Z31 } public: - Z31() {} + Z31() = default; explicit Z31(const uint32_t n) : _n(n) {} - uint32_t get() const { return _n; } + [[nodiscard]] uint32_t get() const { return _n; } bool operator!=(const Z31 & rhs) const { return (_n != rhs._n); } @@ -570,7 +574,7 @@ class Z31 Z31 operator-(const Z31 & rhs) const { return Z31(_sub(_n, rhs._n)); } Z31 operator*(const Z31 & rhs) const { return Z31(_mul(_n, rhs._n)); } - Z31 sqr() const { return Z31(_mul(_n, _n)); } + [[nodiscard]] Z31 sqr() const { return Z31(_mul(_n, _n)); } }; @@ -584,20 +588,20 @@ class GF31 static const uint32_t _h_0 = 7735u, _h_1 = 748621u; public: - GF31() {} + GF31() = default; explicit GF31(const Z31 & s0, const Z31 & s1) : _s0(s0), _s1(s1) {} explicit GF31(const uint32_t n0, const uint32_t n1) : _s0(n0), _s1(n1) {} - const Z31 & s0() const { return _s0; } - const Z31 & s1() const { return _s1; } + [[nodiscard]] const Z31 & s0() const { return _s0; } + [[nodiscard]] const Z31 & s1() const { return _s1; } GF31 operator+(const GF31 & rhs) const { return GF31(_s0 + rhs._s0, _s1 + rhs._s1); } GF31 operator-(const GF31 & rhs) const { return GF31(_s0 - rhs._s0, _s1 - rhs._s1); } - GF31 sqr() const { const Z31 t = _s0 * _s1; return GF31(_s0.sqr() - _s1.sqr(), t + t); } - GF31 mul(const GF31 & rhs) const { return GF31(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } + [[nodiscard]] GF31 sqr() const { const Z31 t = _s0 * _s1; return GF31(_s0.sqr() - _s1.sqr(), t + t); } + [[nodiscard]] GF31 mul(const GF31 & rhs) const { return GF31(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } - GF31 pow(const uint64_t e) const + [[nodiscard]] GF31 pow(const uint64_t e) const { if (e == 0) return GF31(1u, 0u); GF31 r = GF31(1u, 0u), y = *this; @@ -605,26 +609,26 @@ class GF31 return r.mul(y); } - static const GF31 root_one(const size_t n) { return GF31(Z31(_h_0), Z31(_h_1)).pow(_h_order / n); } + static GF31 root_one(const size_t n) { return GF31(Z31(_h_0), Z31(_h_1)).pow(_h_order / n); } static uint8_t log2_root_two(const size_t n) { return uint8_t(((uint64_t(1) << 30) / n) % 31); } }; // Returns the primitive root of unity of order N, to the power k. -uint2 root1GF31(GF31 root1N, u32 k) { - GF31 x = root1N.pow(k); +static uint2 root1GF31(GF31 root1N, u32 k) { + GF31 const x = root1N.pow(k); return { x.s0().get(), x.s1().get() }; } uint2 root1GF31(u32 N, u32 k) { assert(k < N); - GF31 root1N = GF31::root_one(N); + GF31 const root1N = GF31::root_one(N); return root1GF31(root1N, k); } -vector genSmallTrigGF31(u32 size, u32 radix) { - u32 WG = size / radix; +static vector genSmallTrigGF31(u32 size, u32 radix) { + u32 const WG = size / radix; vector tab; - GF31 root1size = GF31::root_one(size); + GF31 const root1size = GF31::root_one(size); for (u32 line = 1; line < radix; ++line) { for (u32 col = 0; col < WG; ++col) { tab.push_back(root1GF31(root1size, col * line)); @@ -635,14 +639,14 @@ vector genSmallTrigGF31(u32 size, u32 radix) { } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab = genSmallTrigGF31(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS31", 0); // Default is reading all trigs from memory + u32 const tail_trigs = args->value("TAIL_TRIGS31", 0); // Default is reading all trigs from memory // From tailSquareGF31 pre-calculate some or all of these: GF31 trig = slowTrigGF31(line + H * lowMe, ND / NH * 2); - u32 height = size; - GF31 root1wmh = GF31::root_one(width * middle * height); + u32 const height = size; + GF31 const root1wmh = GF31::root_one(width * middle * height); if (tail_trigs >= 1) { // Some trig values in memory, some are computed with a complex multiply. Best option on a Radeon VII. // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { @@ -656,8 +660,8 @@ vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with great memory performance. for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1GF31(root1wmh, line + width * middle * me)); } @@ -668,18 +672,18 @@ vector genSmallTrigComboGF31(Args *args, u32 width, u32 middle, u32 size, return tab; } -vector genMiddleTrigGF31(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigGF31(u32 smallH, u32 middle, u32 width) { vector tab; if (middle == 1) { tab.resize(1); } else { - GF31 root1hm = GF31::root_one(smallH * middle); + GF31 const root1hm = GF31::root_one(smallH * middle); for (u32 m = 1; m < middle; ++m) { for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF31(root1hm, k * m)); } } - GF31 root1mw = GF31::root_one(middle * width); + GF31 const root1mw = GF31::root_one(middle * width); for (u32 k = 0; k < width; ++k) { tab.push_back(root1GF31(root1mw, k)); } - GF31 root1wmh = GF31::root_one(width * middle * smallH); + GF31 const root1wmh = GF31::root_one(width * middle * smallH); for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF31(root1wmh, k)); } } return tab; @@ -714,16 +718,16 @@ class Z61 static uint64_t _mul(const uint64_t a, const uint64_t b) { const u128 t = a * u128(b); - const uint64_t lo = uint64_t(t), hi = uint64_t(t >> 64); + const auto lo = uint64_t(t), hi = uint64_t(t >> 64); const uint64_t lo61 = lo & _p, hi61 = (lo >> 61) | (hi << 3); return _add(lo61, hi61); } public: - Z61() {} + Z61() = default; explicit Z61(const uint64_t n) : _n(n) {} - uint64_t get() const { return _n; } + [[nodiscard]] uint64_t get() const { return _n; } bool operator!=(const Z61 & rhs) const { return (_n != rhs._n); } @@ -731,7 +735,7 @@ class Z61 Z61 operator-(const Z61 & rhs) const { return Z61(_sub(_n, rhs._n)); } Z61 operator*(const Z61 & rhs) const { return Z61(_mul(_n, rhs._n)); } - Z61 sqr() const { return Z61(_mul(_n, _n)); } + [[nodiscard]] Z61 sqr() const { return Z61(_mul(_n, _n)); } }; // GF((2^61 - 1)^2): the prime field of order p^2, p = 2^61 - 1 @@ -746,20 +750,20 @@ class GF61 static const uint64_t _h_order = uint64_t(1) << 62; public: - GF61() {} + GF61() = default; explicit GF61(const Z61 & s0, const Z61 & s1) : _s0(s0), _s1(s1) {} explicit GF61(const uint64_t n0, const uint64_t n1) : _s0(n0), _s1(n1) {} - const Z61 & s0() const { return _s0; } - const Z61 & s1() const { return _s1; } + [[nodiscard]] const Z61 & s0() const { return _s0; } + [[nodiscard]] const Z61 & s1() const { return _s1; } GF61 operator+(const GF61 & rhs) const { return GF61(_s0 + rhs._s0, _s1 + rhs._s1); } GF61 operator-(const GF61 & rhs) const { return GF61(_s0 - rhs._s0, _s1 - rhs._s1); } - GF61 sqr() const { const Z61 t = _s0 * _s1; return GF61(_s0.sqr() - _s1.sqr(), t + t); } - GF61 mul(const GF61 & rhs) const { return GF61(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } + [[nodiscard]] GF61 sqr() const { const Z61 t = _s0 * _s1; return GF61(_s0.sqr() - _s1.sqr(), t + t); } + [[nodiscard]] GF61 mul(const GF61 & rhs) const { return GF61(_s0 * rhs._s0 - _s1 * rhs._s1, _s1 * rhs._s0 + _s0 * rhs._s1); } - GF61 pow(const uint64_t e) const + [[nodiscard]] GF61 pow(const uint64_t e) const { if (e == 0) return GF61(1u, 0u); GF61 r = GF61(1u, 0u), y = *this; @@ -767,26 +771,26 @@ class GF61 return r.mul(y); } - static const GF61 root_one(const size_t n) { return GF61(Z61(_h_0), Z61(_h_1)).pow(_h_order / n); } + static GF61 root_one(const size_t n) { return GF61(Z61(_h_0), Z61(_h_1)).pow(_h_order / n); } static uint8_t log2_root_two(const size_t n) { return uint8_t(((uint64_t(1) << 60) / n) % 61); } }; // Returns the primitive root of unity of order N, to the power k. -ulong2 root1GF61(GF61 root1N, u32 k) { - GF61 x = root1N.pow(k); +static ulong2 root1GF61(GF61 root1N, u32 k) { + GF61 const x = root1N.pow(k); return { x.s0().get(), x.s1().get() }; } ulong2 root1GF61(u32 N, u32 k) { assert(k < N); - GF61 root1N = GF61::root_one(N); + GF61 const root1N = GF61::root_one(N); return root1GF61(root1N, k); } -vector genSmallTrigGF61(u32 size, u32 radix) { - u32 WG = size / radix; +static vector genSmallTrigGF61(u32 size, u32 radix) { + u32 const WG = size / radix; vector tab; - GF61 root1size = GF61::root_one(size); + GF61 const root1size = GF61::root_one(size); for (u32 line = 1; line < radix; ++line) { for (u32 col = 0; col < WG; ++col) { tab.push_back(root1GF61(root1size, col * line)); @@ -797,14 +801,14 @@ vector genSmallTrigGF61(u32 size, u32 radix) { } // Generate the small trig values for fft_HEIGHT plus optionally trig values used in pairSq. -vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab = genSmallTrigGF61(size, radix); - u32 tail_trigs = args->value("TAIL_TRIGS61", 0); // Default is reading all trigs from memory + u32 const tail_trigs = args->value("TAIL_TRIGS61", 0); // Default is reading all trigs from memory // From tailSquareGF61 pre-calculate some or all of these: GF61 trig = slowTrigGF61(line + H * lowMe, ND / NH * 2); - u32 height = size; - GF61 root1wmh = GF61::root_one(width * middle * height); + u32 const height = size; + GF61 const root1wmh = GF61::root_one(width * middle * height); if (tail_trigs >= 1) { // Some trig values in memory, some are computed with a complex multiply. Best option on a Radeon VII. // Output line 0 trig values to be read by every u,v pair of lines for (u32 me = 0; me < height / radix; ++me) { @@ -818,8 +822,8 @@ vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size } if (tail_trigs == 0) { // All trig values read from memory. Best option for GPUs with great memory performance. for (u32 u = 0; u <= width * middle / 2; ++u) { - for (u32 v = 0; v < (tail_single_wide ? 1 : 2); ++v) { - u32 line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); + for (u32 v = 0; std::cmp_less(v , (tail_single_wide ? 1 : 2)); ++v) { + u32 const line = (v == 0) ? u : (u ? width * middle - u : width * middle / 2); for (u32 me = 0; me < height / radix; ++me) { tab.push_back(root1GF61(root1wmh, line + width * middle * me)); } @@ -830,18 +834,18 @@ vector genSmallTrigComboGF61(Args *args, u32 width, u32 middle, u32 size return tab; } -vector genMiddleTrigGF61(u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrigGF61(u32 smallH, u32 middle, u32 width) { vector tab; if (middle == 1) { tab.resize(1); } else { - GF61 root1hm = GF61::root_one(smallH * middle); + GF61 const root1hm = GF61::root_one(smallH * middle); for (u32 m = 1; m < middle; ++m) { for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF61(root1hm, k * m)); } } - GF61 root1mw = GF61::root_one(middle * width); + GF61 const root1mw = GF61::root_one(middle * width); for (u32 k = 0; k < width; ++k) { tab.push_back(root1GF61(root1mw, k)); } - GF61 root1wmh = GF61::root_one(width * middle * smallH); + GF61 const root1wmh = GF61::root_one(width * middle * smallH); for (u32 k = 0; k < smallH; ++k) { tab.push_back(root1GF61(root1wmh, k)); } } return tab; @@ -852,7 +856,7 @@ vector genMiddleTrigGF61(u32 smallH, u32 middle, u32 width) { /* Build all the needed trig values into one big buffer */ /**********************************************************/ -vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { +static vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { vector tab; u32 tabsize; @@ -891,7 +895,7 @@ vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { return tab; } -vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { +static vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab; u32 tabsize; @@ -930,7 +934,7 @@ vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 midd return tab; } -vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 width) { +static vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 width) { vector tab; u32 tabsize; @@ -974,32 +978,32 @@ vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 width) /* Code to manage a cache of trigBuffers */ /********************************************************/ -#define make_key_part(b,tt,b31,tt31,b32,tt32,b61,tt61,tk) ((((((((b+tt) << 2) + b31+tt31) << 2) + b32+tt32) << 2) + b61+tt61) << 2) + tk +#define make_key_part(b,tt,b31,tt31,b32,tt32,b61,tt61,tk) ((((((((((b)+(tt)) << 2) + (b31)+(tt31)) << 2) + (b32)+(tt32)) << 2) + (b61)+(tt61)) << 2) + (tk)) TrigBufCache::~TrigBufCache() = default; TrigPtr TrigBufCache::smallTrig(Args *args, FFTConfig fft, u32 width, u32 nW, u32 middle, u32 height, u32 nH, bool tail_single_wide) { - lock_guard lock{mut}; + std::scoped_lock const lock{mut}; auto& m = small; TrigPtr p{}; - u32 tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses - u32 tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory - u32 tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses - u32 tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory - u32 key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); + u32 const tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses + u32 const tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory + u32 const tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses + u32 const tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory + u32 const key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); // See if there is an existing smallTrigCombo that we can return (using only a subset of the data) // In theory, we could match any smallTrigCombo where width matches. However, SMALLTRIG_GF31_SIZE wouldn't be able to figure out the size. // In practice, those cases will likely never arise. if (width == height && nW == nH) { - decay_t::key_type key{height, nH, width, middle, key_part}; + decay_t::key_type const key{height, nH, width, middle, key_part}; auto it = m.find(key); if (it != m.end() && (p = it->second.lock())) return p; } // See if there is an existing non-combo smallTrig that we can return - decay_t::key_type key{width, nW, 0, 0, key_part}; + decay_t::key_type const key{width, nW, 0, 0, key_part}; auto it = m.find(key); if (it != m.end() && (p = it->second.lock())) return p; @@ -1011,19 +1015,19 @@ TrigPtr TrigBufCache::smallTrig(Args *args, FFTConfig fft, u32 width, u32 nW, u3 } TrigPtr TrigBufCache::smallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 middle, u32 height, u32 nH, bool tail_single_wide) { - u32 tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses - u32 tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory - u32 tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses - u32 tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory - u32 key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); + u32 const tail_trigs = args->value("TAIL_TRIGS", 2); // Default is calculating FP64 trigs from scratch, no memory accesses + u32 const tail_trigs31 = args->value("TAIL_TRIGS31", 2); // Default is reading GF31 trigs from memory + u32 const tail_trigs32 = args->value("TAIL_TRIGS32", 2); // Default is calculating FP32 trigs from scratch, no memory accesses + u32 const tail_trigs61 = args->value("TAIL_TRIGS61", 2); // Default is reading GF61 trigs from memory + u32 const key_part = make_key_part(fft.FFT_FP64, tail_trigs, fft.NTT_GF31, tail_trigs31, fft.FFT_FP32, tail_trigs32, fft.NTT_GF61, tail_trigs61, tail_single_wide); // If there are no pre-computed trig values we might be able to share this trig table with fft_WIDTH if (((tail_trigs == 2 && fft.FFT_FP64) || (tail_trigs32 == 2 && fft.FFT_FP32)) && !fft.NTT_GF31 && !fft.NTT_GF61) return smallTrig(args, fft, height, nH, middle, height, nH, tail_single_wide); - lock_guard lock{mut}; + std::scoped_lock const lock{mut}; auto& m = small; - decay_t::key_type key{height, nH, width, middle, key_part}; + decay_t::key_type const key{height, nH, width, middle, key_part}; TrigPtr p{}; auto it = m.find(key); @@ -1035,11 +1039,11 @@ TrigPtr TrigBufCache::smallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 m return p; } -TrigPtr TrigBufCache::middleTrig(Args *args, FFTConfig fft, u32 SMALL_H, u32 MIDDLE, u32 width) { - lock_guard lock{mut}; +TrigPtr TrigBufCache::middleTrig(Args * /*args*/, FFTConfig fft, u32 SMALL_H, u32 MIDDLE, u32 width) { + std::scoped_lock const lock{mut}; auto& m = middle; - u32 key_part = make_key_part(fft.FFT_FP64, 0, fft.NTT_GF31, 0, fft.FFT_FP32, 0, fft.NTT_GF61, 0, 0); - decay_t::key_type key{SMALL_H, MIDDLE, width, key_part}; + u32 const key_part = make_key_part(fft.FFT_FP64, 0, fft.NTT_GF31, 0, fft.FFT_FP32, 0, fft.NTT_GF61, 0, 0); + decay_t::key_type const key{SMALL_H, MIDDLE, width, key_part}; TrigPtr p{}; auto it = m.find(key); diff --git a/src/TrigBufCache.h b/src/TrigBufCache.h index 1808b355..35482084 100644 --- a/src/TrigBufCache.h +++ b/src/TrigBufCache.h @@ -6,6 +6,7 @@ #include "FFTConfig.h" #include +#include using TrigBuf = Buffer; using TrigPtr = shared_ptr; @@ -18,7 +19,7 @@ class StrongCache { explicit StrongCache(u32 size) : ptrs(size) {} void add(TrigPtr ptr) { - ptrs.at(pos) = ptr; + ptrs.at(pos) = std::move(ptr); if (++pos >= ptrs.size()) { pos = 0; } } }; @@ -58,24 +59,24 @@ uint2 root1GF31(u32 N, u32 k); ulong2 root1GF61(u32 N, u32 k); // Compute the size of the largest possible trig buffer given width, middle, height (in number of double2 values) -#define SMALLTRIG_FP64_SIZE(W,M,H,nH) (W != H || H == 0 ? W * 5 : SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH)) // See genSmallTrigFP64 -#define SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH) (H * 5 + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboFP64 -#define MIDDLETRIG_FP64_SIZE(W,M,H) (H + W + H) // See genMiddleTrigFP64 +#define SMALLTRIG_FP64_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) * 5 : SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH)) // See genSmallTrigFP64 +#define SMALLTRIGCOMBO_FP64_SIZE(W,M,H,nH) ((H) * 5 + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboFP64 +#define MIDDLETRIG_FP64_SIZE(W,M,H) ((H) + (W) + (H)) // See genMiddleTrigFP64 // Compute the size of the largest possible trig buffer given width, middle, height (in number of float2 values) -#define SMALLTRIG_FP32_SIZE(W,M,H,nH) (W != H || H == 0 ? W * 5 : SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH)) // See genSmallTrigFP32 -#define SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH) (H * 5 + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboFP32 -#define MIDDLETRIG_FP32_SIZE(W,M,H) (H + W + H) // See genMiddleTrigFP32 +#define SMALLTRIG_FP32_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) * 5 : SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH)) // See genSmallTrigFP32 +#define SMALLTRIGCOMBO_FP32_SIZE(W,M,H,nH) ((H) * 5 + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboFP32 +#define MIDDLETRIG_FP32_SIZE(W,M,H) ((H) + (W) + (H)) // See genMiddleTrigFP32 // Compute the size of the largest possible trig buffer given width, middle, height (in number of uint2 values) -#define SMALLTRIG_GF31_SIZE(W,M,H,nH) (W != H || H == 0 ? W : SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH)) // See genSmallTrigGF31 -#define SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH) (H + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboGF31 -#define MIDDLETRIG_GF31_SIZE(W,M,H) (H * (M - 1) + W + H) // See genMiddleTrigGF31 +#define SMALLTRIG_GF31_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) : SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH)) // See genSmallTrigGF31 +#define SMALLTRIGCOMBO_GF31_SIZE(W,M,H,nH) ((H) + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboGF31 +#define MIDDLETRIG_GF31_SIZE(W,M,H) ((H) * ((M) - 1) + (W) + (H)) // See genMiddleTrigGF31 // Compute the size of the largest possible trig buffer given width, middle, height (in number of ulong2 values) -#define SMALLTRIG_GF61_SIZE(W,M,H,nH) (W != H || H == 0 ? W : SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH)) // See genSmallTrigGF61 -#define SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH) (H + (W * M / 2 + 1) * 2 * H / nH) // See genSmallTrigComboGF61 -#define MIDDLETRIG_GF61_SIZE(W,M,H) (H * (M - 1) + W + H) // See genMiddleTrigGF61 +#define SMALLTRIG_GF61_SIZE(W,M,H,nH) ((W) != (H) || (H) == 0 ? (W) : SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH)) // See genSmallTrigGF61 +#define SMALLTRIGCOMBO_GF61_SIZE(W,M,H,nH) ((H) + ((W) * (M) / 2 + 1) * 2 * (H) / (nH)) // See genSmallTrigComboGF61 +#define MIDDLETRIG_GF61_SIZE(W,M,H) ((H) * ((M) - 1) + (W) + (H)) // See genMiddleTrigGF61 // Convert above sizes to distances (in units of double2) #define SMALLTRIG_FP64_DIST(W,M,H,nH) SMALLTRIG_FP64_SIZE(W,M,H,nH) diff --git a/src/TuneEntry.cpp b/src/TuneEntry.cpp index 68b6915d..e1a0900b 100644 --- a/src/TuneEntry.cpp +++ b/src/TuneEntry.cpp @@ -7,7 +7,7 @@ // Returns whether *results* was updated. bool TuneEntry::update(vector& results) const { - u64 maxExp = fft.maxExp(); + u64 const maxExp = fft.maxExp(); [[maybe_unused]] bool didErase = false; int i{}; @@ -29,11 +29,11 @@ bool TuneEntry::update(vector& results) const { // Returns whether entry *e* represents an improvement over *results* (i.e. would update the results). bool TuneEntry::willUpdate(const vector& results) const { - u64 maxExp = fft.maxExp(); + u64 const maxExp = fft.maxExp(); for (const auto& r : results) { if (r.cost > cost) { break; - } else if (r.fft.maxExp() >= maxExp) { + } if (r.fft.maxExp() >= maxExp) { return false; } } @@ -61,7 +61,7 @@ vector TuneEntry::readTuneFile(const Args& args) { if (sscanf(line.c_str(), "%lf %31s", &cost, specBuf) < 2) { log("tune.txt line '%s' ignored\n", line.c_str()); } - FFTConfig fft{specBuf}; + FFTConfig const fft{specBuf}; assert(cost >= prevCost && fft.maxExp() > prevMaxExp); prevCost = cost; prevMaxExp = fft.maxExp(); @@ -76,7 +76,7 @@ void TuneEntry::writeTuneFile(const vector& results) { [[maybe_unused]] double prevCost{}; CycleFile tune{"tune.txt"}; for (const TuneEntry& r : results) { - u64 maxExp = r.fft.maxExp(); + u64 const maxExp = r.fft.maxExp(); assert(r.cost >= prevCost && maxExp > prevMaxExp); prevCost = r.cost; prevMaxExp = maxExp; diff --git a/src/TuneEntry.h b/src/TuneEntry.h index 79a0d06a..73a86e9e 100644 --- a/src/TuneEntry.h +++ b/src/TuneEntry.h @@ -14,7 +14,7 @@ class TuneEntry { FFTConfig fft; bool update(std::vector&) const; - bool willUpdate(const vector&) const; + [[nodiscard]] bool willUpdate(const vector&) const; static vector readTuneFile(const Args& args); static void writeTuneFile(const vector&); diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 63762490..0fd025dd 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -36,7 +36,7 @@ std::optional parse(const std::string& line) { bool isCERT = false; if (topParts.size() == 2) { - string kind = topParts.front(); + const string& kind = topParts.front(); if (kind == "PRP" || kind == "PRPDC") { isPRP = true; } else if (kind == "Test" || kind == "DoubleCheck") { @@ -58,14 +58,14 @@ std::optional parse(const std::string& line) { parts.erase(parts.begin()); } - string s = (parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && (parts[3] == "-1" || parts[3] == "-1\n")) ? parts[2] + string const s = (parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && (parts[3] == "-1" || parts[3] == "-1\n")) ? parts[2] : (!parts.empty() ? parts[0] : ""); const char *end = s.c_str() + s.size(); u64 exp{}; auto [ptr, _] = from_chars(s.c_str(), end, exp, 10); if (ptr != end) { exp = 0; } - if (exp > 1000) { return {{isPRP ? Task::PRP : Task::LL, u32(exp), AID, line, 0}}; } + if (exp > 1000) { return {{.kind=isPRP ? Task::PRP : Task::LL, .exponent=u32(exp), .AID=AID, .line=line, .squarings=0}}; } } if (isCERT) { vector parts = split(topParts.back(), ','); @@ -84,7 +84,7 @@ std::optional parse(const std::string& line) { u64 squarings{0}; from_chars(s.c_str(), end, squarings, 10); //printf ("Exec cert %d %d \n", (int) exp, (int) squarings); - if (exp > 1000 && squarings > 100) { return {{Task::CERT, u32(exp), AID, line, u32(squarings) }}; } + if (exp > 1000 && squarings > 100) { return {{.kind=Task::CERT, .exponent=u32(exp), .AID=AID, .line=line, .squarings=u32(squarings) }}; } } } } @@ -110,7 +110,7 @@ string workName(i32 instance) { return "worktodo-" + to_string(instance) + ".txt optional getWork(Args& args, i32 instance) { string filename = workName(instance); // Used for printf statements. Using fd::path is problematic because it 8-bit char in Linux and 16-bit char in Windows. - fs::path localWork = filename; + fs::path const localWork = filename; // Try to get a task from the local worktodo- file. if (optional task = bestTask(localWork, args.smallest)) { return task; } @@ -118,7 +118,7 @@ optional getWork(Args& args, i32 instance) { if (args.masterDir.empty()) { log("No work to do found. Add work to %s.\n", filename.c_str()); return {}; } filename = "worktodo.txt"; - fs::path worktodo = args.masterDir / filename; + fs::path const worktodo = args.masterDir / filename; /* We need to aquire a task from the global worktodo.txt, and "atomically" @@ -139,13 +139,13 @@ optional getWork(Args& args, i32 instance) { */ for (int retry = 0; retry < 2; ++retry) { - u64 initialSize = fileSize(worktodo); + u64 const initialSize = fileSize(worktodo); if (!initialSize) { return {}; } optional task = bestTask(worktodo, args.smallest); if (!task) { return {}; } - string workLine = task->line; + string const workLine = task->line; File::append(localWork, workLine); if (deleteLine(worktodo, workLine, initialSize)) { @@ -153,7 +153,7 @@ optional getWork(Args& args, i32 instance) { } // Undo add to local worktodo. Attempt twice. - bool found = deleteLine(localWork, workLine) || deleteLine(localWork, workLine); + bool const found = deleteLine(localWork, workLine) || deleteLine(localWork, workLine); assert(found); if (!found) { return {}; } } @@ -169,14 +169,14 @@ optional getWork(Args& args, i32 instance) { std::optional Worktodo::getTask(Args &args, i32 instance) { if (instance == 0) { if (args.prpExp) { - u64 exp = args.prpExp; + u64 const exp = args.prpExp; args.prpExp = 0; - return Task{Task::PRP, exp}; - } else if (args.llExp) { - u64 exp = args.llExp; + return Task{.kind=Task::PRP, .exponent=exp}; + } if (args.llExp) { + u64 const exp = args.llExp; args.llExp = 0; - return Task{Task::LL, exp}; - } else if (!args.verifyPath.empty()) { + return Task{.kind=Task::LL, .exponent=exp}; + } if (!args.verifyPath.empty()) { auto path = args.verifyPath; args.verifyPath.clear(); return Task{.kind=Task::VERIFY, .verifyPath=path}; diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 1152a7bb..99a7e6de 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -1,6 +1,5 @@ // Copyright (C) 2017-2024 Mihai Preda. -#include "timeutil.h" #include "File.h" #include "clwrap.h" @@ -16,7 +15,7 @@ using namespace std; // starting at 0 to -70 -array ERR_MES = { +static array ERR_MES = { "SUCCESS", "DEVICE_NOT_FOUND", "DEVICE_NOT_AVAILABLE", "COMPILER_NOT_AVAILABLE", "MEM_OBJECT_ALLOCATION_FAILURE", "OUT_OF_RESOURCES", "OUT_OF_HOST_MEMORY", "PROFILING_INFO_NOT_AVAILABLE", "MEM_COPY_OVERLAP", "IMAGE_FORMAT_MISMATCH", "IMAGE_FORMAT_NOT_SUPPORTED", "BUILD_PROGRAM_FAILURE", @@ -37,8 +36,8 @@ array ERR_MES = { }; string errMes(int err) { - string nb = " ("s + to_string(err) + ")"; - string mes = (err <= 0 && err >= -70) ? ERR_MES[-err] : + string const nb = " ("s + to_string(err) + ")"; + string const mes = (err <= 0 && err >= -70) ? ERR_MES[-err] : (err == -1001) ? "ICD_NOT_FOUND" : ""s; return mes + nb; } @@ -62,10 +61,10 @@ void check(int err, const char *file, int line, const char *func, string_view me } static void getInfo_(cl_device_id id, int what, size_t bufSize, void *buf, string_view whatStr) { - CHECK2(clGetDeviceInfo(id, what, bufSize, buf, NULL), whatStr); + CHECK2(clGetDeviceInfo(id, what, bufSize, buf, nullptr), whatStr); } -#define GET_INFO(id, what, where) getInfo_(id, what, sizeof(where), &where, #what) +#define GET_INFO(id, what, where) getInfo_(id, what, sizeof(where), &(where), #what) string getBdfFromDevice(cl_device_id id) { @@ -215,7 +214,7 @@ cl_device_id getDevice(u32 argsDeviceId) { cl_context createContext(cl_device_id id) { int err; - cl_context context = clCreateContext(NULL, 1, &id, NULL, NULL, &err); + cl_context context = clCreateContext(nullptr, 1, &id, nullptr, nullptr, &err); CHECK2(err, "clCreateContext"); return context; } @@ -230,7 +229,7 @@ void release(cl_event event) { CHECK1(clReleaseEvent(event)); } Program loadSource(cl_context context, const string &source) { const char *ptr = source.c_str(); - size_t size = source.size(); + size_t const size = source.size(); int err = 0; cl_program program = clCreateProgramWithSource(context, 1, &ptr, &size, &err); CHECK2(err, "clCreateProgramWithSource"); @@ -248,7 +247,7 @@ string getBuildLog(cl_program program, cl_device_id deviceId) { log("getBuildLog: log size is %lu bytes, not showing\n", (unsigned long) logSize); return {}; } - std::unique_ptr buf(new char[logSize + 1]); + std::unique_ptr const buf(new char[logSize + 1]); err = clGetProgramBuildInfo(program, deviceId, CL_PROGRAM_BUILD_LOG, logSize, buf.get(), &logSize); CHECK2(err, "clGetProgramBuildInfo"); buf.get()[logSize] = 0; @@ -260,28 +259,28 @@ string getBuildLog(cl_program program, cl_device_id deviceId) { Program loadBinary(cl_context context, cl_device_id id, string_view fileName) { File f = File::openRead(fileName); if (!f) { return {}; } - string bytes = f.readAll(); - size_t size = bytes.size(); - const unsigned char *ptr = reinterpret_cast(bytes.c_str()); + string const bytes = f.readAll(); + size_t const size = bytes.size(); + const auto *ptr = reinterpret_cast(bytes.c_str()); int err = 0; - cl_program program = clCreateProgramWithBinary(context, 1, &id, &size, &ptr, NULL, &err); + cl_program program = clCreateProgramWithBinary(context, 1, &id, &size, &ptr, nullptr, &err); if (err) { log("Load binary %s : %s\n", string(fileName).c_str(), errMes(err).c_str()); return {}; } - if ((err = clBuildProgram(program, 1, &id, NULL, NULL, NULL))) { + if ((err = clBuildProgram(program, 1, &id, nullptr, nullptr, nullptr))) { log("Build binary %s : %s\n", string(fileName).c_str(), errMes(err).c_str()); return {}; } return Program{program}; } -string getBinary(cl_program program) { +static string getBinary(cl_program program) { size_t size; - CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARY_SIZES, sizeof(size), &size, NULL)); + CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARY_SIZES, sizeof(size), &size, nullptr)); auto buf = make_unique(size + 1); - char *ptr = buf.get(); - CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARIES, sizeof(&buf), &ptr, NULL)); + char const*ptr = buf.get(); + CHECK1(clGetProgramInfo(program, CL_PROGRAM_BINARIES, sizeof(&buf), &ptr, nullptr)); return {buf.get(), size}; } @@ -330,27 +329,27 @@ EventHolder run(cl_queue queue, cl_kernel kernel, vector&& waits, const string &name, bool genEvent) { cl_event event{}; - CHECK2(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &workSize, &groupSize, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr), - name.c_str()); + CHECK2(clEnqueueNDRangeKernel(queue, kernel, 1, nullptr, &workSize, &groupSize, + waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr), + name); return genEvent ? EventHolder{event} : EventHolder{}; } EventHolder read(cl_queue queue, vector&& waits, bool blocking, cl_mem buf, size_t size, void *data, bool genEvent) { - size_t start = 0; + size_t const start = 0; cl_event event{}; CHECK1(clEnqueueReadBuffer(queue, buf, blocking, start, size, data, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } EventHolder write(cl_queue queue, vector&& waits, bool blocking, cl_mem buf, size_t size, const void *data, bool genEvent) { - size_t start = 0; + size_t const start = 0; cl_event event{}; CHECK1(clEnqueueWriteBuffer(queue, buf, blocking, start, size, data, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -358,7 +357,7 @@ EventHolder copyBuf(cl_queue queue, vector&& waits, const cl_mem src, cl_mem dst, size_t size, bool genEvent) { cl_event event{}; CHECK1(clEnqueueCopyBuffer(queue, src, dst, 0, 0, size, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -367,19 +366,19 @@ EventHolder fillBuf(cl_queue q, vector&& waits, assert(size); cl_event event{}; CHECK1(clEnqueueFillBuffer(q, buf, pat, patSize, 0 /*start*/, size, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr)); + waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } EventHolder enqueueMarker(cl_queue q) { cl_event event{}; - CHECK1(clEnqueueMarkerWithWaitList(q, 0, 0, &event)); + CHECK1(clEnqueueMarkerWithWaitList(q, 0, nullptr, &event)); return EventHolder{event}; } EventHolder enqueueMarkerWithWaits(cl_queue q, vector&& waits) { cl_event event{}; - CHECK1(clEnqueueMarkerWithWaitList(q, waits.size(), waits.empty() ? 0 : waits.data(), &event)); + CHECK1(clEnqueueMarkerWithWaitList(q, waits.size(), waits.empty() ? nullptr : waits.data(), &event)); return EventHolder{event}; } @@ -391,13 +390,13 @@ void waitForEvents(vector&& waits) { int getKernelNumArgs(cl_kernel k) { int nArgs = 0; - CHECK1(clGetKernelInfo(k, CL_KERNEL_NUM_ARGS, sizeof(nArgs), &nArgs, NULL)); + CHECK1(clGetKernelInfo(k, CL_KERNEL_NUM_ARGS, sizeof(nArgs), &nArgs, nullptr)); return nArgs; } int getWorkGroupSize(cl_kernel k, cl_device_id device, const char *name) { size_t size[3]; - CHECK2(clGetKernelWorkGroupInfo(k, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, sizeof(size), &size, NULL), name); + CHECK2(clGetKernelWorkGroupInfo(k, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, sizeof(size), &size, nullptr), name); return size[0]; } @@ -412,7 +411,7 @@ std::string getKernelArgName(cl_kernel k, int pos) { u32 getEventInfo(cl_event event) { u32 status = -1; - CHECK1(clGetEventInfo(event, CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(status), &status, 0)); + CHECK1(clGetEventInfo(event, CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(status), &status, nullptr)); return status; } @@ -433,7 +432,7 @@ array getEventNanos(cl_event event) { for (int i = 0; i < 4; ++i) { u64 t{}; - CHECK1(clGetEventProfilingInfo(event, what[i], sizeof(t), &t, 0)); + CHECK1(clGetEventProfilingInfo(event, what[i], sizeof(t), &t, nullptr)); if (i) { ret[i - 1] = delta(prev, t); } prev = t; } @@ -442,12 +441,12 @@ array getEventNanos(cl_event event) { cl_context getQueueContext(cl_command_queue q) { cl_context ret; - CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ret, 0)); + CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_CONTEXT, sizeof(cl_context), &ret, nullptr)); return ret; } -cl_device_id getQueueDevice(cl_command_queue q) { +static cl_device_id getQueueDevice(cl_command_queue q) { cl_device_id id; - CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_DEVICE, sizeof(id), &id, 0)); + CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_DEVICE, sizeof(id), &id, nullptr)); return id; } diff --git a/src/clwrap.h b/src/clwrap.h index 41b2bb93..917c03d0 100644 --- a/src/clwrap.h +++ b/src/clwrap.h @@ -84,7 +84,7 @@ void saveBinary(cl_program program, string_view fileName); template void setArg(cl_kernel k, int pos, const T &value, const string& name) { - CHECK2(clSetKernelArg(k, pos, sizeof(value), &value), (name + '[' + to_string(pos) + "] size " + to_string(sizeof(value))).c_str()); + CHECK2(clSetKernelArg(k, pos, sizeof(value), &value), name + '[' + to_string(pos) + "] size " + to_string(sizeof(value))); } /* @@ -92,7 +92,7 @@ template<> void setArg(cl_kernel k, int pos, const int &value, const string& name); */ -cl_mem makeBuf_(cl_context context, unsigned kind, size_t size, const void *ptr = 0); +cl_mem makeBuf_(cl_context context, unsigned kind, size_t size, const void *ptr = nullptr); cl_queue makeQueue(cl_device_id d, cl_context c, bool enableProfile); void flush( cl_queue q); diff --git a/src/common.cpp b/src/common.cpp index c3a1ef80..29409bdd 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -1,16 +1,10 @@ // GpuOwl Mersenne primality tester; Copyright (C) 2017-2018 Mihai Preda. #include "common.h" -#include "File.h" -#include "timeutil.h" -#include #include -#include -#include #include #include -#include string hex(u64 x) { ostringstream out{}; @@ -24,7 +18,7 @@ std::string rstripNewline(std::string s) { } u32 crc32(const void *data, size_t size) { - u32 tab[16] = { + u32 const tab[16] = { 0x00000000, 0x1DB71064, 0x3B6E20C8, 0x26D930AC, 0x76DC4190, 0x6B6B51F4, 0x4DB26158, 0x5005713C, 0xEDB88320, 0xF00F9344, 0xD6D6A3E8, 0xCB61B38C, @@ -41,11 +35,11 @@ u32 crc32(const void *data, size_t size) { string formatBound(u32 b) { if (b >= 1'000'000 && b % 1'000'000 == 0) { return to_string(b / 1'000'000) + 'M'; - } else if (b >= 500'000 && b % 100'000 == 0) { + } if (b >= 500'000 && b % 100'000 == 0) { char buf[32]; snprintf(buf, sizeof(buf), "%.1fM", float(b) / 1'000'000); return buf; - } else { + } return to_string(b); - } + } diff --git a/src/common.h b/src/common.h index 9bd88471..3da39734 100644 --- a/src/common.h +++ b/src/common.h @@ -33,7 +33,7 @@ namespace fs = std::filesystem; // When using multiple primes in an NTT the size of an integer FFT "word" can be 64 bits. Original FP64 FFT needs only 32 bits. // C code will use i64 integer data. The code that reads and writes GPU buffers will downsize the integers to 32 bits when required. -typedef i64 Word; +using Word = i64; // Create datatype names that mimic the ones used in OpenCL code using double2 = pair; diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 5df25791..f1026af5 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -8,14 +8,13 @@ #include #include #include +#include #include +#include #include #include #include #include -#include -#include -#include #include #ifdef __linux__ #include @@ -41,7 +40,7 @@ static void ensureContextCurrent() { static bool g_cudaInitialized = false; static void ensureCudaInit() { if (!g_cudaInitialized) { - CUresult err = cuInit(0); + CUresult const err = cuInit(0); if (err != CUDA_SUCCESS) { fprintf(stderr, "cuInit failed: %d\n", (int)err); } @@ -78,7 +77,7 @@ unsigned clGetPlatformIDs(unsigned num, cl_platform_id* platforms, unsigned* num int clGetDeviceIDs(cl_platform_id, cl_device_type, unsigned num, cl_device_id* devices, unsigned* numRet) { enumerateDevices(); - unsigned n = g_devices.size(); + unsigned const n = g_devices.size(); if (numRet) *numRet = n; if (devices) { for (unsigned i = 0; i < min(num, n); i++) { @@ -97,7 +96,7 @@ cl_context clCreateContext(const intptr_t*, unsigned nDevices, const cl_device_i CUctxCreateParams params{}; CUresult r = cuCtxCreate_v4(&ctx->ctx, ¶ms, 0, ctx->dev); #else - CUresult r = cuCtxCreate(&ctx->ctx, 0, ctx->dev); + CUresult const r = cuCtxCreate(&ctx->ctx, 0, ctx->dev); #endif if (r != CUDA_SUCCESS) { delete ctx; @@ -145,7 +144,7 @@ int clReleaseCommandQueue(cl_command_queue q) { // ---- Program compilation (NVRTC) ---- -cl_program clCreateProgramWithSource(cl_context ctx, unsigned count, const char** strings, +cl_program clCreateProgramWithSource(cl_context /*ctx*/, unsigned count, const char** strings, const size_t* lengths, int* err) { auto* prog = new _cl_program; for (unsigned i = 0; i < count; i++) { @@ -159,7 +158,7 @@ cl_program clCreateProgramWithSource(cl_context ctx, unsigned count, const char* return prog; } -cl_program clCreateProgramWithBinary(cl_context ctx, unsigned nDevices, const cl_device_id*, +cl_program clCreateProgramWithBinary(cl_context /*ctx*/, unsigned /*nDevices*/, const cl_device_id*, const size_t* lengths, const unsigned char** binaries, int* binaryStatus, int* err) { // "Binary" in CUDA land = PTX string @@ -169,7 +168,7 @@ cl_program clCreateProgramWithBinary(cl_context ctx, unsigned nDevices, const cl prog->compiled = true; // Load the module (JIT-compile PTX to SASS) ensureContextCurrent(); - CUresult r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); + CUresult const r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); if (r == CUDA_SUCCESS) { prog->moduleLoaded = true; if (binaryStatus) binaryStatus[0] = CL_SUCCESS; @@ -187,13 +186,13 @@ cl_program clCreateProgramWithBinary(cl_context ctx, unsigned nDevices, const cl // Build log storage (per-program) static string g_lastBuildLog; -int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* devices, const char* options, +int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id* devices, const char* options, unsigned numHeaders, const cl_program* headers, const char* const* headerNames, void (*)(cl_program, void*), void*) { if (!prog) return CL_INVALID_PROGRAM; // Get device arch for NVRTC - CUdevice dev = devices ? devices[0]->dev : g_devices[0].dev; + CUdevice const dev = devices ? devices[0]->dev : g_devices[0].dev; int major = 0, minor = 0; cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev); cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev); @@ -207,17 +206,17 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev // -cl-finite-math-only → --fmad=true (enable FMA contraction, the safe subset) // -Dfoo=bar → -Dfoo=bar (pass through) vector nvrtcOpts; - nvrtcOpts.push_back(archOpt); - nvrtcOpts.push_back("-default-device"); - nvrtcOpts.push_back("-std=c++17"); - nvrtcOpts.push_back("-w"); // Suppress NVRTC macro redefinition warnings + nvrtcOpts.emplace_back(archOpt); + nvrtcOpts.emplace_back("-default-device"); + nvrtcOpts.emplace_back("-std=c++17"); + nvrtcOpts.emplace_back("-w"); // Suppress NVRTC macro redefinition warnings // FMA contraction: OpenCL uses -cl-finite-math-only + #pragma OPENCL FP_CONTRACT ON // to allow the compiler to contract a*b+c into FMA instructions. NVRTC's --fmad=true // is the safe equivalent — it ONLY enables FMA contraction without the dangerous // parts of -use_fast_math (no flush-to-zero, no reduced-precision division/sqrt). // This is critical for FFT performance: every butterfly is multiply-add pairs. - nvrtcOpts.push_back("--fmad=true"); + nvrtcOpts.emplace_back("--fmad=true"); // NOTE: --restrict (all kernel pointers are __restrict__) was tested but causes GPU read // errors — some PRPLL kernels use in-place operations where in/out buffers alias. @@ -240,22 +239,22 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev istringstream iss(options); string tok; while (iss >> tok) { - if (tok.substr(0, 2) == "-D") { + if (tok.starts_with("-D")) { // Fix AMD-only FFT variants for NVIDIA: variant_W=0 and variant_H=0 require // AMD builtins (__builtin_amdgcn_ds_bpermute etc). Replace with variant 2. // FFT_VARIANT is a 3-digit number WMH: e.g. 000, 101, 202 if (tok.find("FFT_VARIANT=") != string::npos) { - size_t eqPos = tok.find('='); + size_t const eqPos = tok.find('='); string valStr = tok.substr(eqPos + 1); // Strip trailing 'u' suffix if (!valStr.empty() && valStr.back() == 'u') valStr.pop_back(); - int val = atoi(valStr.c_str()); + int const val = atoi(valStr.c_str()); int vW = val / 100; - int vM = (val % 100) / 10; + int const vM = (val % 100) / 10; int vH = val % 10; if (vW == 0) vW = 2; // AMD BCAST → NVIDIA generic if (vH == 0) vH = 2; - int newVal = vW * 100 + vM * 10 + vH; + int const newVal = vW * 100 + vM * 10 + vH; tok = "-DFFT_VARIANT=" + to_string(newVal) + "u"; } nvrtcOpts.push_back(tok); @@ -263,7 +262,7 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev // FMA contraction already enabled above via --fmad=true. // Do NOT use -use_fast_math here — it enables flush-to-zero and // reduced-precision division/sqrt which breaks tailMul accuracy. - } else if (tok.substr(0, 14) == "--maxrregcount") { + } else if (tok.starts_with("--maxrregcount")) { nvrtcOpts.push_back(tok); maxregcount = atoi(tok.substr(15, 3).c_str()); } @@ -278,12 +277,12 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev for (unsigned i = 0; i < numHeaders; i++) { // First header: opencl_compat.cuh (inject as virtual NVRTC header) if (i == 0) { - nvrtcHeaders.push_back({"opencl_compat.cuh", headers[0]->source}); + nvrtcHeaders.emplace_back("opencl_compat.cuh", headers[0]->source); } // Remaining headers need preprocessing else if (headers[i] && headerNames[i]) { // Preprocess OpenCL source for CUDA compatibility - string processedSrc = NvrtcProgram::preprocessOpenCL(headers[i]->source); + string const processedSrc = NvrtcProgram::preprocessOpenCL(headers[i]->source); // Debug: verify KERNEL macro replacement // I'm not sure what Sherpa was trying to print out here. It prints out nothing useful. // { @@ -296,7 +295,7 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev // } // } // } - nvrtcHeaders.push_back({headerNames[i], processedSrc}); + nvrtcHeaders.emplace_back(headerNames[i], processedSrc); } } @@ -371,10 +370,10 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev // If --maxrregcount is set it seems nvrtc compile ignores the setting. Instead modify the PTX and load the modified PTX. if (maxregcount) { - string maxntidPattern = ".maxntid "; - string maxnregPattern = ".maxnreg " + to_string(maxregcount) + "\n"; + string const maxntidPattern = ".maxntid "; + string const maxnregPattern = ".maxnreg " + to_string(maxregcount) + "\n"; for (size_t startpos = 0; ; ) { - size_t pos = prog->ptx.find(maxntidPattern, startpos); + size_t const pos = prog->ptx.find(maxntidPattern, startpos); if (pos == string::npos) break; prog->ptx.insert(pos, maxnregPattern); startpos = pos + 20; @@ -384,8 +383,8 @@ int clCompileProgram(cl_program prog, unsigned nDevices, const cl_device_id* dev return CL_SUCCESS; } -cl_program clLinkProgram(cl_context ctx, unsigned nDevices, const cl_device_id*, - const char* options, unsigned nProgs, const cl_program* progs, +cl_program clLinkProgram(cl_context /*ctx*/, unsigned /*nDevices*/, const cl_device_id*, + const char* /*options*/, unsigned nProgs, const cl_program* progs, void (*)(cl_program, void*), void*, int* err) { ensureContextCurrent(); // In CUDA, compilation produces PTX directly — no separate link step needed. @@ -417,7 +416,7 @@ cl_program clLinkProgram(cl_context ctx, unsigned nDevices, const cl_device_id*, (void*)(size_t)sizeof(jitErrorLog), (void*)jitErrorLog, (void*)(size_t)sizeof(jitInfoLog), (void*)jitInfoLog }; - CUresult r = cuModuleLoadDataEx(&linked->module, linked->ptx.c_str(), 4, jitOpts, jitOptVals); + CUresult const r = cuModuleLoadDataEx(&linked->module, linked->ptx.c_str(), 4, jitOpts, jitOptVals); if (r != CUDA_SUCCESS) { const char* errName = nullptr; cuGetErrorName(r, &errName); @@ -471,19 +470,19 @@ int clBuildProgram(cl_program prog, unsigned nDevices, const cl_device_id* devic } // clBuildProgram = compile + link in one step - int err = clCompileProgram(prog, nDevices, devices, options, 0, nullptr, nullptr, nullptr, nullptr); + int const err = clCompileProgram(prog, nDevices, devices, options, 0, nullptr, nullptr, nullptr, nullptr); if (err != CL_SUCCESS) return err; - CUresult r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); + CUresult const r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); if (r != CUDA_SUCCESS) return CL_BUILD_PROGRAM_FAILURE; prog->moduleLoaded = true; return CL_SUCCESS; } -int clGetProgramBuildInfo(cl_program prog, cl_device_id, cl_program_build_info info, +int clGetProgramBuildInfo(cl_program /*prog*/, cl_device_id, cl_program_build_info info, size_t size, void* value, size_t* sizeRet) { if (info == CL_PROGRAM_BUILD_LOG) { - size_t len = g_lastBuildLog.size() + 1; + size_t const len = g_lastBuildLog.size() + 1; if (sizeRet) *sizeRet = len; if (value && size >= len) { memcpy(value, g_lastBuildLog.c_str(), len); @@ -501,7 +500,7 @@ int clGetProgramInfo(cl_program prog, cl_program_info info, size_t size, void* v } else if (info == CL_PROGRAM_BINARIES) { if (sizeRet) *sizeRet = sizeof(unsigned char*); if (value && size >= sizeof(unsigned char*)) { - unsigned char** ptrs = (unsigned char**)value; + auto* const* ptrs = (unsigned char**)value; if (ptrs[0]) memcpy(ptrs[0], prog->ptx.data(), prog->ptx.size()); } } @@ -519,7 +518,7 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { auto* k = new _cl_kernel; k->name = name; k->parentModule = prog->module; - CUresult r = cuModuleGetFunction(&k->func, prog->module, name); + CUresult const r = cuModuleGetFunction(&k->func, prog->module, name); if (r != CUDA_SUCCESS) { fprintf(stderr, "cuModuleGetFunction('%s') failed: %d, moduleLoaded=%d, module=%p\n", name, (int)r, prog->moduleLoaded, (void*)prog->module); @@ -553,16 +552,16 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { k->reqWorkGroupSize = 256; // fallback { const string& ptx = prog->ptx; - string entryPattern = ".entry " + string(name) + "("; - size_t pos = ptx.find(entryPattern); + string const entryPattern = ".entry " + string(name) + "("; + size_t const pos = ptx.find(entryPattern); if (pos != string::npos) { // Found the kernel entry. Now find .maxntid before the next .entry or opening brace size_t searchEnd = ptx.find(".entry ", pos + 1); if (searchEnd == string::npos) searchEnd = ptx.size(); - string maxntidPattern = ".maxntid "; - size_t mpos = ptx.find(maxntidPattern, pos); + string const maxntidPattern = ".maxntid "; + size_t const mpos = ptx.find(maxntidPattern, pos); if (mpos != string::npos && mpos < searchEnd) { - int val = atoi(ptx.c_str() + mpos + maxntidPattern.size()); + int const val = atoi(ptx.c_str() + mpos + maxntidPattern.size()); if (val > 0) { k->reqWorkGroupSize = val; } @@ -588,7 +587,7 @@ int clSetKernelArg(cl_kernel k, unsigned pos, size_t size, const void* value) { // We need to store the CUdeviceptr (GPU address) instead of the cl_mem (host pointer). if (size == sizeof(cl_mem) && value) { cl_mem mem = *(cl_mem*)value; - if (mem && g_allocatedBuffers.count(mem)) { + if (mem && g_allocatedBuffers.contains(mem)) { CUdeviceptr devPtr = mem->ptr; k->setArg(pos, sizeof(CUdeviceptr), &devPtr); return CL_SUCCESS; @@ -607,11 +606,11 @@ int clSetKernelArg(cl_kernel k, unsigned pos, size_t size, const void* value) { // ---- Buffer ---- -cl_mem clCreateBuffer(cl_context ctx, cl_mem_flags flags, size_t size, void* hostPtr, int* err) { +cl_mem clCreateBuffer(cl_context /*ctx*/, cl_mem_flags flags, size_t size, void* hostPtr, int* err) { auto* buf = new _cl_mem; buf->size = size; ensureContextCurrent(); - CUresult r = cuMemAlloc(&buf->ptr, size); + CUresult const r = cuMemAlloc(&buf->ptr, size); if (r != CUDA_SUCCESS) { delete buf; if (err) *err = CL_MEM_OBJECT_ALLOCATION_FAILURE; @@ -638,7 +637,7 @@ int clReleaseMemObject(cl_mem buf) { // ---- Command Queue ---- -cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id dev, +cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id /*dev*/, const cl_queue_properties* props, int* err) { auto* q = new _cl_command_queue; q->context = ctx; @@ -655,7 +654,7 @@ cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id // Make sure context is current cuCtxSetCurrent(ctx->ctx); - CUresult r = cuStreamCreate(&q->stream, CU_STREAM_NON_BLOCKING); + CUresult const r = cuStreamCreate(&q->stream, CU_STREAM_NON_BLOCKING); if (r != CUDA_SUCCESS) { delete q; if (err) *err = CL_OUT_OF_RESOURCES; @@ -667,23 +666,23 @@ cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id // ---- Enqueue operations ---- -int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, - const size_t* globalOffset, const size_t* globalSize, - const size_t* localSize, unsigned nWaits, - const cl_event* waits, cl_event* event) { +int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned /*workDim*/, + const size_t* /*globalOffset*/, const size_t* globalSize, + const size_t* localSize, unsigned /*nWaits*/, + const cl_event* /*waits*/, cl_event* event) { if (!q || !k) return CL_INVALID_VALUE; ensureContextCurrent(); - size_t gs = globalSize[0]; - size_t ls = localSize ? localSize[0] : 256; - size_t numBlocks = (gs + ls - 1) / ls; + size_t const gs = globalSize[0]; + size_t const ls = localSize ? localSize[0] : 256; + size_t const numBlocks = (gs + ls - 1) / ls; // Build args array void* argPtrs[_cl_kernel::MAX_ARGS]; k->buildArgPointers(argPtrs); // Env-gated kernel profiling (PRPLL_PROFILE=1) — takes priority over event profiling - static bool doProfile = (getenv("PRPLL_PROFILE") != nullptr); + static bool const doProfile = (getenv("PRPLL_PROFILE") != nullptr); // Event handling (skipped when env profiler is active) if (!doProfile && event && q->profiling) { @@ -693,7 +692,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, ev->hasTimings = true; ev->commandType = CL_COMMAND_NDRANGE_KERNEL; cuEventRecord(ev->start, q->stream); - CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + CUresult const r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); cuEventRecord(ev->end, q->stream); *event = ev; return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; @@ -708,14 +707,14 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, if (!pStart) { cuEventCreate(&pStart, CU_EVENT_DEFAULT); cuEventCreate(&pEnd, CU_EVENT_DEFAULT); } cuEventRecord(pStart, q->stream); - CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + CUresult const r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); cuEventRecord(pEnd, q->stream); cuEventSynchronize(pEnd); float ms = 0; cuEventElapsedTime(&ms, pStart, pEnd); kTime[k->name] += ms; kCount[k->name]++; - if (!kRegs.count(k->name)) { + if (!kRegs.contains(k->name)) { int regs = 0, shmem = 0; cuFuncGetAttribute(®s, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); @@ -728,7 +727,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, fprintf(stderr, "\n=== NTT KERNEL PROFILE (%d launches) ===\n", totalLaunches); std::vector> sorted; double totalMs = 0; - for (auto& [n, t] : kTime) { sorted.push_back({t, n}); totalMs += t; } + for (auto& [n, t] : kTime) { sorted.emplace_back(t, n); totalMs += t; } std::sort(sorted.rbegin(), sorted.rend()); for (auto& [t, n] : sorted) { fprintf(stderr, " %6.1f ms (%5.1f%%) %5d calls avg %.3f ms regs=%d shmem=%d %s\n", @@ -740,7 +739,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; } - CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + CUresult const r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); if (r != CUDA_SUCCESS) { const char* errName = nullptr; cuGetErrorName(r, &errName); @@ -753,7 +752,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, int clEnqueueReadBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, size_t offset, size_t size, void* ptr, - unsigned nWaits, const cl_event* waits, cl_event* event) { + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { // Must use stream-ordered copy because the stream was created with CU_STREAM_NON_BLOCKING, // which means cuMemcpyDtoH (NULL stream) won't wait for pending kernels on this stream. CUresult r = cuMemcpyDtoHAsync(ptr, buf->ptr + offset, size, q->stream); @@ -766,7 +765,7 @@ int clEnqueueReadBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, int clEnqueueWriteBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, size_t offset, size_t size, const void* ptr, - unsigned nWaits, const cl_event* waits, cl_event* event) { + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { // Must use stream-ordered copy (same reason as clEnqueueReadBuffer above) CUresult r = cuMemcpyHtoDAsync(buf->ptr + offset, ptr, size, q->stream); if (r == CUDA_SUCCESS && blocking) { @@ -778,15 +777,15 @@ int clEnqueueWriteBuffer(cl_command_queue q, cl_mem buf, cl_bool blocking, int clEnqueueCopyBuffer(cl_command_queue q, cl_mem src, cl_mem dst, size_t srcOffset, size_t dstOffset, size_t size, - unsigned nWaits, const cl_event* waits, cl_event* event) { - CUresult r = cuMemcpyDtoDAsync(dst->ptr + dstOffset, src->ptr + srcOffset, size, q->stream); + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { + CUresult const r = cuMemcpyDtoDAsync(dst->ptr + dstOffset, src->ptr + srcOffset, size, q->stream); if (event) *event = nullptr; return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; } int clEnqueueFillBuffer(cl_command_queue q, cl_mem buf, const void* pattern, size_t patternSize, size_t offset, size_t size, - unsigned nWaits, const cl_event* waits, cl_event* event) { + unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { CUresult r; if (patternSize == 1) { unsigned char val; @@ -820,7 +819,7 @@ int clEnqueueMarkerWithWaitList(cl_command_queue q, unsigned nWaits, const cl_ev return CL_SUCCESS; } -int clFlush(cl_command_queue q) { +int clFlush(cl_command_queue /*q*/) { // CUDA streams auto-flush; no-op return CL_SUCCESS; } @@ -851,7 +850,7 @@ int clGetEventInfo(cl_event ev, cl_event_info info, size_t size, void* value, si if (info == CL_EVENT_COMMAND_EXECUTION_STATUS) { int status = CL_COMPLETE; if (ev->end) { - CUresult r = cuEventQuery(ev->end); + CUresult const r = cuEventQuery(ev->end); if (r == CUDA_ERROR_NOT_READY) status = CL_RUNNING; } if (sizeRet) *sizeRet = sizeof(int); @@ -893,7 +892,7 @@ int clGetDeviceInfo(cl_device_id dev, cl_device_info info, size_t size, void* va case CL_DEVICE_NAME: { char name[256]; cuDeviceGetName(name, sizeof(name), dev->dev); - size_t len = strlen(name) + 1; + size_t const len = strlen(name) + 1; if (sizeRet) *sizeRet = len; if (value && size >= len) memcpy(value, name, len); break; @@ -935,7 +934,7 @@ int clGetDeviceInfo(cl_device_id dev, cl_device_info info, size_t size, void* va cuDriverGetVersion(&ver); char verStr[64]; snprintf(verStr, sizeof(verStr), "CUDA %d.%d", ver / 1000, (ver % 1000) / 10); - size_t len = strlen(verStr) + 1; + size_t const len = strlen(verStr) + 1; if (sizeRet) *sizeRet = len; if (value && size >= len) memcpy(value, verStr, len); break; @@ -992,7 +991,7 @@ int clGetDeviceInfo(cl_device_id dev, cl_device_info info, size_t size, void* va int clGetPlatformInfo(cl_platform_id, cl_device_info info, size_t size, void* value, size_t* sizeRet) { if (info == CL_PLATFORM_VERSION) { const char* ver = "CUDA (via PRPLL CUDA backend)"; - size_t len = strlen(ver) + 1; + size_t const len = strlen(ver) + 1; if (sizeRet) *sizeRet = len; if (value && size >= len) memcpy(value, ver, len); return CL_SUCCESS; @@ -1026,19 +1025,19 @@ int clGetKernelInfo(cl_kernel k, cl_kernel_info info, size_t size, void* value, return CL_SUCCESS; } -int clGetKernelArgInfo(cl_kernel k, unsigned pos, cl_kernel_arg_info info, +int clGetKernelArgInfo(cl_kernel /*k*/, unsigned pos, cl_kernel_arg_info info, size_t size, void* value, size_t* sizeRet) { if (info == CL_KERNEL_ARG_NAME) { char name[32]; snprintf(name, sizeof(name), "arg%u", pos); - size_t len = strlen(name) + 1; + size_t const len = strlen(name) + 1; if (sizeRet) *sizeRet = len; if (value && size >= len) memcpy(value, name, len); } return CL_SUCCESS; } -int clGetKernelWorkGroupInfo(cl_kernel k, cl_device_id dev, cl_kernel_work_group_info info, +int clGetKernelWorkGroupInfo(cl_kernel k, cl_device_id /*dev*/, cl_kernel_work_group_info info, size_t size, void* value, size_t* sizeRet) { if (!k) return CL_INVALID_KERNEL; ensureContextCurrent(); @@ -1048,7 +1047,7 @@ int clGetKernelWorkGroupInfo(cl_kernel k, cl_device_id dev, cl_kernel_work_group // Previously we used CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK which returns the hardware max // based on register/shared memory usage — NOT the declared group size. This caused wrong // block sizes for every kernel (e.g., tailMul expected 64 threads but got 1024). - int wgSize = k->reqWorkGroupSize > 0 ? k->reqWorkGroupSize : 256; + int const wgSize = k->reqWorkGroupSize > 0 ? k->reqWorkGroupSize : 256; size_t wgs[3] = { (size_t)wgSize, 1, 1 }; if (sizeRet) *sizeRet = sizeof(wgs); if (value && size >= sizeof(wgs)) memcpy(value, wgs, sizeof(wgs)); @@ -1071,7 +1070,7 @@ void clSVMFree(cl_context, void* ptr) { } int clSetKernelArgSVMPointer(cl_kernel k, unsigned pos, const void* ptr) { - CUdeviceptr dp = (CUdeviceptr)(uintptr_t)ptr; + auto dp = (CUdeviceptr)(uintptr_t)ptr; k->setArg(pos, sizeof(dp), &dp); return CL_SUCCESS; } @@ -1084,7 +1083,7 @@ int clSetKernelArgSVMPointer(cl_kernel k, unsigned pos, const void* ptr) { // Computes the minimum address span covering all buffers, then sets one access policy // window with hitRatio sized so that only the actual buffer bytes get persisting treatment, // not the gaps between non-contiguous allocations. -void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) { +static void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) { if (!q) return; // Find address span and total data size @@ -1094,21 +1093,21 @@ void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) for (auto buf : buffers) { if (!buf || buf->size == 0) continue; - CUdeviceptr lo = buf->ptr; - CUdeviceptr hi = buf->ptr + buf->size; - if (lo < minAddr) minAddr = lo; - if (hi > maxAddr) maxAddr = hi; + CUdeviceptr const lo = buf->ptr; + CUdeviceptr const hi = buf->ptr + buf->size; + minAddr = std::min(lo, minAddr); + maxAddr = std::max(hi, maxAddr); totalDataBytes += buf->size; } if (totalDataBytes == 0 || maxAddr <= minAddr) return; - size_t spanBytes = (size_t)(maxAddr - minAddr); + auto spanBytes = (size_t)(maxAddr - minAddr); // Query the device's max access policy window size int maxWindowSize = 0; cuDeviceGetAttribute(&maxWindowSize, CU_DEVICE_ATTRIBUTE_MAX_ACCESS_POLICY_WINDOW_SIZE, 0); - if (maxWindowSize > 0 && spanBytes > (size_t)maxWindowSize) { + if (maxWindowSize > 0 && std::cmp_greater(spanBytes, maxWindowSize)) { fprintf(stderr, "L2 persist: span %zuMB exceeds max window %dMB, clamping\n", spanBytes / (1024*1024), maxWindowSize / (1024*1024)); spanBytes = maxWindowSize; @@ -1117,7 +1116,7 @@ void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) // hitRatio = actual data / window span. This way only the real buffer data gets // persisting treatment, and any gaps between allocations get streaming treatment. float hitRatio = (float)totalDataBytes / (float)spanBytes; - if (hitRatio > 1.0f) hitRatio = 1.0f; + hitRatio = std::min(hitRatio, 1.0f); CUstreamAttrValue attr; memset(&attr, 0, sizeof(attr)); @@ -1127,7 +1126,7 @@ void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) attr.accessPolicyWindow.hitProp = CU_ACCESS_PROPERTY_PERSISTING; attr.accessPolicyWindow.missProp = CU_ACCESS_PROPERTY_STREAMING; - CUresult r = cuStreamSetAttribute(q->stream, CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW, &attr); + CUresult const r = cuStreamSetAttribute(q->stream, CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW, &attr); if (r != CUDA_SUCCESS) { fprintf(stderr, "L2 persist: cuStreamSetAttribute failed (%d)\n", (int)r); } else { diff --git a/src/cuda/cudawrap.cpp b/src/cuda/cudawrap.cpp index 474d7522..f4b4cb6a 100644 --- a/src/cuda/cudawrap.cpp +++ b/src/cuda/cudawrap.cpp @@ -3,7 +3,6 @@ #include "cudawrap.h" #include -#include #include #include #include @@ -76,7 +75,7 @@ float getGpuRamGB(CUdevice dev) { return bytes / (1024.0f * 1024.0f * 1024.0f); } -u64 getFreeMem(CUdevice dev) { +u64 getFreeMem(CUdevice /*dev*/) { // Need a context to query free memory size_t free_bytes = 0, total = 0; CU_CHECK(cuMemGetInfo(&free_bytes, &total)); @@ -119,7 +118,7 @@ CudaModule::CudaModule(const std::string& ptx, const std::string& name) { char errorLog[4096] = {}; void* optionValues[] = { (void*)(size_t)sizeof(errorLog), (void*)errorLog }; - CUresult err = cuModuleLoadDataEx(&module, ptx.c_str(), 2, options, optionValues); + CUresult const err = cuModuleLoadDataEx(&module, ptx.c_str(), 2, options, optionValues); if (err != CUDA_SUCCESS) { cuda_log("Module load error for '%s': %s\n", name.c_str(), errorLog); checkCuda(err, __FILE__, __LINE__, __func__, "cuModuleLoadDataEx"); @@ -226,7 +225,7 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { while (i < source.size()) { // Strip #pragma OPENCL ... lines if (i + 14 <= source.size() && source.compare(i, 14, "#pragma OPENCL") == 0) { - bool atLineStart = (i == 0 || source[i-1] == '\n'); + bool const atLineStart = (i == 0 || source[i-1] == '\n'); if (atLineStart) { while (i < source.size() && source[i] != '\n') i++; result += "// [stripped pragma]"; @@ -238,7 +237,7 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { // OpenCL: #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void // CUDA: #define KERNEL(x) extern "C" __global__ void __launch_bounds__(x) if (i + 15 <= source.size() && source.compare(i, 15, "#define KERNEL(") == 0) { - bool atLineStart = (i == 0 || source[i-1] == '\n'); + bool const atLineStart = (i == 0 || source[i-1] == '\n'); if (atLineStart) { while (i < source.size() && source[i] != '\n') i++; result += "#ifdef CUDA_MIN_BLOCKS\n"; @@ -254,7 +253,7 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { // Only strip exact OpenCL patterns (using OpenCL types like 'long', 'ulong', 'uint') // NOT our compat header's versions (which use 'long long', 'unsigned long long', etc.) if (i + 7 <= source.size() && source.compare(i, 7, "typedef") == 0) { - bool atLineStart = (i == 0 || source[i-1] == '\n'); + bool const atLineStart = (i == 0 || source[i-1] == '\n'); if (atLineStart) { size_t lineEnd = source.find('\n', i); if (lineEnd == std::string::npos) lineEnd = source.size(); @@ -289,7 +288,7 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { // Handle __attribute__((...)) for overloadable and reqd_work_group_size if (i + 15 <= source.size() && source.compare(i, 15, "__attribute__((") == 0) { - size_t nameStart = i + 15; + size_t const nameStart = i + 15; if (nameStart + 12 <= source.size() && source.compare(nameStart, 12, "overloadable") == 0) { // Strip __attribute__((overloadable)) — C++ has native overloading i = skipBalancedParens(source, i + 13); @@ -304,7 +303,7 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { size_t numEnd = argsStart; while (numEnd < source.size() && source[numEnd] >= '0' && source[numEnd] <= '9') numEnd++; if (numEnd > argsStart) { - std::string wgSize = source.substr(argsStart, numEnd - argsStart); + std::string const wgSize = source.substr(argsStart, numEnd - argsStart); result += "__launch_bounds__(" + wgSize + ") "; } // Skip past the entire __attribute__((...)) @@ -325,7 +324,7 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { }; bool matched = false; for (int vi = 0; vecTypes[vi]; vi++) { - size_t tlen = strlen(vecTypes[vi]); + size_t const tlen = strlen(vecTypes[vi]); if (i + 1 + tlen <= source.size() && source.compare(i + 1, tlen, vecTypes[vi]) == 0) { // Check what follows: should be whitespace then '(' for a cast constructor size_t after = i + 1 + tlen; @@ -363,13 +362,13 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { // Also handles: " local T lds[IN_WG / 2 * (MIDDLE <= 8 ? 2 * MIDDLE : MIDDLE)];" // But NOT: "local T2 *lds" in function params (which becomes just "T2 *lds" via macro) if (i + 6 <= source.size() && source.compare(i, 6, "local ") == 0) { - bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); if (!preceded) { // Check if this "local" is followed by a type then a name then '[' // i.e., it's a shared memory array declaration size_t lineEnd = source.find('\n', i); if (lineEnd == std::string::npos) lineEnd = source.size(); - std::string line = source.substr(i, lineEnd - i); + std::string const line = source.substr(i, lineEnd - i); // Match: "local TYPE IDENT[" pattern — indicates array declaration // Array declarations have '[' and end with ';'. They may also contain '(' in // the array size expression (e.g., ternary operators). The key distinction is @@ -386,11 +385,11 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { } // Same for __local if (i + 8 <= source.size() && source.compare(i, 8, "__local ") == 0) { - bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); if (!preceded) { size_t lineEnd = source.find('\n', i); if (lineEnd == std::string::npos) lineEnd = source.size(); - std::string line = source.substr(i, lineEnd - i); + std::string const line = source.substr(i, lineEnd - i); if (line.find('[') != std::string::npos) { result += "__shared__ "; i += 8; @@ -420,7 +419,7 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { if (i + 8 < source.size() && source[i + 8] == '_') { result += source[i++]; } else { - bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_')); if (preceded) { result += source[i++]; } else { @@ -430,8 +429,8 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { } // Match standalone "global" (not inside a word or PTX instruction) else if (i + 6 <= source.size() && source.compare(i, 6, "global") == 0) { - bool preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_' || source[i-1] == '.')); - bool followed = (i + 6 < source.size() && (isalnum(source[i + 6]) || source[i + 6] == '_')); + bool const preceded = (i > 0 && (isalnum(source[i-1]) || source[i-1] == '_' || source[i-1] == '.')); + bool const followed = (i + 6 < source.size() && (isalnum(source[i + 6]) || source[i + 6] == '_')); if (!preceded && !followed) { i += 6; } else { @@ -462,9 +461,10 @@ std::string NvrtcProgram::compile(const std::string& source, const std::string& // Convert options to char* std::vector opts; - for (auto& o : options) opts.push_back(o.c_str()); + opts.reserve(options.size()); +for (auto& o : options) opts.push_back(o.c_str()); - nvrtcResult compileResult = nvrtcCompileProgram(prog, (int)opts.size(), opts.data()); + nvrtcResult const compileResult = nvrtcCompileProgram(prog, (int)opts.size(), opts.data()); // Get compilation log size_t logSize; diff --git a/src/cuda/cudawrap.h b/src/cuda/cudawrap.h index bd0e6e46..2be6f72b 100644 --- a/src/cuda/cudawrap.h +++ b/src/cuda/cudawrap.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -47,8 +48,8 @@ class CudaContext { explicit CudaContext(CUdevice dev); ~CudaContext(); - CUcontext get() const { return ctx; } - CUdevice getDevice() const { return device; } + [[nodiscard]] CUcontext get() const { return ctx; } + [[nodiscard]] CUdevice getDevice() const { return device; } void makeCurrent(); }; @@ -66,7 +67,7 @@ class CudaModule { return *this; } - CUmodule get() const { return module; } + [[nodiscard]] CUmodule get() const { return module; } CUfunction getFunction(const char* name) const; }; @@ -77,7 +78,7 @@ class CudaStream { CudaStream(); ~CudaStream(); - CUstream get() const { return stream; } + [[nodiscard]] CUstream get() const { return stream; } void sync(); CudaStream(CudaStream&& rhs) noexcept : stream(rhs.stream) { rhs.stream = nullptr; } @@ -99,8 +100,8 @@ class CudaBuffer { return *this; } - CUdeviceptr get() const { return ptr; } - size_t size() const { return bytes; } + [[nodiscard]] CUdeviceptr get() const { return ptr; } + [[nodiscard]] size_t size() const { return bytes; } void readSync(void* dst, size_t n) const; void writeSync(const void* src, size_t n); @@ -128,11 +129,11 @@ class CudaKernelLauncher { public: CudaKernelLauncher() = default; - CudaKernelLauncher(CUfunction f, const std::string& name, u32 blockSize) - : func(f), name(name), blockSize(blockSize) {} + CudaKernelLauncher(CUfunction f, std::string name, u32 blockSize) + : func(f), name(std::move(name)), blockSize(blockSize) {} void launch(CUstream stream, u32 gridSize, void** args, u32 sharedMem = 0); - CUfunction get() const { return func; } - const std::string& getName() const { return name; } + [[nodiscard]] CUfunction get() const { return func; } + [[nodiscard]] const std::string& getName() const { return name; } }; diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index d01946df..21b19fec 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -20,11 +20,11 @@ // cl_device_id wraps CUdevice (int) struct _cl_device_id { CUdevice dev; }; -typedef _cl_device_id* cl_device_id; +using cl_device_id = _cl_device_id*; // cl_context wraps CUcontext struct _cl_context { CUcontext ctx; CUdevice dev; }; -typedef _cl_context* cl_context; +using cl_context = _cl_context*; // cl_command_queue wraps CUstream struct _cl_command_queue { @@ -32,35 +32,35 @@ struct _cl_command_queue { cl_context context; bool profiling; }; -typedef _cl_command_queue* cl_command_queue; +using cl_command_queue = _cl_command_queue*; // cl_mem wraps CUdeviceptr + size struct _cl_mem { CUdeviceptr ptr; size_t size; }; -typedef _cl_mem* cl_mem; +using cl_mem = _cl_mem*; // cl_program: dual-purpose — stores either source string or compiled PTX/module struct _cl_program { std::string source; // OpenCL source (before NVRTC compilation) std::string preprocessedSource; // CUDA source after preprocessOpenCL (for parsing __launch_bounds__) std::string ptx; // Compiled PTX (after NVRTC compilation) - CUmodule module; // Loaded module (after cuModuleLoadData) - bool compiled; - bool moduleLoaded; + CUmodule module{}; // Loaded module (after cuModuleLoadData) + bool compiled{false}; + bool moduleLoaded{false}; - _cl_program() : module{}, compiled{false}, moduleLoaded{false} {} + _cl_program() {} }; -typedef _cl_program* cl_program; +using cl_program = _cl_program*; // cl_kernel wraps CUfunction + accumulated arguments struct _cl_kernel { - CUfunction func; + CUfunction func{}; std::string name; - CUmodule parentModule; // Keep reference so module isn't unloaded - int numArgs; - int reqWorkGroupSize; // From __launch_bounds__(N) in source, matches OpenCL reqd_work_group_size + CUmodule parentModule{}; // Keep reference so module isn't unloaded + int numArgs{0}; + int reqWorkGroupSize{0}; // From __launch_bounds__(N) in source, matches OpenCL reqd_work_group_size // Argument accumulator for setArg/launch pattern static constexpr int MAX_ARGS = 32; @@ -69,7 +69,7 @@ struct _cl_kernel { size_t argSizes[MAX_ARGS]; size_t argOffsets[MAX_ARGS]; - _cl_kernel() : func{}, parentModule{}, numArgs{0}, reqWorkGroupSize{0} { + _cl_kernel() { memset(argData, 0, sizeof(argData)); memset(argSizes, 0, sizeof(argSizes)); memset(argOffsets, 0, sizeof(argOffsets)); @@ -82,7 +82,7 @@ struct _cl_kernel { // Fixed 8-byte slots per arg position. CUDA kernel args are pointers (8 bytes) // or small scalars (4 bytes). Using fixed slots avoids data corruption when // args are set out of order (e.g., setFixedArgs(2,3) then operator()(0,1)). - size_t offset = pos * 8; + size_t const offset = pos * 8; argOffsets[pos] = offset; argSizes[pos] = size; if (offset + size <= MAX_ARG_BYTES && value) { @@ -97,42 +97,42 @@ struct _cl_kernel { } } }; -typedef _cl_kernel* cl_kernel; +using cl_kernel = _cl_kernel*; // cl_event wraps CUevent pair (start + end for profiling) struct _cl_event { - CUevent start; - CUevent end; - bool hasTimings; - u32 commandType; + CUevent start{}; + CUevent end{}; + bool hasTimings{false}; + u32 commandType{0}; - _cl_event() : start{}, end{}, hasTimings{false}, commandType{0} {} + _cl_event() {} ~_cl_event() { if (start) cuEventDestroy(start); if (end) cuEventDestroy(end); } }; -typedef _cl_event* cl_event; +using cl_event = _cl_event*; // Unused types — just need to exist for compilation -typedef struct _cl_platform_id* cl_platform_id; -typedef struct _cl_sampler* cl_sampler; - -typedef unsigned cl_bool; -typedef unsigned cl_program_build_info; -typedef unsigned cl_program_info; -typedef unsigned cl_device_info; -typedef unsigned cl_kernel_info; -typedef unsigned cl_kernel_arg_info; -typedef unsigned cl_kernel_work_group_info; -typedef unsigned cl_profiling_info; -typedef unsigned cl_event_info; -typedef unsigned cl_command_queue_info; - -typedef u64 cl_mem_flags; -typedef u64 cl_svm_mem_flags; -typedef u64 cl_device_type; -typedef u64 cl_queue_properties; +using cl_platform_id = struct _cl_platform_id*; +using cl_sampler = struct _cl_sampler*; + +using cl_bool = unsigned; +using cl_program_build_info = unsigned; +using cl_program_info = unsigned; +using cl_device_info = unsigned; +using cl_kernel_info = unsigned; +using cl_kernel_arg_info = unsigned; +using cl_kernel_work_group_info = unsigned; +using cl_profiling_info = unsigned; +using cl_event_info = unsigned; +using cl_command_queue_info = unsigned; + +using cl_mem_flags = u64; +using cl_svm_mem_flags = u64; +using cl_device_type = u64; +using cl_queue_properties = u64; using cl_queue = cl_command_queue; @@ -217,10 +217,10 @@ using cl_queue = cl_command_queue; #define CL_DEVICE_BOARD_NAME_AMD 0x4038 #define CL_DEVICE_GLOBAL_FREE_MEMORY_AMD 0x4039 -typedef union { +using cl_device_topology_amd = union { struct { u32 type; u32 data[5]; } raw; struct { u32 type; char unused[17]; char bus; char device; char function; } pcie; -} cl_device_topology_amd; +}; // Error codes #define CL_DEVICE_NOT_FOUND -1 diff --git a/src/fs.cpp b/src/fs.cpp index 7f7ab719..8eefa860 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -4,6 +4,7 @@ #include "File.h" #include +#include namespace { @@ -53,14 +54,14 @@ void fancyRename(const fs::path& src, const fs::path& dst) { u64 fileSize(const fs::path& path) { error_code dummy; auto size = fs::file_size(path, dummy); - if (size == decltype(size)(-1)) { size = 0; } + if (std::cmp_equal(size, -1)) { size = 0; } return size; } bool deleteLine(const fs::path& path, const string& targetLine, u64 initialSize) { if (!initialSize) { initialSize = fileSize(path); } - fs::path tmp = path + ("-"s + toString(this_thread::get_id())); + fs::path const tmp = path + ("-"s + toString(this_thread::get_id())); if (!copyWithout(targetLine, path, tmp) || !sizeMatches(path, initialSize)) { return false; } diff --git a/src/gpuid.cpp b/src/gpuid.cpp index 8a24590d..f27be4f3 100644 --- a/src/gpuid.cpp +++ b/src/gpuid.cpp @@ -1,13 +1,14 @@ // Copyright (C) 2017-2024 Mihai Preda. #include "gpuid.h" +#include #include "clwrap.h" #include "File.h" using namespace std; static bool startsWith(string_view a, string_view b) { - return a.substr(0, b.length()) == b; + return a.starts_with(b); } string getBdfFromSysfs(int pos) { @@ -58,18 +59,18 @@ string getUidFromSysfs(int pos) { /* BDF is PCIe Bus:Device.Function e.g. "6a:00.0" */ string getUidFromBdf(const string& bdf) { - int pos = getSysfsFromBdf(bdf); + int const pos = getSysfsFromBdf(bdf); return pos >= 0 ? getUidFromSysfs(pos) : ""; } string getBdfFromUid(const string& uid) { - int pos = getSysfsFromUid(uid); + int const pos = getSysfsFromUid(uid); return (pos >= 0) ? getBdfFromSysfs(pos) : ""; } int getPosFromBdf(const string& bdf) { auto openclIds = getAllDeviceIDs(); - for (int pos = 0; pos < int(openclIds.size()); ++pos) { + for (int pos = 0; std::cmp_less(pos, openclIds.size()); ++pos) { auto bdfAtPos = getBdfFromDevice(openclIds[pos]); // log("BDF '%s' at %d\n", bdfAtPos.c_str(), pos); if (bdf == bdfAtPos) { return pos; } @@ -85,7 +86,7 @@ string getBdfFromPos(int pos) { } int getPosFromUid(const string& uid) { - string bdf = getBdfFromUid(uid); + string const bdf = getBdfFromUid(uid); if (bdf.empty()) { return -1; } return getPosFromBdf(bdf); } diff --git a/src/log.cpp b/src/log.cpp index 1b93ee57..0f80be05 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -11,7 +11,7 @@ thread_local vector contextParts; thread_local File logFile; -File stdoutFile{stdout, "stdout"}; +static File stdoutFile{stdout, "stdout"}; string logContext() { return context; } @@ -20,7 +20,7 @@ void initLog(const char *logName) { logFile = File::openAppend(logName); } -string longTimeStr() { return timeStr("%Y-%m-%d %H:%M:%S %Z"); } +static string longTimeStr() { return timeStr("%Y-%m-%d %H:%M:%S %Z"); } string shortTimeStr() { return timeStr("%Y%m%d %H:%M:%S"); } static char logBuf[32 * 1024]; @@ -28,9 +28,9 @@ static char logBuf[32 * 1024]; void log(const char *fmt, ...) { static std::mutex logMutex; - string prefix = shortTimeStr() + ' ' + context; + string const prefix = shortTimeStr() + ' ' + context; - std::unique_lock lock(logMutex); + std::unique_lock const lock(logMutex); int pos = 0; snprintf(logBuf, sizeof(logBuf), "%s %n", prefix.c_str(), &pos); @@ -38,7 +38,7 @@ void log(const char *fmt, ...) { va_start(va, fmt); vsnprintf(logBuf + pos, sizeof(logBuf) - pos, fmt, va); va_end(va); - string_view s{logBuf}; + string_view const s{logBuf}; if (logFile) { logFile.write(s); } stdoutFile.write(s); diff --git a/src/main.cpp b/src/main.cpp index efdfc38d..c19294b8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -18,9 +18,10 @@ #include #include +#include // #include from GCC-13 onwards -void gpuWorker(GpuCommon shared, i32 instance) { +static void gpuWorker(GpuCommon shared, i32 instance) { // LogContext context{(instance ? shared.args->tailDir() : ""s) + to_string(instance) + ' '}; // log("Starting worker %d\n", instance); if (instance > 0) { @@ -50,7 +51,7 @@ int main(int argc, char **argv) { _set_printf_count_output(1); // I'm not sure what this does (it's from CrazeTheDragon) #endif -#if defined(__MSYS__) +#ifdef __MSYS__ // I was unable to get putenv to link in MSYS2 #elif defined(__MINGW32__) || defined(__MINGW64__) putenv("ROC_SIGNAL_POOL_SIZE=32"); @@ -61,10 +62,10 @@ int main(int argc, char **argv) { setenv("ROC_SIGNAL_POOL_SIZE", "32", 0); #endif - int exitCode = 0; + int const exitCode = 0; try { - string mainLine = Args::mergeArgs(argc, argv); + string const mainLine = Args::mergeArgs(argc, argv); { Args args{true}; args.parse(mainLine); @@ -94,7 +95,7 @@ int main(int argc, char **argv) { if (args.maxAlloc) { AllocTrac::setMaxAlloc(args.maxAlloc); } Context context(getDevice(args.device)); - Signal signal; + Signal const signal; Background background; GpuCommon shared; shared.context = &context; @@ -118,7 +119,7 @@ int main(int argc, char **argv) { } else { { vector threads; - for (int i = 1; i < int(args.workers); ++i) { + for (int i = 1; std::cmp_less(i, args.workers); ++i) { threads.emplace_back(gpuWorker, shared, i); } gpuWorker(shared, 0); diff --git a/src/md5.cpp b/src/md5.cpp index 2bf8329f..bbecb022 100644 --- a/src/md5.cpp +++ b/src/md5.cpp @@ -5,21 +5,21 @@ * This code is in the public domain; do with it what you wish. */ #include "MD5.h" -#include +#include #define byteReverse(A,B) /* The four core functions - F1 is optimized somewhat */ /* #define F1(x, y, z) (x & y | ~x & z) */ -#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F1(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) #define F2(x, y, z) F1(z, x, y) -#define F3(x, y, z) (x ^ y ^ z) -#define F4(x, y, z) (y ^ (x | ~z)) +#define F3(x, y, z) ((x) ^ (y) ^ (z)) +#define F4(x, y, z) ((y) ^ ((x) | ~(z))) /* This is the central step in the MD5 algorithm. */ #define MD5STEP(f, w, x, y, z, data, s) \ - ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + ( (w) += f(x, y, z) + (data), (w) = (w)<<(s) | (w)>>(32-(s)), (w) += (x) ) /* * The core of the MD5 algorithm, this alters an existing MD5 hash to @@ -132,7 +132,7 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ /* Update bitcount */ t = ctx->bits[0]; - if ((ctx->bits[0] = t + ((unsigned)len << 3)) < t) + if ((ctx->bits[0] = t + (len << 3)) < t) ctx->bits[1]++; /* Carry from low to high */ ctx->bits[1] += len >> 29; diff --git a/src/sha3.cpp b/src/sha3.cpp index cf0d8b10..48c4d2c0 100644 --- a/src/sha3.cpp +++ b/src/sha3.cpp @@ -90,7 +90,7 @@ static void KeccakF1600Step(SHA3Context *p){ # define A42 (p->u.s[22]) # define A43 (p->u.s[23]) # define A44 (p->u.s[24]) -# define ROL64(a,x) ((a<>(64-x))) +# define ROL64(a,x) (((a)<<(x))|((a)>>(64-(x)))) for(i=0; i<24; i+=4){ C0 = A00^A10^A20^A30^A40; @@ -395,7 +395,7 @@ void SHA3Update( ){ unsigned int i = 0; #if SHA3_BYTEORDER==1234 - if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)nullptr)&7)==0 ){ for(; i+7u.s[p->nLoaded/8] ^= *(u64*)&aData[i]; p->nLoaded += 8; diff --git a/src/shared.h b/src/shared.h index 6405d9c1..33ff1113 100644 --- a/src/shared.h +++ b/src/shared.h @@ -1,4 +1,4 @@ // included from both C++ and OpenCL. -u32 bitposToWord(u64 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } -u32 wordToBitpos(u64 E, u32 N, u32 word) { return (word * ((u64) E) + (N - 1)) / N; } +inline u32 bitposToWord(u64 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } +inline u32 wordToBitpos(u64 E, u32 N, u32 word) { return (word * ( E) + (N - 1)) / N; } diff --git a/src/state.cpp b/src/state.cpp index 942aeaab..8ac49307 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -2,18 +2,15 @@ #include "state.h" #include "shared.h" -#include "log.h" -#include "timeutil.h" #include -#include static i64 lowBits(i64 u, int bits) { return (u << (64 - bits)) >> (64 - bits); } std::vector compactBits(const vector &dataVect, u64 E) { if (dataVect.empty()) { return {}; } // Indicating all zero - u32 N = dataVect.size(); + u32 const N = dataVect.size(); const Word *data = dataVect.data(); std::vector out; @@ -29,14 +26,14 @@ std::vector compactBits(const vector &dataVect, u64 E) { assert(nBits > 0); // Be careful adding in the carry -- it could overflow a 32-bit word. Convert value into desired unsigned range. - i64 tmp = (i64) data[p] + carry; + i64 const tmp = (i64) data[p] + carry; carry = (int) (tmp >> nBits); u64 w = (u64) (tmp - ((i64) carry << nBits)); assert(w < (1ULL << nBits)); assert(haveBits < 32); while (nBits) { - int needBits = 32 - haveBits; + int const needBits = 32 - haveBits; outWord |= w << haveBits; if (nBits >= needBits) { w >>= needBits; @@ -56,7 +53,7 @@ std::vector compactBits(const vector &dataVect, u64 E) { out.push_back(outWord); for (int p = 0; carry; ++p) { - i64 v = i64(out[p]) + carry; + i64 const v = i64(out[p]) + carry; out[p] = v & 0xffffffff; carry = v >> 32; } @@ -66,10 +63,10 @@ std::vector compactBits(const vector &dataVect, u64 E) { } struct BitBucket { - u128 bits; - u32 size; + u128 bits{0}; + u32 size{0}; - BitBucket() : bits(0), size(0) {} + BitBucket() = default; void put32(u32 b) { assert(size <= 96); @@ -79,7 +76,7 @@ struct BitBucket { i64 popSigned(u32 n) { assert(size >= n); - i64 b = lowBits((i64) bits, n); + i64 const b = lowBits((i64) bits, n); size -= n; bits >>= n; bits += (b < 0); // carry fixup. @@ -97,7 +94,7 @@ vector expandBits(const vector &compactBits, u32 N, u64 E) { auto it = compactBits.cbegin(); [[maybe_unused]] auto itEnd = compactBits.cend(); for (u32 p = 0; p < N; ++p) { - u32 len = bitlen(N, E, p); + u32 const len = bitlen(N, E, p); while (bucket.size < len) { assert(it != itEnd); bucket.put32(*it++); diff --git a/src/timeutil.cpp b/src/timeutil.cpp index f3102145..f5fa1f8b 100644 --- a/src/timeutil.cpp +++ b/src/timeutil.cpp @@ -5,14 +5,14 @@ #include std::string timeStr(const char *format) { - time_t t = time(NULL); + time_t const t = time(nullptr); char buf[64]; strftime(buf, sizeof(buf), format, localtime(&t)); return buf; } std::string timeStr() { - time_t t = time(NULL); + time_t const t = time(nullptr); char buf[64]; strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", gmtime(&t)); // equivalent to: "%F %T" return buf; diff --git a/src/timeutil.h b/src/timeutil.h index ecc21c72..e2bbe7db 100644 --- a/src/timeutil.h +++ b/src/timeutil.h @@ -18,11 +18,11 @@ class Timer { Timer() : start(clock::now()) {} - double at() const { return std::chrono::duration(clock::now() - start).count(); } + [[nodiscard]] double at() const { return std::chrono::duration(clock::now() - start).count(); } double reset() { auto now = clock::now(); - double ret = std::chrono::duration(now - start).count(); + double const ret = std::chrono::duration(now - start).count(); start = now; return ret; } diff --git a/src/tinycl.h b/src/tinycl.h index 90fa8404..6732f8aa 100644 --- a/src/tinycl.h +++ b/src/tinycl.h @@ -7,31 +7,31 @@ #include #include -typedef struct _cl_platform_id * cl_platform_id; -typedef struct _cl_device_id * cl_device_id; -typedef struct _cl_context * cl_context; -typedef struct _cl_command_queue * cl_command_queue; -typedef struct _cl_mem * cl_mem; -typedef struct _cl_program * cl_program; -typedef struct _cl_kernel * cl_kernel; -typedef struct _cl_event * cl_event; -typedef struct _cl_sampler * cl_sampler; - -typedef unsigned cl_bool; -typedef unsigned cl_program_build_info; -typedef unsigned cl_program_info; -typedef unsigned cl_device_info; -typedef unsigned cl_kernel_info; -typedef unsigned cl_kernel_arg_info; -typedef unsigned cl_kernel_work_group_info; -typedef unsigned cl_profiling_info; -typedef unsigned cl_event_info; -typedef unsigned cl_command_queue_info; - -typedef u64 cl_mem_flags; -typedef u64 cl_svm_mem_flags; -typedef u64 cl_device_type; -typedef u64 cl_queue_properties; +using cl_platform_id = struct _cl_platform_id *; +using cl_device_id = struct _cl_device_id *; +using cl_context = struct _cl_context *; +using cl_command_queue = struct _cl_command_queue *; +using cl_mem = struct _cl_mem *; +using cl_program = struct _cl_program *; +using cl_kernel = struct _cl_kernel *; +using cl_event = struct _cl_event *; +using cl_sampler = struct _cl_sampler *; + +using cl_bool = unsigned; +using cl_program_build_info = unsigned; +using cl_program_info = unsigned; +using cl_device_info = unsigned; +using cl_kernel_info = unsigned; +using cl_kernel_arg_info = unsigned; +using cl_kernel_work_group_info = unsigned; +using cl_profiling_info = unsigned; +using cl_event_info = unsigned; +using cl_command_queue_info = unsigned; + +using cl_mem_flags = u64; +using cl_svm_mem_flags = u64; +using cl_device_type = u64; +using cl_queue_properties = u64; extern "C" { diff --git a/src/tune.cpp b/src/tune.cpp index dea4f864..68353e95 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -12,11 +12,11 @@ #include #include +#include #include #include #include -using std::accumulate; using namespace std; @@ -24,13 +24,13 @@ vector split(const string& s, char delim) { vector ret; size_t start = 0; while (true) { - size_t p = s.find(delim, start); + size_t const p = s.find(delim, start); if (p == string::npos) { ret.push_back(s.substr(start)); break; - } else { + } ret.push_back(s.substr(start, p - start)); - } + start = p + 1; } return ret; @@ -41,12 +41,12 @@ namespace { vector permute(const vector>>& params) { vector configs; - int n = params.size(); + int const n = params.size(); vector vpos(n); while (true) { TuneConfig config; for (int i = 0; i < n; ++i) { - config.push_back({params[i].first, params[i].second[vpos[i]]}); + config.emplace_back(params[i].first, params[i].second[vpos[i]]); } configs.push_back(config); @@ -55,9 +55,9 @@ vector permute(const vector>>& params) { if (vpos[i] < int(params[i].second.size()) - 1) { ++vpos[i]; break; - } else { + } vpos[i] = 0; - } + } if (i < 0) { return configs; } @@ -69,14 +69,14 @@ vector getTuneConfigs(const string& tune) { for (auto& part : split(tune, ';')) { auto keyVal = split(part, '='); assert(keyVal.size() == 2); - string key = keyVal.front(); - string val = keyVal.back(); - params.push_back({key, split(val, ',')}); + string const& key = keyVal.front(); + const string& val = keyVal.back(); + params.emplace_back(key, split(val, ',')); } return permute(params); } -string toString(TuneConfig config) { +string toString(const TuneConfig& config) { string s{}; for (const auto& [k, v] : config) { s += k + '=' + v + ','; } s.pop_back(); @@ -89,7 +89,7 @@ struct Entry { double cost; }; -string formatEntry(Entry e) { +string formatEntry(const Entry& e) { char buf[256]; snprintf(buf, sizeof(buf), "! %s %s # %.0f\n", e.shape.spec().c_str(), toString(e.config).c_str(), e.cost); @@ -134,14 +134,14 @@ float Tune::maxBpw(FFTConfig fft) { float z1 = zForBpw(bpw1, fft, 1); printf ("Guess bpw for %s is %.2f first Z34 is %.2f\n", fft.spec().c_str(), bpw1, z1); while (z1 < 31.0f || z1 > 37.0f) { - float prev_bpw1 = bpw1; - float prev_z1 = z1; + float const prev_bpw1 = bpw1; + float const prev_z1 = z1; bpw1 = bpw1 + (z1 - 34.0f) * bpw_step; z1 = zForBpw(bpw1, fft, 1); printf ("Reguess bpw for %s is %.2f first Z34 is %.2f\n", fft.spec().c_str(), bpw1, z1); bpw_step = - (bpw1 - prev_bpw1) / (z1 - prev_z1); - if (bpw_step < 0.005f) bpw_step = 0.005f; - if (bpw_step > 0.025f) bpw_step = 0.025f; + bpw_step = std::max(bpw_step, 0.005f); + bpw_step = std::min(bpw_step, 0.025f); } // Get more samples for this bpw -- average in the sample we already have @@ -152,8 +152,8 @@ printf ("Reguess bpw for %s is %.2f first Z34 is %.2f\n", fft.spec().c_str(), bp float z2 = zForBpw(bpw2, fft, 1); printf ("Guess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bpw2, z2); while (z2 < 20.0f || z2 > 25.0f) { - float prev_bpw2 = bpw2; - float prev_z2 = z2; + float const prev_bpw2 = bpw2; + float const prev_z2 = z2; // bool error_recovery = (z2 <= 0.0); // if (error_recovery) bpw2 -= bpw_step; else bpw2 = bpw2 + (z2 - 21.0f) * bpw_step; @@ -161,8 +161,8 @@ printf ("Guess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bpw2 printf ("Reguess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bpw2, z2); // if (error_recovery) { if (z2 >= 20.0) break; else continue; } bpw_step = - (bpw2 - prev_bpw2) / (z2 - prev_z2); - if (bpw_step < 0.005f) bpw_step = 0.005f; - if (bpw_step > 0.025f) bpw_step = 0.025f; + bpw_step = std::max(bpw_step, 0.005f); + bpw_step = std::min(bpw_step, 0.025f); } // Get more samples for this bpw -- average in the sample we already have @@ -177,7 +177,7 @@ float Tune::zForBpw(float bpw, FFTConfig fft, u32 count) { float total_z = 0.0f; for (u32 i = 0; i < count; i++, exponent = primes.nextPrime (exponent + 1)) { auto [ok, res, roeSq, roeMul] = Gpu::make(exponent, shared, fft, {}, false)->measureROE(true); - float z = roeSq.z(); + float const z = roeSq.z(); total_z += z; log("Zforbpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); if (!ok) { log("Error at bpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); continue; } @@ -187,17 +187,17 @@ log("Zforbpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); } void Tune::ztune() { - File ztune = File::openAppend("ztune.txt"); + File const ztune = File::openAppend("ztune.txt"); ztune.printf("\n// %s\n\n", shortTimeStr().c_str()); // Study a specific shape and variant - if (0) { - FFTShape shape = FFTShape(FFT64, 512, 15, 512); - u32 variant = 202; - u32 sample_size = 5; - FFTConfig fft{shape, variant, CARRY_AUTO}; + if (false) { + FFTShape const shape = FFTShape(FFT64, 512, 15, 512); + u32 const variant = 202; + u32 const sample_size = 5; + FFTConfig const fft{shape, variant, CARRY_AUTO}; for (float bpw = 18.18f; bpw < 18.305f; bpw += 0.02f) { - float z = zForBpw(bpw, fft, sample_size); + float const z = zForBpw(bpw, fft, sample_size); log ("Avg zForBpw %s %.2f %.2f\n", fft.spec().c_str(), bpw, z); } } @@ -207,7 +207,7 @@ void Tune::ztune() { // Over this narrow Z range, linear curve fit should work well. The Z data is noisy, so more samples is better. auto configs = FFTShape::multiSpec(shared.args->fftSpec); - for (FFTShape shape : configs) { + for (FFTShape const shape : configs) { // 4K widths store data on variants 100, 101, 202, 110, 111, 212 u32 bpw_variants[NUM_BPW_ENTRIES] = {000, 101, 202, 10, 111, 212}; @@ -224,10 +224,10 @@ void Tune::ztune() { // Test specific variants needed for the maximum bpw table in fftbpw.h for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) { - FFTConfig fft{shape, bpw_variants[j], CARRY_AUTO}; + FFTConfig const fft{shape, bpw_variants[j], CARRY_AUTO}; bpw[j] = maxBpw(fft); } - string s = "\""s + shape.spec() + "\""; + string const s = "\""s + shape.spec() + "\""; // ztune.printf("{%12s, {%.3f, %.3f, %.3f, %.3f, %.3f, %.3f}},\n", s.c_str(), bpw[0], bpw[1], bpw[2], bpw[3], bpw[4], bpw[5]); ztune.printf("{%12s, {", s.c_str()); for (u32 j = 0; j < NUM_BPW_ENTRIES; ++j) ztune.printf("%s%.3f", j ? ", " : "", bpw[j]); @@ -236,47 +236,48 @@ void Tune::ztune() { } void Tune::carryTune() { - File fo = File::openAppend("carrytune.txt"); + File const fo = File::openAppend("carrytune.txt"); fo.printf("\n// %s\n\n", shortTimeStr().c_str()); shared.args->flags["STATS"] = "1"; u32 prevSize = 0; - for (FFTShape shape : FFTShape::multiSpec(shared.args->fftSpec)) { - FFTConfig fft{shape, LAST_VARIANT, CARRY_AUTO}; + for (FFTShape const shape : FFTShape::multiSpec(shared.args->fftSpec)) { + FFTConfig const fft{shape, LAST_VARIANT, CARRY_AUTO}; if (prevSize == fft.size()) { continue; } prevSize = fft.size(); vector zv; double m = 0; const float mid = fft.shape.carry32BPW(); - for (float bpw : {mid - 0.05f, mid + 0.05f}) { - u64 exponent = primes.nearestPrime(fft.size() * bpw); + for (float const bpw : {mid - 0.05f, mid + 0.05f}) { + u64 const exponent = primes.nearestPrime(fft.size() * bpw); auto [ok, carry] = Gpu::make(exponent, shared, fft, {}, false)->measureCarry(); m = carry.max; if (!ok) { log("Error %s at %f\n", fft.spec().c_str(), bpw); } zv.push_back(carry.z()); } - float avg = (zv[0] + zv[1]) / 2; - u64 exponent = fft.shape.carry32BPW() * fft.size(); - double pErr100 = -expm1(-exp(-avg) * exponent * 100); + float const avg = (zv[0] + zv[1]) / 2; + u64 const exponent = fft.shape.carry32BPW() * fft.size(); + double const pErr100 = -expm1(-exp(-avg) * exponent * 100); log("%14s %.3f : %.3f (%.3f %.3f) %f %.0f%%\n", fft.spec().c_str(), mid, avg, zv[0], zv[1], m, pErr100 * 100); fo.printf("%f %f\n", log2(fft.size()), avg); } } template -void add(vector& a, const vector& b) { +static void add(vector& a, const vector& b) { a.insert(a.end(), b.begin(), b.end()); } void Tune::ctune() { - Args *args = shared.args; + Args const*args = shared.args; vector ctune = args->ctune; - if (ctune.empty()) { ctune.push_back("IN_WG=256,128,64;IN_SIZEX=32,16,8;OUT_WG=256,128,64;OUT_SIZEX=32,16,8"); } + if (ctune.empty()) { ctune.emplace_back("IN_WG=256,128,64;IN_SIZEX=32,16,8;OUT_WG=256,128,64;OUT_SIZEX=32,16,8"); } vector> configsVect; - for (const string& s : ctune) { + configsVect.reserve(ctune.size()); +for (const string& s : ctune) { configsVect.push_back(getTuneConfigs(s)); } @@ -290,13 +291,13 @@ void Tune::ctune() { log("FFTs: %s\n", str.c_str()); } - for (FFTShape shape : shapes) { - FFTConfig fft{shape, 101, CARRY_32}; - u64 exponent = primes.prevPrime(fft.maxExp()); + for (FFTShape const shape : shapes) { + FFTConfig const fft{shape, 101, CARRY_32}; + u64 const exponent = primes.prevPrime(fft.maxExp()); // log("tuning %10s with exponent %" PRIu64 "\n", fft.shape.spec().c_str(), exponent); vector bestPos(configsVect.size()); - Entry best{{}, {}, 1e9}; + Entry best{.shape={}, .config={}, .cost=1e9}; for (u32 i = 0; i < configsVect.size(); ++i) { for (u32 pos = i ? 1 : 0; pos < configsVect[i].size(); ++pos) { @@ -311,10 +312,10 @@ void Tune::ctune() { } auto cost = Gpu::make(exponent, shared, fft, c, false)->timePRP(); - bool isBest = (cost < best.cost); + bool const isBest = (cost < best.cost); if (isBest) { bestPos[i] = pos; - best = {shape, c, cost}; + best = {.shape=shape, .config=c, .cost=cost}; } log("%c %6.0f : %s %s\n", isBest ? '*' : ' ', cost, shape.spec().c_str(), toString(c).c_str()); @@ -327,14 +328,14 @@ void Tune::ctune() { } // Add better -use settings to list of changes to be made to config.txt -void configsUpdate(double current_cost, double best_cost, double threshold, const char *key, u32 value, vector> &newConfigKeyVals, vector> &suggestedConfigKeyVals) { +static void configsUpdate(double current_cost, double best_cost, double threshold, const char *key, u32 value, vector> &newConfigKeyVals, vector> &suggestedConfigKeyVals) { if (best_cost == current_cost) return; // If best cost is better than current cost by a substantial margin (the threshold) then add the key value pair to suggestedConfigKeyVals if (best_cost < (1.0 - threshold) * current_cost) - newConfigKeyVals.push_back({key, value}); + newConfigKeyVals.emplace_back(key, value); // Otherwise, add the key value pair to newConfigKeyVals else - suggestedConfigKeyVals.push_back({key, value}); + suggestedConfigKeyVals.emplace_back(key, value); } void Tune::tune() { @@ -342,15 +343,15 @@ void Tune::tune() { vector shapes = FFTShape::multiSpec(args->fftSpec); // There are some options and variants that are different based on GPU manufacturer - bool AMDGPU = isAmdGpu(shared.context->deviceId()); - bool NVIDIAGPU = isNvidiaGpu(shared.context->deviceId()); - int NO_ASM = args->value("NO_ASM", 0); - - bool tune_config = 1; - bool time_FFTs = 0; - bool time_NTTs = 0; - bool time_FP32 = 1; - bool time_inplace_only = NVIDIAGPU ? 1 : 0; // Default is nVidia is better off with INPLACE=1, AMD GPUs need to time extra options used when INPLACE=0 + bool const AMDGPU = isAmdGpu(shared.context->deviceId()); + bool const NVIDIAGPU = isNvidiaGpu(shared.context->deviceId()); + int const NO_ASM = args->value("NO_ASM", 0); + + bool tune_config = true; + bool time_FFTs = false; + bool time_NTTs = false; + bool time_FP32 = true; + bool time_inplace_only = NVIDIAGPU; // Default is nVidia is better off with INPLACE=1, AMD GPUs need to time extra options used when INPLACE=0 int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) u64 min_exponent = 75000000; u64 max_exponent = 350000000; @@ -359,11 +360,11 @@ void Tune::tune() { // Parse input args for (const string& s : split(args->tune, ',')) { if (s.empty()) continue; - if (s == "noconfig") tune_config = 0; - if (s == "fp64") time_FFTs = 1; - if (s == "ntt") time_NTTs = 1; - if (s == "nofp32") time_FP32 = 0; - if (s == "inplace") time_inplace_only = 1; + if (s == "noconfig") tune_config = false; + if (s == "fp64") time_FFTs = true; + if (s == "ntt") time_NTTs = true; + if (s == "nofp32") time_FP32 = false; + if (s == "inplace") time_inplace_only = true; auto keyVal = split(s, '='); if (keyVal.size() == 2) { if (keyVal.front() == "quick") quick = stod(keyVal.back()); @@ -371,8 +372,8 @@ void Tune::tune() { if (keyVal.front() == "maxexp") max_exponent = stoull(keyVal.back()); } } - if (quick < 1) quick = 1; - if (quick > 10) quick = 10; + quick = std::max(quick, 1); + quick = std::min(quick, 10); // Look for best settings of various options. Append best settings to config.txt. if (tune_config) { @@ -384,13 +385,13 @@ void Tune::tune() { // If user gave us an fft-spec, use that to time options if (!args->fftSpec.empty()) { - defaultShape = &shapes[0]; + defaultShape = shapes.data(); if (shapes[0].fft_type == FFT64) { defaultFFTShape = shapes[0]; - time_FFTs = 1; + time_FFTs = true; } else { defaultNTTShape = shapes[0]; - time_NTTs = 1; + time_NTTs = true; } } // If user specified FP64-timings, time a wavefront exponent using an 7.5M FFT @@ -409,30 +410,30 @@ void Tune::tune() { else { log("Checking whether this GPU is better suited for double-precision FFTs or integer NTTs.\n"); defaultFFTShape = FFTShape(FFT64, 512, 16, 512); - FFTConfig fft{defaultFFTShape, 101, CARRY_32}; - double fp64_time = Gpu::make(141000001, shared, fft, {}, false)->timePRP(quick); + FFTConfig const fft{defaultFFTShape, 101, CARRY_32}; + double const fp64_time = Gpu::make(141000001, shared, fft, {}, false)->timePRP(quick); log("Time for FP64 FFT %12s is %6.1f\n", fft.spec().c_str(), fp64_time); defaultNTTShape = FFTShape(FFT3161, 512, 8, 512); - FFTConfig ntt{defaultNTTShape, 202, CARRY_AUTO}; - double ntt_time = Gpu::make(141000001, shared, ntt, {}, false)->timePRP(quick); + FFTConfig const ntt{defaultNTTShape, 202, CARRY_AUTO}; + double const ntt_time = Gpu::make(141000001, shared, ntt, {}, false)->timePRP(quick); log("Time for M31*M61 NTT %12s is %6.1f\n", ntt.spec().c_str(), ntt_time); if (fp64_time < ntt_time) { defaultShape = &defaultFFTShape; - time_FFTs = 1; + time_FFTs = true; if (fp64_time < 0.80 * ntt_time) { log("FP64 FFTs are significantly faster than integer NTTs. No NTT tuning will be performed.\n"); } else { log("FP64 FFTs are not significantly faster than integer NTTs. NTT tuning will be performed.\n"); - time_NTTs = 1; + time_NTTs = true; } } else { defaultShape = &defaultNTTShape; - time_NTTs = 1; + time_NTTs = true; if (fp64_time > 1.20 * ntt_time) { log("FP64 FFTs are significantly slower than integer NTTs. No FP64 tuning will be performed.\n"); } else { log("FP64 FFTs are not significantly slower than integer NTTs. FP64 tuning will be performed.\n"); - time_FFTs = 1; + time_FFTs = true; } } } @@ -442,28 +443,28 @@ void Tune::tune() { log("Please read config.txt after -tune completes.\n"); log("\n"); - u32 variant = (defaultShape == &defaultFFTShape) ? 101 : 202; + u32 const variant = (defaultShape == &defaultFFTShape) ? 101 : 202; //GW: if fft spec on the command line specifies a variant then we should use that variant (I get some interesting results with 000 vs 101 vs 201 vs 202 likely due to rocm optimizer) // IN_WG/SIZEX, OUT_WG/SIZEX, PAD, MIDDLE_IN/OUT_LDS_TRANSPOSE apply only if INPLACE=0 - u32 current_inplace = args->value("INPLACE", 0); + u32 const current_inplace = args->value("INPLACE", 0); args->flags["INPLACE"] = to_string(0); // Find best IN_WG,IN_SIZEX,OUT_WG,OUT_SIZEX settings if (!time_inplace_only) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_in_wg = 0; u32 best_in_sizex = 0; - u32 current_in_wg = args->value("IN_WG", 128); - u32 current_in_sizex = args->value("IN_SIZEX", 16); + u32 const current_in_wg = args->value("IN_WG", 128); + u32 const current_in_sizex = args->value("IN_SIZEX", 16); double best_cost = -1.0; double current_cost = -1.0; - for (u32 in_wg : {64, 128, 256}) { - for (u32 in_sizex : {8, 16, 32}) { + for (u32 const in_wg : {64, 128, 256}) { + for (u32 const in_sizex : {8, 16, 32}) { args->flags["IN_WG"] = to_string(in_wg); args->flags["IN_SIZEX"] = to_string(in_sizex); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using IN_WG=%u, IN_SIZEX=%u is %6.1f\n", fft.spec().c_str(), in_wg, in_sizex, cost); if (in_wg == current_in_wg && in_sizex == current_in_sizex) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_in_wg = in_wg; best_in_sizex = in_sizex; } @@ -477,15 +478,15 @@ void Tune::tune() { u32 best_out_wg = 0; u32 best_out_sizex = 0; - u32 current_out_wg = args->value("OUT_WG", 128); - u32 current_out_sizex = args->value("OUT_SIZEX", 16); + u32 const current_out_wg = args->value("OUT_WG", 128); + u32 const current_out_sizex = args->value("OUT_SIZEX", 16); best_cost = -1.0; current_cost = -1.0; - for (u32 out_wg : {64, 128, 256}) { - for (u32 out_sizex : {8, 16, 32}) { + for (u32 const out_wg : {64, 128, 256}) { + for (u32 const out_sizex : {8, 16, 32}) { args->flags["OUT_WG"] = to_string(out_wg); args->flags["OUT_SIZEX"] = to_string(out_sizex); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using OUT_WG=%u, OUT_SIZEX=%u is %6.1f\n", fft.spec().c_str(), out_wg, out_sizex, cost); if (out_wg == current_out_wg && out_sizex == current_out_sizex) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_out_wg = out_wg; best_out_sizex = out_sizex; } @@ -500,15 +501,15 @@ void Tune::tune() { // Find best PAD setting. Default is 256 bytes for AMD, 0 for all others. if (!time_inplace_only) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_pad = 0; - u32 current_pad = args->value("PAD", AMDGPU ? 256 : 0); + u32 const current_pad = args->value("PAD", AMDGPU ? 256 : 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 pad : {0, 64, 128, 256, 512}) { + for (u32 const pad : {0, 64, 128, 256, 512}) { args->flags["PAD"] = to_string(pad); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using PAD=%u is %6.1f\n", fft.spec().c_str(), pad, cost); if (pad == current_pad) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_pad = pad; } @@ -520,15 +521,15 @@ void Tune::tune() { // Find best MIDDLE_IN_LDS_TRANSPOSE setting if (!time_inplace_only) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_in_lds_transpose = 0; - u32 current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); + u32 const current_middle_in_lds_transpose = args->value("MIDDLE_IN_LDS_TRANSPOSE", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 middle_in_lds_transpose : {0, 1}) { + for (u32 const middle_in_lds_transpose : {0, 1}) { args->flags["MIDDLE_IN_LDS_TRANSPOSE"] = to_string(middle_in_lds_transpose); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MIDDLE_IN_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_in_lds_transpose, cost); if (middle_in_lds_transpose == current_middle_in_lds_transpose) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_in_lds_transpose = middle_in_lds_transpose; } @@ -540,15 +541,15 @@ void Tune::tune() { // Find best MIDDLE_OUT_LDS_TRANSPOSE setting if (!time_inplace_only) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_middle_out_lds_transpose = 0; - u32 current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); + u32 const current_middle_out_lds_transpose = args->value("MIDDLE_OUT_LDS_TRANSPOSE", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 middle_out_lds_transpose : {0, 1}) { + for (u32 const middle_out_lds_transpose : {0, 1}) { args->flags["MIDDLE_OUT_LDS_TRANSPOSE"] = to_string(middle_out_lds_transpose); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MIDDLE_OUT_LDS_TRANSPOSE=%u is %6.1f\n", fft.spec().c_str(), middle_out_lds_transpose, cost); if (middle_out_lds_transpose == current_middle_out_lds_transpose) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_middle_out_lds_transpose = middle_out_lds_transpose; } @@ -561,18 +562,18 @@ void Tune::tune() { // If only timing INPLACE=1 options, then set INPLACE if (time_inplace_only) { args->flags["INPLACE"] = to_string(1); - newConfigKeyVals.push_back({"INPLACE", 1}); + newConfigKeyVals.emplace_back("INPLACE", 1); } // Find best INPLACE setting else { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_inplace = 0; double best_cost = -1.0; double current_cost = -1.0; - for (u32 inplace : {0, 1}) { + for (u32 const inplace : {0, 1}) { args->flags["INPLACE"] = to_string(inplace); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using INPLACE=%u is %6.1f\n", fft.spec().c_str(), inplace, cost); if (inplace == current_inplace) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_inplace = inplace; } @@ -583,20 +584,20 @@ void Tune::tune() { } // Find best LOADS/STORES settings - if (1) { + if (true) { u32 loads = args->value("LOADS", 0); u32 stores = args->value("STORES", 0); // Find best FFT data LOADS setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_fft_load = 0; double best_cost = -1.0; - for (u32 fft_load : {0, 1, 2, 3, 4}) { + for (u32 const fft_load : {0, 1, 2, 3, 4}) { if (fft_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10 * 10 + fft_load); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FFT load=%u is %6.1f\n", fft.spec().c_str(), fft_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_load = fft_load; } } @@ -606,15 +607,15 @@ void Tune::tune() { } // Find best FFT data STORES setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_fft_store = 0; double best_cost = -1.0; - for (u32 fft_store : {0, 1, 2, 3, 4}) { + for (u32 const fft_store : {0, 1, 2, 3, 4}) { if (fft_store >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["STORES"] = to_string(stores / 10 * 10 + fft_store); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FFT store=%u is %6.1f\n", fft.spec().c_str(), fft_store, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fft_store = fft_store; } } @@ -624,18 +625,18 @@ void Tune::tune() { } // Find best carryShuttle LOADS/STORES settings - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_cs_load = 0, best_cs_store = 0; double best_cost = -1.0; - for (u32 cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load with L2 store + for (u32 const cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load with L2 store if (cs >= 2 && (!NVIDIAGPU || NO_ASM)) continue; - u32 cs_load = cs == 0 ? 0 : cs == 1 ? 1 : 4; - u32 cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; + u32 const cs_load = cs == 0 ? 0 : cs == 1 ? 1 : 4; + u32 const cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; args->flags["LOADS"] = to_string(loads / 100 * 100 + cs_load * 10 + loads % 10); args->flags["STORES"] = to_string(stores / 100 * 100 + cs_store * 10 + stores % 10); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using carry shuttle load=%u, store=%u is %6.1f\n", fft.spec().c_str(), cs_load, cs_store, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_cs_load = cs_load; best_cs_store = cs_store; } } @@ -647,15 +648,15 @@ void Tune::tune() { } // Find best TRIG frequently used data LOADS setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_trig_load = 0; double best_cost = -1.0; - for (u32 trig_load : {0, 5}) { + for (u32 const trig_load : {0, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 1000 * 1000 + trig_load * 100 + loads % 100); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig frequently used load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } } @@ -665,15 +666,15 @@ void Tune::tune() { } // Find best TRIG several uses data LOADS setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_trig_load = 0; double best_cost = -1.0; - for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { + for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10000 * 10000 + trig_load * 1000 + loads % 1000); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig several uses load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } } @@ -683,15 +684,15 @@ void Tune::tune() { } // Find best TRIG used once data LOADS setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_trig_load = 0; double best_cost = -1.0; - for (u32 trig_load : {0, 1, 2, 3, 4, 5}) { + for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(trig_load * 10000 + loads % 10000); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using Trig used once load=%u is %6.1f\n", fft.spec().c_str(), trig_load, cost); if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_trig_load = trig_load; } } @@ -708,16 +709,16 @@ void Tune::tune() { } // Find best FAST_BARRIER setting - if (1 /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_fast_barrier = 0; - u32 current_fast_barrier = args->value("FAST_BARRIER", 0); + u32 const current_fast_barrier = args->value("FAST_BARRIER", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 fast_barrier : {0, 1}) { + for (u32 const fast_barrier : {0, 1}) { args->flags["FAST_BARRIER"] = to_string(fast_barrier); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using FAST_BARRIER=%u is %6.1f\n", fft.spec().c_str(), fast_barrier, cost); if (fast_barrier == current_fast_barrier) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_fast_barrier = fast_barrier; } @@ -728,16 +729,16 @@ void Tune::tune() { } // Find best TAIL_KERNELS setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_kernels = 0; - u32 current_tail_kernels = args->value("TAIL_KERNELS", 2); + u32 const current_tail_kernels = args->value("TAIL_KERNELS", 2); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_kernels : {0, 1, 2, 3}) { + for (u32 const tail_kernels : {0, 1, 2, 3}) { args->flags["TAIL_KERNELS"] = to_string(tail_kernels); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_KERNELS=%u is %6.1f\n", fft.spec().c_str(), tail_kernels, cost); if (tail_kernels == current_tail_kernels) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_kernels = tail_kernels; } @@ -752,15 +753,15 @@ void Tune::tune() { // Find best TAIL_TRIGS setting if (time_FFTs) { - FFTConfig fft{defaultFFTShape, 101, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{defaultFFTShape, 101, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS", 2); + u32 const current_tail_trigs = args->value("TAIL_TRIGS", 2); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1, 2}) { + for (u32 const tail_trigs : {0, 1, 2}) { args->flags["TAIL_TRIGS"] = to_string(tail_trigs); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -774,14 +775,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u64 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS31", 0); + u32 const current_tail_trigs = args->value("TAIL_TRIGS31", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1}) { + for (u32 const tail_trigs : {0, 1}) { args->flags["TAIL_TRIGS31"] = to_string(tail_trigs); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS31=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -795,14 +796,14 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u64 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 const exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS32", 2); + u32 const current_tail_trigs = args->value("TAIL_TRIGS32", 2); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1, 2}) { + for (u32 const tail_trigs : {0, 1, 2}) { args->flags["TAIL_TRIGS32"] = to_string(tail_trigs); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS32=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -816,14 +817,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF61) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u64 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tail_trigs = 0; - u32 current_tail_trigs = args->value("TAIL_TRIGS61", 0); + u32 const current_tail_trigs = args->value("TAIL_TRIGS61", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tail_trigs : {0, 1}) { + for (u32 const tail_trigs : {0, 1}) { args->flags["TAIL_TRIGS61"] = to_string(tail_trigs); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TAIL_TRIGS61=%u is %6.1f\n", fft.spec().c_str(), tail_trigs, cost); if (tail_trigs == current_tail_trigs) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tail_trigs = tail_trigs; } @@ -835,15 +836,15 @@ void Tune::tune() { // Find best TABMUL_CHAIN setting if (time_FFTs) { - FFTConfig fft{defaultFFTShape, 101, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + FFTConfig const fft{defaultFFTShape, 101, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN"] = to_string(tabmul_chain); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -857,14 +858,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u64 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN31", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN31", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN31"] = to_string(tabmul_chain); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN31=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -878,14 +879,14 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u64 exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 const exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN32", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN32", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN32"] = to_string(tabmul_chain); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN32=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -899,14 +900,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF61) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u64 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_tabmul_chain = 0; - u32 current_tabmul_chain = args->value("TABMUL_CHAIN61", 0); + u32 const current_tabmul_chain = args->value("TABMUL_CHAIN61", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 tabmul_chain : {0, 1}) { + for (u32 const tabmul_chain : {0, 1}) { args->flags["TABMUL_CHAIN61"] = to_string(tabmul_chain); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using TABMUL_CHAIN61=%u is %6.1f\n", fft.spec().c_str(), tabmul_chain, cost); if (tabmul_chain == current_tabmul_chain) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_tabmul_chain = tabmul_chain; } @@ -920,14 +921,14 @@ void Tune::tune() { if (time_NTTs) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.NTT_GF31) fft = FFTConfig(FFTShape(FFT3161, 512, 8, 512), 202, CARRY_AUTO); - u64 exponent = primes.prevPrime(fft.maxExp()); + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_modm31 = 0; - u32 current_modm31 = args->value("MODM31", 0); + u32 const current_modm31 = args->value("MODM31", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 modm31 : {0, 1, 2}) { + for (u32 const modm31 : {0, 1, 2}) { args->flags["MODM31"] = to_string(modm31); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using MODM31=%u is %6.1f\n", fft.spec().c_str(), modm31, cost); if (modm31 == current_modm31) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_modm31 = modm31; } @@ -938,16 +939,16 @@ void Tune::tune() { } // Find best UNROLL_W setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_unroll_w = 0; - u32 current_unroll_w = args->value("UNROLL_W", AMDGPU ? 0 : 1); + u32 const current_unroll_w = args->value("UNROLL_W", AMDGPU ? 0 : 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 unroll_w : {0, 1}) { + for (u32 const unroll_w : {0, 1}) { args->flags["UNROLL_W"] = to_string(unroll_w); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using UNROLL_W=%u is %6.1f\n", fft.spec().c_str(), unroll_w, cost); if (unroll_w == current_unroll_w) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_unroll_w = unroll_w; } @@ -958,16 +959,16 @@ void Tune::tune() { } // Find best UNROLL_H setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_unroll_h = 0; - u32 current_unroll_h = args->value("UNROLL_H", AMDGPU && defaultShape->height >= 1024 ? 0 : 1); + u32 const current_unroll_h = args->value("UNROLL_H", AMDGPU && defaultShape->height >= 1024 ? 0 : 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 unroll_h : {0, 1}) { + for (u32 const unroll_h : {0, 1}) { args->flags["UNROLL_H"] = to_string(unroll_h); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using UNROLL_H=%u is %6.1f\n", fft.spec().c_str(), unroll_h, cost); if (unroll_h == current_unroll_h) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_unroll_h = unroll_h; } @@ -978,16 +979,16 @@ void Tune::tune() { } // Find best ZEROHACK_W setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_zerohack_w = 0; - u32 current_zerohack_w = args->value("ZEROHACK_W", 1); + u32 const current_zerohack_w = args->value("ZEROHACK_W", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 zerohack_w : {0, 1}) { + for (u32 const zerohack_w : {0, 1}) { args->flags["ZEROHACK_W"] = to_string(zerohack_w); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using ZEROHACK_W=%u is %6.1f\n", fft.spec().c_str(), zerohack_w, cost); if (zerohack_w == current_zerohack_w) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_zerohack_w = zerohack_w; } @@ -998,16 +999,16 @@ void Tune::tune() { } // Find best ZEROHACK_H setting - if (1) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_zerohack_h = 0; - u32 current_zerohack_h = args->value("ZEROHACK_H", 1); + u32 const current_zerohack_h = args->value("ZEROHACK_H", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 zerohack_h : {0, 1}) { + for (u32 const zerohack_h : {0, 1}) { args->flags["ZEROHACK_H"] = to_string(zerohack_h); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using ZEROHACK_H=%u is %6.1f\n", fft.spec().c_str(), zerohack_h, cost); if (zerohack_h == current_zerohack_h) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_zerohack_h = zerohack_h; } @@ -1018,16 +1019,16 @@ void Tune::tune() { } // Find best WMUL setting - if (1 && defaultShape->width != 4096) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (true && defaultShape->width != 4096) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_wmul = 0; - u32 current_wmul = args->value("WMUL", 2); + u32 const current_wmul = args->value("WMUL", 2); double best_cost = -1.0; double current_cost = -1.0; - for (u32 wmul : {1, 2, 4}) { + for (u32 const wmul : {1, 2, 4}) { args->flags["WMUL"] = to_string(wmul); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using WMUL=%u is %6.1f\n", fft.spec().c_str(), wmul, cost); if (wmul == current_wmul) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_wmul = wmul; } @@ -1039,16 +1040,16 @@ void Tune::tune() { // Find best CUDA compiler options #if CUDA_BACKEND - if (0) { - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (false) { + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_noreg = 0; - u32 current_noreg = args->value("NOREG", 0); + u32 const current_noreg = args->value("NOREG", 0); double best_cost = -1.0; double current_cost = -1.0; - for (u32 noreg : {0, 1}) { + for (u32 const noreg : {0, 1}) { args->flags["NOREG"] = to_string(noreg); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using NOREG=%u is %6.1f\n", fft.spec().c_str(), noreg, cost); if (noreg == current_noreg) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_noreg = noreg; } @@ -1060,16 +1061,16 @@ void Tune::tune() { #endif // Find best BIGLIT setting - if (0 && time_FFTs) { // Deprecated - FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; - u64 exponent = primes.prevPrime(fft.maxExp()); + if (false && time_FFTs) { // Deprecated + FFTConfig const fft{*defaultShape, variant, CARRY_AUTO}; + u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_biglit = 0; - u32 current_biglit = args->value("BIGLIT", 1); + u32 const current_biglit = args->value("BIGLIT", 1); double best_cost = -1.0; double current_cost = -1.0; - for (u32 biglit : {0, 1}) { + for (u32 const biglit : {0, 1}) { args->flags["BIGLIT"] = to_string(biglit); - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); log("Time for %12s using BIGLIT=%u is %6.1f\n", fft.spec().c_str(), biglit, cost); if (biglit == current_biglit) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_biglit = biglit; } @@ -1081,7 +1082,7 @@ void Tune::tune() { // Output new settings to config.txt File config = File::openAppend("config.txt"); - if (newConfigKeyVals.size()) { + if (!newConfigKeyVals.empty()) { config.write("\n# New settings based on a -tune run."); for (u32 i = 0; i < newConfigKeyVals.size(); ++i) { config.write(i == 0 ? "\n -use " : ","); @@ -1089,7 +1090,7 @@ void Tune::tune() { } config.write("\n"); } - if (suggestedConfigKeyVals.size()) { + if (!suggestedConfigKeyVals.empty()) { config.write("\n# These settings were slightly faster in a -tune run."); config.write("\n# It is suggested that each setting be timed over a longer duration to see if the setting really is faster."); for (u32 i = 0; i < suggestedConfigKeyVals.size(); ++i) { @@ -1118,11 +1119,11 @@ void Tune::tune() { int skip_some_WH_variants = 1; // 0 = skip nothing, 1 = skip slower widths/heights unless they have better Z, 2 = only run fastest widths/heights // The width = height = 512 FFT shape is so good, we probably don't need to time the width = 1024, height = 256 shape. - bool skip_1K_256 = 1; + bool skip_1K_256 = true; // make command line args for this? skip_some_WH_variants = 2; // should default be 1?? -skip_1K_256 = 0; +skip_1K_256 = false; // For each width, time the 001, 101, and 201 FP64 variants to find the fastest width variant. // In an ideal world we'd use the -time feature and look at the kCarryFused timing. Then we'd save this info in config.txt or tune.txt. @@ -1143,10 +1144,10 @@ skip_1K_256 = 0; if ((shape.fft_type == FFT3261 || shape.fft_type == FFT323161 || shape.fft_type == FFT3231 || shape.fft_type == FFT32) && !time_FP32) continue; // Time an exponent that's good for all variants and carry-config. - u64 exponent = primes.prevPrime(FFTConfig{shape, shape.width <= 1024 ? 0u : 100u, CARRY_32}.maxExp()); + u64 const exponent = primes.prevPrime(FFTConfig{shape, shape.width <= 1024 ? 0u : 100u, CARRY_32}.maxExp()); u32 adjusted_quick = (exponent < 50000000) ? quick - 1 : (exponent < 170000000) ? quick : (exponent < 350000000) ? quick + 1 : quick + 2; - if (adjusted_quick < 1) adjusted_quick = 1; - if (adjusted_quick > 10) adjusted_quick = 10; + adjusted_quick = std::max(adjusted_quick, 1); + adjusted_quick = std::min(adjusted_quick, 10); // Loop through all possible variants for (u32 variant = 0; variant <= LAST_VARIANT; variant = next_variant (variant)) { @@ -1170,7 +1171,7 @@ skip_1K_256 = 0; // Reject shapes that won't be used to test exponents in the user's desired range { - FFTConfig fft{shape, variant, CARRY_AUTO}; + FFTConfig const fft{shape, variant, CARRY_AUTO}; if (fft.maxExp() < min_exponent) continue; if (fft.maxExp() > 2*max_exponent) continue; if (shape.fft_type == FFT64 && fft.maxExp() > 1.2*max_exponent) continue; @@ -1192,12 +1193,12 @@ skip_1K_256 = 0; if (auto it = fastest_width_variants.find(shape.width); it != fastest_width_variants.end()) { fastest_width = it->second; } else { - FFTShape test = FFTShape(FFT64, shape.width, 12, 256); + FFTShape const test = FFTShape(FFT64, shape.width, 12, 256); double cost, min_cost = -1.0; for (u32 w = 0; w < N_VARIANT_W; w++) { if (w == 0 && !AMDGPU) continue; if (w == 0 && test.width > 1024) continue; - FFTConfig fft{test, variant_WMH (w, 0, 1), CARRY_32}; + FFTConfig const fft{test, variant_WMH (w, 0, 1), CARRY_32}; cost = Gpu::make(primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(adjusted_quick); log("Fast width search %6.1f %12s\n", cost, fft.spec().c_str()); if (min_cost < 0.0 || cost < min_cost) { min_cost = cost; fastest_width = w; } @@ -1214,12 +1215,12 @@ skip_1K_256 = 0; if (auto it = fastest_height_variants.find(shape.height); it != fastest_height_variants.end()) { fastest_height = it->second; } else { - FFTShape test = FFTShape(FFT64, shape.height, 12, shape.height); + FFTShape const test = FFTShape(FFT64, shape.height, 12, shape.height); double cost, min_cost = -1.0; for (u32 h = 0; h < N_VARIANT_H; h++) { if (h == 0 && !AMDGPU) continue; if (h == 0 && test.height > 1024) continue; - FFTConfig fft{test, variant_WMH (1, 0, h), CARRY_32}; + FFTConfig const fft{test, variant_WMH (1, 0, h), CARRY_32}; cost = Gpu::make(primes.prevPrime(fft.maxExp()), shared, fft, {}, false)->timePRP(quick); log("Fast height search %6.1f %12s\n", cost, fft.spec().c_str()); if (min_cost < 0.0 || cost < min_cost) { min_cost = cost; fastest_height = h; } @@ -1245,13 +1246,13 @@ skip_1K_256 = 0; } for (auto carry : carryToTest) { - FFTConfig fft{shape, variant, carry}; + FFTConfig const fft{shape, variant, carry}; // Skip middle = 1, CARRY_32 if maximum exponent would be the same as middle = 0, CARRY_32 if (variant_M(variant) > 0 && carry == CARRY_32 && fft.maxExp() <= FFTConfig{shape, variant - 10, CARRY_32}.maxExp()) continue; - double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); - bool isUseful = TuneEntry{cost, fft}.update(results); + double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + bool const isUseful = TuneEntry{.cost=cost, .fft=fft}.update(results); log("%c %6.1f %12s %9" PRIu64 "\n", isUseful ? '*' : ' ', cost, fft.spec().c_str(), fft.maxExp()); if (isUseful) TuneEntry::writeTuneFile(results); } From 294cc485ac8cf53c8b69144a3039832eda573849 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 29 Jun 2026 18:40:54 +0000 Subject: [PATCH 085/214] Workaround AMD's Windows OpenCL compiler whining about always true if statements when WMUL=1. --- src/cl/carryfused.cl | 57 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index d5d52610..9a887a76 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -32,7 +32,9 @@ void spin() { // The last WMUL workgroup's carries have been written to global memory. Now we shuffle WMUL-1 workgroups carries up using local memory. void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) { // If WMUL is one, there is no shuffling of carries - if (WMUL == 1) return; + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead of the clean looking if statement below we use the uglier #if + //if (WMUL == 1) return; +#if WMUL > 1 const u32 lds_i64s = LDS_BYTES / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL line local i64 *lds = (local i64 *) lds2; @@ -80,12 +82,16 @@ void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl and no trailing bar() is required. if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; } + +#endif } // The last WMUL workgroup's carries have been written to global memory. Now we shuffle WMUL-1 workgroup carries up using local memory. void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) { // If WMUL is one, there is no shuffling of carries - if (WMUL == 1) return; + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead of the clean looking if statement below we use the uglier #if + //if (WMUL == 1) return; +#if WMUL > 1 const u32 lds_i32s = LDS_BYTES / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL line local i32 *lds = (local i32 *) lds2; @@ -99,6 +105,8 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) bar(); // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl and no trailing bar() is required. if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; + +#endif } @@ -199,7 +207,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -406,7 +419,12 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -619,7 +637,12 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -836,7 +859,12 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -1068,7 +1096,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -1327,7 +1360,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -1582,7 +1620,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -1832,7 +1875,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready @@ -2114,7 +2162,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. + // AMD's OpenCL Windows compiler generates warnings about always true if statements for WMUL-1. So instead an #if is required +#if WMUL == 1 + if (gr < H) { +#else if (gr < H / WMUL && me >= (WMUL-1) * G_W) { +#endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready From 14a981cbf24cdfdadba4d3d4592a854470791e68 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sat, 18 Jul 2026 19:41:06 -0600 Subject: [PATCH 086/214] CUDA backend: unload CUmodules via refcount to fix OOM leak The CUDA shim never called cuModuleUnload, so every Gpu::make() leaked its full set of ~36 kernel modules into the single process-lifetime CUDA context. Since Gpu::make() runs once per work unit (Task.cpp) and dozens of times during tuning (tune.cpp), device memory grew unbounded until cuLaunchKernel failed with CUDA_ERROR_OUT_OF_MEMORY. clReleaseProgram couldn't simply unload the module because loadAux() releases the program while the derived kernel's CUfunction is still in use (OpenCL keeps the program alive via clCreateKernel's implicit retain). Reference-count the module instead: the owning program holds one ref (at load), each kernel created from it holds another, and the module is unloaded only when the count reaches zero. Co-Authored-By: Claude Opus 4.8 --- src/cuda/clwrap_cuda.cpp | 59 ++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 5df25791..672180af 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -37,6 +37,41 @@ static void ensureContextCurrent() { } } +// Reference-count CUmodules so they get unloaded once nothing uses them. +// +// In OpenCL, clCreateKernel retains the program, so the underlying code object +// stays alive until BOTH the program and every kernel derived from it are +// released. PRPLL relies on this: KernelCompiler::loadAux() creates a kernel, +// then releases the program while the kernel keeps running. We therefore cannot +// unload the module in clReleaseProgram — a live CUfunction would be invalidated. +// +// Instead we count references: the owning program holds one ref (set when the +// module is loaded), and each kernel created from it holds one more. The module +// is unloaded when the count reaches zero. This is essential because Gpu::make() +// builds a fresh set of ~36 kernels per work unit (and dozens of times during +// tuning), all into the single process-lifetime CUDA context. Without unloading, +// device memory grows unbounded and eventually cuLaunchKernel fails with +// CUDA_ERROR_OUT_OF_MEMORY. +// +// Loading is single-threaded (KernelCompiler's async path is disabled), so a +// plain map without locking is sufficient. +static std::map g_moduleRefCount; + +static void moduleRetain(CUmodule m) { + if (m) { ++g_moduleRefCount[m]; } +} + +static void moduleRelease(CUmodule m) { + if (!m) return; + auto it = g_moduleRefCount.find(m); + if (it == g_moduleRefCount.end()) return; // untracked module — leave as-is + if (--it->second <= 0) { + ensureContextCurrent(); + cuModuleUnload(m); + g_moduleRefCount.erase(it); + } +} + // Global state for CUDA initialization static bool g_cudaInitialized = false; static void ensureCudaInit() { @@ -123,13 +158,12 @@ int clReleaseContext(cl_context ctx) { int clReleaseProgram(cl_program p) { if (p) { - // NOTE: Do NOT unload the module here. PRPLL's loadAux() gets a kernel from - // the program, then releases the program. The kernel's CUfunction remains valid - // only while the CUmodule is loaded. In OpenCL, clCreateKernel retains the - // program. In our CUDA shim, we simply never unload modules — they persist for - // the process lifetime. This is safe because PRPLL creates a fixed set of kernels - // at startup and uses them until exit. - // if (p->moduleLoaded) cuModuleUnload(p->module); + // Drop the program's reference to its module. The module is unloaded only once + // every kernel created from it has also been released (see moduleRelease and the + // refcount rationale near the top of this file). This lets loadAux() release the + // program while keeping the kernel's CUfunction valid, matching OpenCL semantics, + // without leaking a module per Gpu::make(). + if (p->moduleLoaded) { moduleRelease(p->module); } delete p; } return CL_SUCCESS; @@ -172,6 +206,7 @@ cl_program clCreateProgramWithBinary(cl_context ctx, unsigned nDevices, const cl CUresult r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); if (r == CUDA_SUCCESS) { prog->moduleLoaded = true; + moduleRetain(prog->module); // program owns one reference if (binaryStatus) binaryStatus[0] = CL_SUCCESS; } else { fprintf(stderr, "cuModuleLoadData from cache failed: %d, PTX size=%zu\n", (int)r, lengths[0]); @@ -440,6 +475,7 @@ cl_program clLinkProgram(cl_context ctx, unsigned nDevices, const cl_device_id*, return nullptr; } linked->moduleLoaded = true; + moduleRetain(linked->module); // program owns one reference // Dump PTX to file when PRPLL_DUMP_PTX is set (e.g., PRPLL_DUMP_PTX=kernel) // Creates files like kernel_0.ptx, kernel_1.ptx, etc. @@ -477,6 +513,7 @@ int clBuildProgram(cl_program prog, unsigned nDevices, const cl_device_id* devic CUresult r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); if (r != CUDA_SUCCESS) return CL_BUILD_PROGRAM_FAILURE; prog->moduleLoaded = true; + moduleRetain(prog->module); // program owns one reference return CL_SUCCESS; } @@ -523,10 +560,11 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { if (r != CUDA_SUCCESS) { fprintf(stderr, "cuModuleGetFunction('%s') failed: %d, moduleLoaded=%d, module=%p\n", name, (int)r, prog->moduleLoaded, (void*)prog->module); - delete k; + delete k; // never retained the module, so nothing to release if (err) *err = CL_INVALID_KERNEL_NAME; return nullptr; } + moduleRetain(k->parentModule); // kernel keeps the module alive past clReleaseProgram // Shared memory carveout: default adaptive carveout is optimal for mixed kernel workloads. @@ -575,7 +613,10 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { } int clReleaseKernel(cl_kernel k) { - delete k; + if (k) { + moduleRelease(k->parentModule); + delete k; + } return CL_SUCCESS; } From 485fdadcff4a6a28c4bc3f76ff78c939d0cd28df Mon Sep 17 00:00:00 2001 From: george Date: Mon, 20 Jul 2026 00:33:00 +0000 Subject: [PATCH 087/214] Created an openCL-like interface to support CUDA graphs. Of course, the interface is a no-op in openCL, but uses CUDA graphs with the CUDA backend. Also, two-dimensional kernel launches are supported (not presently used). Markers now enqueued without timing - a minor optimization. --- src/clwrap.cpp | 19 ++++++++++++-- src/clwrap.h | 8 ++++-- src/cuda/clwrap_cuda.cpp | 54 +++++++++++++++++++++++++++++++++----- src/cuda/opencl_compat.cuh | 4 +-- src/cuda/tinycuda.h | 17 ++++++++++++ src/tinycl.h | 16 +++++++++-- 6 files changed, 103 insertions(+), 15 deletions(-) diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 1152a7bb..6d9a6346 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -227,6 +227,7 @@ void release(cl_mem buf) { CHECK1(clReleaseMemObject(buf)); } void release(cl_queue queue) { CHECK1(clReleaseCommandQueue(queue)); } void release(cl_kernel k) { CHECK1(clReleaseKernel(k)); } void release(cl_event event) { CHECK1(clReleaseEvent(event)); } +void release(cl_graph graph) { CHECK1(clReleaseGraph(graph));} Program loadSource(cl_context context, const string &source) { const char *ptr = source.c_str(); @@ -326,11 +327,13 @@ void flush( cl_queue q) { CHECK1(clFlush(q)); } void finish(cl_queue q) { CHECK1(clFinish(q)); } EventHolder run(cl_queue queue, cl_kernel kernel, - size_t groupSize, size_t workSize, + size_t groupSizeX, size_t workSizeX, size_t workSizeY, vector&& waits, const string &name, bool genEvent) { cl_event event{}; - CHECK2(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &workSize, &groupSize, + size_t workSizes[2] = {workSizeX, workSizeY}; + size_t groupSizes[2] = {groupSizeX, 1}; + CHECK2(clEnqueueNDRangeKernel(queue, kernel, workSizeY == 1 ? 1 : 2, NULL, workSizes, groupSizes, waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr), name.c_str()); return genEvent ? EventHolder{event} : EventHolder{}; @@ -451,3 +454,15 @@ cl_device_id getQueueDevice(cl_command_queue q) { CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_DEVICE, sizeof(id), &id, 0)); return id; } + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features. +// These routines are defined in clwrap_cuda.cpp for the CUDA translation of openCL. +// The dummy implementation below is for the native openCL builds. + +#ifndef CUDA_BACKEND +bool clIsGraphSupported(cl_device_id dev) { return 0; } +int clGraphBeginRecording(cl_command_queue q) { return CL_INVALID_VALUE; } +int clGraphEndRecording(cl_command_queue q, cl_graph* g) { return CL_INVALID_VALUE; } +int clGraphLaunch(cl_graph g) { return CL_INVALID_VALUE; } +int clReleaseGraph(cl_graph g) { return CL_INVALID_VALUE; } +#endif diff --git a/src/clwrap.h b/src/clwrap.h index 41b2bb93..018eb84c 100644 --- a/src/clwrap.h +++ b/src/clwrap.h @@ -15,13 +15,13 @@ using cl_queue = cl_command_queue; - void release(cl_context context); void release(cl_kernel k); void release(cl_mem buf); void release(cl_program program); void release(cl_queue queue); void release(cl_event event); +void release(cl_graph graph); template struct Deleter { @@ -36,6 +36,7 @@ template<> struct default_delete : public Deleter {}; template<> struct default_delete : public Deleter {}; template<> struct default_delete : public Deleter {}; template<> struct default_delete : public Deleter {}; +template<> struct default_delete : public Deleter {}; } template using Holder = std::unique_ptr >; @@ -43,6 +44,7 @@ template using Holder = std::unique_ptr >; using QueueHolder = std::unique_ptr; using KernelHolder = std::unique_ptr; using EventHolder = std::unique_ptr; +using GraphHolder = std::unique_ptr; using Program = std::unique_ptr; class Context; @@ -98,7 +100,7 @@ cl_queue makeQueue(cl_device_id d, cl_context c, bool enableProfile); void flush( cl_queue q); void finish(cl_queue q); -EventHolder run(cl_queue queue, cl_kernel kernel, size_t groupSize, size_t workSize, +EventHolder run(cl_queue queue, cl_kernel kernel, size_t groupSizeX, size_t workSizeX, size_t workSizeY, vector&& waits, const string &name, bool genEvent); EventHolder read(cl_queue queue, vector&& waits, @@ -136,3 +138,5 @@ cl_context getQueueContext(cl_command_queue q); // Buffers that are nullptr or zero-size are skipped. void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers); #endif + + diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 5df25791..796f2128 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -674,9 +674,12 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, if (!q || !k) return CL_INVALID_VALUE; ensureContextCurrent(); - size_t gs = globalSize[0]; - size_t ls = localSize ? localSize[0] : 256; - size_t numBlocks = (gs + ls - 1) / ls; + size_t gsX = globalSize[0]; + size_t lsX = localSize ? localSize[0] : 256; + size_t numBlocksX = (gsX + lsX - 1) / lsX; + size_t gsY = (workDim > 1) ? globalSize[1] : 1; + size_t lsY = (workDim > 1) ? localSize[1] : 1; + size_t numBlocksY = (gsY + lsY - 1) / lsY; // Build args array void* argPtrs[_cl_kernel::MAX_ARGS]; @@ -693,7 +696,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, ev->hasTimings = true; ev->commandType = CL_COMMAND_NDRANGE_KERNEL; cuEventRecord(ev->start, q->stream); - CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + CUresult r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); cuEventRecord(ev->end, q->stream); *event = ev; return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; @@ -708,7 +711,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, if (!pStart) { cuEventCreate(&pStart, CU_EVENT_DEFAULT); cuEventCreate(&pEnd, CU_EVENT_DEFAULT); } cuEventRecord(pStart, q->stream); - CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + CUresult r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); cuEventRecord(pEnd, q->stream); cuEventSynchronize(pEnd); float ms = 0; @@ -740,7 +743,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; } - CUresult r = cuLaunchKernel(k->func, numBlocks, 1, 1, ls, 1, 1, 0, q->stream, argPtrs, nullptr); + CUresult r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); if (r != CUDA_SUCCESS) { const char* errName = nullptr; cuGetErrorName(r, &errName); @@ -812,7 +815,7 @@ int clEnqueueMarkerWithWaitList(cl_command_queue q, unsigned nWaits, const cl_ev } if (event) { auto* ev = new _cl_event; - cuEventCreate(&ev->end, CU_EVENT_DEFAULT); + cuEventCreate(&ev->end, CU_EVENT_DISABLE_TIMING); cuEventRecord(ev->end, q->stream); ev->commandType = CL_COMMAND_MARKER; *event = ev; @@ -1137,3 +1140,40 @@ void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) } } + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features + +// Interface to nVidia CUDA graphs feature + +bool clIsGraphSupported(cl_device_id dev) { + int major = 0; + cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev->dev); + return (major >= 6); +} + +int clGraphBeginRecording(cl_command_queue q) { + ensureContextCurrent(); + CUresult r = cuStreamBeginCapture(q->stream, CU_STREAM_CAPTURE_MODE_GLOBAL); + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clGraphEndRecording(cl_command_queue q, cl_graph* graph) { + ensureContextCurrent(); + auto* g = new _cl_graph; + g->queue = q; + CUresult r = cuStreamEndCapture(q->stream, &g->graph); + if (r == CUDA_SUCCESS) r = cuGraphInstantiate(&g->graphExec, g->graph, 0); + *graph = g; + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clGraphLaunch(cl_graph graph) { + ensureContextCurrent(); + CUresult r = cuGraphLaunch(graph->graphExec, graph->queue->stream); + return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; +} + +int clReleaseGraph(cl_graph graph) { + delete graph; + return CL_SUCCESS; +} diff --git a/src/cuda/opencl_compat.cuh b/src/cuda/opencl_compat.cuh index b72ef5e4..6b1a2807 100644 --- a/src/cuda/opencl_compat.cuh +++ b/src/cuda/opencl_compat.cuh @@ -33,8 +33,8 @@ // ---- Work-item functions ---- #define get_local_id(d) ((unsigned int)threadIdx.x) -#define get_group_id(d) ((unsigned int)blockIdx.x) -#define get_local_size(d) ((unsigned int)blockDim.x) +#define get_group_id(d) ((unsigned int)(d == 0 ? blockIdx.x : blockIdx.y)) +#define get_local_size(d) ((unsigned int)(d == 0 ? blockDim.x : blockDim.y)) #define get_global_id(d) ((unsigned int)(blockIdx.x * blockDim.x + threadIdx.x)) #define get_num_groups(d) ((unsigned int)gridDim.x) #define get_global_size(d) ((unsigned int)(gridDim.x * blockDim.x)) diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index d01946df..55c5f79c 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -319,3 +319,20 @@ void clSVMFree(cl_context, void*); int clSetKernelArgSVMPointer(cl_kernel, unsigned, const void*); } + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features + +// cl_graph provides an openCL-like interface for CUgraph and CUgraphExec pair +struct _cl_graph { + CUgraph graph; + CUgraphExec graphExec; + cl_command_queue queue; // Queue to launch the graph on (same as queue that graph was recorded on) + _cl_graph() : graph{}, graphExec{} {} + ~_cl_graph() { if (graph) cuGraphDestroy(graph); if (graphExec) cuGraphExecDestroy(graphExec); } +}; +typedef _cl_graph* cl_graph; +bool clIsGraphSupported(cl_device_id); +int clGraphBeginRecording(cl_command_queue); +int clGraphEndRecording(cl_command_queue, cl_graph*); +int clGraphLaunch(cl_graph); +int clReleaseGraph(cl_graph); diff --git a/src/tinycl.h b/src/tinycl.h index 90fa8404..4fccdf7e 100644 --- a/src/tinycl.h +++ b/src/tinycl.h @@ -95,12 +95,12 @@ int clGetKernelWorkGroupInfo(cl_kernel, cl_device_id, cl_kernel_work_group_info, int clGetEventInfo(cl_event, cl_event_info paramName, size_t paramValueSize, void* paramValue, size_t* sizeRet); int clGetEventProfilingInfo(cl_event, cl_profiling_info, size_t, void*, size_t* sizeRet); - + void* clSVMAlloc(cl_context, cl_svm_mem_flags, size_t, unsigned alignment); void clSVMFree(cl_context, void*); int clSetKernelArgSVMPointer(cl_kernel, unsigned, const void *); - + } #define CL_SUCCESS 0 @@ -320,3 +320,15 @@ typedef union #define CL_INVALID_DEVICE_PARTITION_COUNT -68 #define CL_INVALID_PIPE_SIZE -69 #define CL_INVALID_DEVICE_QUEUE -70 + +// OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features +// This is only supported by our CUDA translation of openCL. Since this file is for the +// native openCL builds, these extension routines basicly do nothing. + +struct _cl_graph {}; +typedef _cl_graph* cl_graph; +bool clIsGraphSupported(cl_device_id); +int clGraphBeginRecording(cl_command_queue); +int clGraphEndRecording(cl_command_queue, cl_graph*); +int clGraphLaunch(cl_graph); +int clReleaseGraph(cl_graph); From b4992d6a37f3db02cb3b0ecbae32cba53212bd5b Mon Sep 17 00:00:00 2001 From: george Date: Mon, 20 Jul 2026 01:36:30 +0000 Subject: [PATCH 088/214] Implemented L2-striping (a lot of work that sadly was not faster on any GPUs I've tried). Support for CUDA graphs added (slower on a 3080, but a 1% gain on 5070ti). Calculate ROE without using the roePos argument (needed for CUDA graphs). Better algorithm for determining how long to sleep when queue fills (by explicit counting of squarings/muls enqueued). MULTI_Q and GRAPHS now supported by -tune. Support for two dimensional kernels added. --- src/Gpu.cpp | 795 +++++++++++++++++++++++++++++------------ src/Gpu.h | 8 +- src/Kernel.cpp | 22 +- src/Kernel.h | 11 +- src/Queue.cpp | 28 +- src/Queue.h | 36 +- src/cl/base.cl | 8 + src/cl/carry.cl | 52 ++- src/cl/carryfused.cl | 114 +++--- src/cl/carryutil.cl | 50 ++- src/cl/ffthin.cl | 72 ++-- src/cl/fftmiddlein.cl | 90 ++++- src/cl/fftmiddleout.cl | 75 +++- src/cl/tailmul.cl | 59 ++- src/cl/tailsquare.cl | 156 ++++---- src/tune.cpp | 47 ++- 16 files changed, 1138 insertions(+), 485 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index f16fe0e9..123c4dba 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -231,7 +231,7 @@ constexpr bool isInList(const string& s, initializer_list list) { return false; } -string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u64 E, bool doLog, +string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u64 E, bool doLog, bool &tail_single_wide, bool &tail_single_kernel, u32 &in_place, u32 &pad_size, u32 &wmul) { map config; @@ -287,7 +287,9 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< "MODM31", "LOADS","STORES", "NOREG", // CUDA - experimental - "WMUL" + "WMUL", + "MULTI_Q", + "GRAPHS" }); if (!isValid) { log("Warning: unrecognized -use key '%s'\n", k.c_str()); @@ -323,6 +325,29 @@ string clDefines(const Args& args, cl_device_id id, FFTConfig fft, const vector< } } + // L2_STRIPING is not allowed if INPLACE=0. Maximum L2_STRIPING is WIDTH/64 if MULTI_Q=0 and WIDTH/128 if MULTI_Q=1. + // Technically, L2_STRIPING of WIDTH/32, MULTI_Q=0 could be allowed but that is just a more complicated way to implement L2_STRIPING=0. + // Also, WIDTH/64, MULTI_Q=1 could be allowed with some marker/sync code changes but that is very similar to L2_STRIPING=0. + { + u32 l2_striping = args.value("L2_STRIPING", 0); + u32 multi_q = args.value("MULTI_Q", 0); + if (l2_striping && !in_place) { + config["L2_STRIPING"] = to_string(0); + args.flags["L2_STRIPING"] = to_string(0); + log("L2_STRIPING is only allowed if INPLACE=1. Changing to L2_STRIPING=0.\n"); + } + else if (multi_q == 0 && l2_striping > fft.shape.width/64) { + config["L2_STRIPING"] = to_string(fft.shape.width/64); + args.flags["L2_STRIPING"] = to_string(fft.shape.width/64); + log("Max L2_STRIPING when MULTI_Q=0 exceeded. Changing to L2_STRIPING=%u.\n", fft.shape.width/64); + } + else if (multi_q > 0 && l2_striping > fft.shape.width/128) { + config["L2_STRIPING"] = to_string(fft.shape.width/128); + args.flags["L2_STRIPING"] = to_string(fft.shape.width/128); + log("Max L2_STRIPING when MULTI_Q=1 exceeded. Changing to L2_STRIPING=%u.\n", fft.shape.width/128); + } + } + string defines = toDefine(config); if (doLog) { log("config: %s\n", defines.c_str()); } @@ -850,8 +875,8 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo BUF(bufSmallOut, 256), BUF(bufSumOut, 1), BUF(bufTrue, 1), - BUF(bufROE, ROE_SIZE), - BUF(bufStatsCarry, CARRY_SIZE), + BUF(bufROE, ROE_SIZE + 2), + BUF(bufStatsCarry, CARRY_SIZE + 2), BUF(buf1, TOTAL_DATA_SIZE(fft, WIDTH, fft.shape.middle, SMALL_H, in_place, pad_size)), BUF(buf2, TOTAL_DATA_SIZE(fft, WIDTH, fft.shape.middle, SMALL_H, in_place, pad_size)), @@ -862,9 +887,11 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo timeBufVect{profile.make("proofBufVect")}, recorded_kernels{}, - recorded_kernel_args{} -{ + recorded_kernel_args{}, + use_graphs{}, + graph_square{} +{ float bitsPerWord = E / float(N); if (logFftSize) { log("FFT: %s %s (%.2f bpw)\n", numberK(N).c_str(), fft.spec().c_str(), bitsPerWord); @@ -886,35 +913,35 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo if (useLongCarry) { log("Using long carry!\n"); } if (fft.FFT_FP64 || fft.FFT_FP32) { - kfftMidIn.setFixedArgs(2, bufTrigM); - kfftHin.setFixedArgs(2, bufTrigH); + kfftMidIn.setFixedArgs(3, bufTrigM); + kfftHin.setFixedArgs(3, bufTrigH); ktailSquareZero.setFixedArgs(2, bufTrigH); - ktailSquare.setFixedArgs(2, bufTrigH); - ktailMulLow.setFixedArgs(3, bufTrigH); - ktailMul.setFixedArgs(3, bufTrigH); - kfftMidOut.setFixedArgs(2, bufTrigM); + ktailSquare.setFixedArgs(3, bufTrigH); + ktailMulLow.setFixedArgs(4, bufTrigH); + ktailMul.setFixedArgs(4, bufTrigH); + kfftMidOut.setFixedArgs(3, bufTrigM); kfftW.setFixedArgs(2, bufTrigW); } if (fft.NTT_GF31) { - kfftMidInGF31.setFixedArgs(2, bufTrigM); - kfftHinGF31.setFixedArgs(2, bufTrigH); + kfftMidInGF31.setFixedArgs(3, bufTrigM); + kfftHinGF31.setFixedArgs(3, bufTrigH); ktailSquareZeroGF31.setFixedArgs(2, bufTrigH); - ktailSquareGF31.setFixedArgs(2, bufTrigH); - ktailMulLowGF31.setFixedArgs(3, bufTrigH); - ktailMulGF31.setFixedArgs(3, bufTrigH); - kfftMidOutGF31.setFixedArgs(2, bufTrigM); + ktailSquareGF31.setFixedArgs(3, bufTrigH); + ktailMulLowGF31.setFixedArgs(4, bufTrigH); + ktailMulGF31.setFixedArgs(4, bufTrigH); + kfftMidOutGF31.setFixedArgs(3, bufTrigM); kfftWGF31.setFixedArgs(2, bufTrigW); } if (fft.NTT_GF61) { - kfftMidInGF61.setFixedArgs(2, bufTrigM); - kfftHinGF61.setFixedArgs(2, bufTrigH); + kfftMidInGF61.setFixedArgs(3, bufTrigM); + kfftHinGF61.setFixedArgs(3, bufTrigH); ktailSquareZeroGF61.setFixedArgs(2, bufTrigH); - ktailSquareGF61.setFixedArgs(2, bufTrigH); - ktailMulLowGF61.setFixedArgs(3, bufTrigH); - ktailMulGF61.setFixedArgs(3, bufTrigH); - kfftMidOutGF61.setFixedArgs(2, bufTrigM); + ktailSquareGF61.setFixedArgs(3, bufTrigH); + ktailMulLowGF61.setFixedArgs(4, bufTrigH); + ktailMulGF61.setFixedArgs(4, bufTrigH); + kfftMidOutGF61.setFixedArgs(3, bufTrigM); kfftWGF61.setFixedArgs(2, bufTrigW); } @@ -953,240 +980,521 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo selftestTrig(); } - // If MULTI_Q option is set there are fewer kernels executed in the main queue, but there are some additional syncEvents and waits. - // That's a total of 4 kernels (carryFused, MidIn/Out, TailSquare) plus 1 syncEvent plus 1 or more syncWaits. - // If MULTI_Q option is not set there is carryFused + MidIn/Out and TailSquare for each NTT modulus. - // NOTE: We dont take into account the optional tailSquareZero kernel. We theoretically should. - if (args.value("MULTI_Q", 0)) - queue.setSquareKernels(5 + ((fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61) - 1)); - else - queue.setSquareKernels(1 + 3 * (fft.FFT_FP64 + fft.FFT_FP32 + fft.NTT_GF31 + fft.NTT_GF61)); + // Create aux queues. For now, we only have one auxiliary queue. We could do more. + if (args.value("MULTI_Q", 0)) { + auxQueues.push_back(Queue{*shared.context, args.profile, true}); + } + + // Set flag indicating we're going to use CUDA graphs + use_graphs = graph_square[0].isSupported(shared.context->deviceId()) && args.value("GRAPHS", 1); + + // Process the queue. I don't know if this is really needed. queue.finish(); } -// Optionallly split some of the MiddleIn/Tail/MiddleOut kernels off od executing on the main queue to run on an auxiliary queue. +// Optionallly split some of the MiddleIn/Tail/MiddleOut kernels off of executing on the main queue to run on an auxiliary queue. // This will increase GPU occupancy but will negatively impact L2 cache coherency. // If the L2 cache is large enough so that all FFT data fits in the cache, this ought to be a win. // If the L2 cache is small enough such that L2 cache hits are very low anyway, this might be a win. - void Gpu::splitQueue(void) { - - // If MULTI_Q -use not set, return - if (!args.value("MULTI_Q", 0)) return; - - // Create aux queues. For now, we only have one auxiliary queue. We could do more. - if (auxQueues.size() == 0) { - auxQueues.push_back(Queue{*shared.context, args.profile, true}); - } - // Queue a sync event in the main queue. Have all auxiliary queues wait on the event. EventHolder event = queue.createSyncEvent(); for (size_t i = 0; i < auxQueues.size(); ++i) { auxQueues[i].waitForSyncEvent(&event); } - - // Assign kernels to running on the main queue or an auxiliary queue - - int which_queue = -1; - - // For no particularly good reason, put a kernel that operates on 64-bit vaules in the main queue. - if (fft.NTT_GF61) { - if (which_queue != -1) { - kfftMidInGF61.setQueue(&auxQueues[which_queue]); - kfftHinGF61.setQueue(&auxQueues[which_queue]); - ktailSquareZeroGF61.setQueue(&auxQueues[which_queue]); - ktailSquareGF61.setQueue(&auxQueues[which_queue]); - ktailMulGF61.setQueue(&auxQueues[which_queue]); - ktailMulLowGF61.setQueue(&auxQueues[which_queue]); - kfftMidOutGF61.setQueue(&auxQueues[which_queue]); - kfftWGF61.setQueue(&auxQueues[which_queue]); - } - which_queue++; - } - - if (fft.FFT_FP64 || fft.FFT_FP32) { - if (which_queue != -1) { - kfftMidIn.setQueue(&auxQueues[which_queue]); - kfftHin.setQueue(&auxQueues[which_queue]); - ktailSquareZero.setQueue(&auxQueues[which_queue]); - ktailSquare.setQueue(&auxQueues[which_queue]); - ktailMul.setQueue(&auxQueues[which_queue]); - ktailMulLow.setQueue(&auxQueues[which_queue]); - kfftMidOut.setQueue(&auxQueues[which_queue]); - kfftW.setQueue(&auxQueues[which_queue]); - } - // For no particularly good reason, put kernels that operate on 32-bit value in the same queue unless there are no kernels operating on 64-bit values - if (fft.FFT_FP64 || (fft.FFT_FP32 && which_queue == -1)) { - which_queue++; - } - } - - if (fft.NTT_GF31) { - if (which_queue != -1) { - kfftMidInGF31.setQueue(&auxQueues[which_queue]); - kfftHinGF31.setQueue(&auxQueues[which_queue]); - ktailSquareZeroGF31.setQueue(&auxQueues[which_queue]); - ktailSquareGF31.setQueue(&auxQueues[which_queue]); - ktailMulGF31.setQueue(&auxQueues[which_queue]); - ktailMulLowGF31.setQueue(&auxQueues[which_queue]); - kfftMidOutGF31.setQueue(&auxQueues[which_queue]); - kfftWGF31.setQueue(&auxQueues[which_queue]); - } - //which_queue++; - } } void Gpu::mergeQueue(void) { - - // If MULTI_Q -use not set, return - if (!args.value("MULTI_Q", 0)) return; - // Queue a sync event in each auxiliary queue(s). Wait on the event(s) in the main queue. for (size_t i = 0; i < auxQueues.size(); ++i) { EventHolder event = auxQueues[i].createSyncEvent(); queue.waitForSyncEvent(&event); } +} - // Return kernels to running on the main queue - // NOTE: I believe there is no need to switch queues back and forth between the main and auxiliary queues. No one currently uses the cache_group == 0 option. - if (fft.NTT_GF61) { - kfftMidInGF61.setQueue(&queue); - kfftHinGF61.setQueue(&queue); - ktailSquareZeroGF61.setQueue(&queue); - ktailSquareGF61.setQueue(&queue); - ktailMulGF61.setQueue(&queue); - ktailMulLowGF61.setQueue(&queue); - kfftMidOutGF61.setQueue(&queue); - kfftWGF61.setQueue(&queue); - } - if (fft.FFT_FP64 || fft.FFT_FP32) { - kfftMidIn.setQueue(&queue); - kfftHin.setQueue(&queue); - ktailSquareZero.setQueue(&queue); - ktailSquare.setQueue(&queue); - ktailMul.setQueue(&queue); - ktailMulLow.setQueue(&queue); - kfftMidOut.setQueue(&queue); - kfftW.setQueue(&queue); - } - if (fft.NTT_GF31) { - kfftMidInGF31.setQueue(&queue); - kfftHinGF31.setQueue(&queue); - ktailSquareZeroGF31.setQueue(&queue); - ktailSquareGF31.setQueue(&queue); - ktailMulGF31.setQueue(&queue); - ktailMulLowGF31.setQueue(&queue); - kfftMidOutGF31.setQueue(&queue); - kfftWGF31.setQueue(&queue); - } +// We've finished the "bottom half" of a squaring or multiply. Replay the recorded bottom half kernel calls. +void Gpu::endBottomHalf(void) { + replay(); + // Increment the squaring count. The queue's squarings/multiplies count determines how long to sleep when the queue is full. + // We only do this for the main command queue. Auxiliary queues are not allowed to cause a CPU sleep. + queue.incSquareCount(); } -// Replay the recorded bottom half kernels in a cache friendly order. We support several -// options here using multiple openCl command queues. +// Replay the recorded bottom half kernels in a cache friendly order. We support several options here using multiple openCl command queues. void Gpu::replay(void) { - - // If using multiple command queues, handle that now. - splitQueue(); - - // For better L2 cache locality, operate on all the FP data, then operate on all the GF31 data, then GF61. - for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { - - // Check for irrelevant cache gouup - if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; - if (cache_group == 2 && !fft.NTT_GF31) continue; - if (cache_group == 3 && !fft.NTT_GF61) continue; - - // Iterate over the recorded kernels - int arg = 0; - for (auto kern : recorded_kernels) { - - // Call the appropriate kernel - if (kern == KMIDIN) { - Buffer *buf = recorded_kernel_args[arg++]; - // If not in place, the input is from the scratch buffer - Buffer *in = in_place ? buf : &buf3; - Buffer *out = buf; - if (cache_group == 1) kfftMidIn(*out, *in); - if (cache_group == 2) kfftMidInGF31(*out, *in); - if (cache_group == 3) kfftMidInGF61(*out, *in); + // If there are no recorded kernels to replay, we're done + if (recorded_kernels.size() == 0) return; + + // Get MULTI_Q and L2_STRIPING settings + bool multi_q = args.value("MULTI_Q", 0); + int l2_striping = args.value("L2_STRIPING", 0); + + // In the simplest case, we use one command queue and process one data type at a time. By processing one data type at a time, we reduce maximum L2 cache used. + // For example, a 4M GF61+GF31 NTT needs just 32MB L2 cache during GF61 processing of fftMiddleIn, tailSquare, and fftMiddleOut (and only 16MB duing GF31 processing). + // Without MULTI_Q, PRPLL needs 32MB + 16MB of L2 cache by processing both fftMiddleIns, then both tailSquares, then both fftMiddleOuts. + + if ((!multi_q || fft.shape.fft_type == FFT64 || fft.shape.fft_type == FFT61 || fft.shape.fft_type == FFT31 || fft.shape.fft_type == FFT32) && !l2_striping) { + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + replay_one(kern, cache_group, arg); + arg = replay_next_arg(kern, arg); } + } + } - if (kern == KFFTHIN) { - Buffer *out = recorded_kernel_args[arg++]; - Buffer *in = recorded_kernel_args[arg++]; - if (cache_group == 1) kfftHin(*out, *in); - if (cache_group == 2) kfftHinGF31(*out, *in); - if (cache_group == 3) kfftHinGF61(*out, *in); + // The next simple case, we use two command queues and process one data type in each queue. This works well for large L2 caches where all FFT data fits in the cache. + // The extra command queue can hide the latency in starting up kernels for each data type. Also occupancy may benefit as the queue may be executing kernels with different + // workgroup size, register usage, and local memory usage. This case requires an FFT using at least two data types. + + else if (multi_q && !l2_striping) { + // Switch tp using multiple command queues. + splitQueue(); + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // To better balance the load on the two command queues, put a 64-bit data type in one queue and two 32-bit data types in the other queue. + Queue *q; + if (cache_group == 1) q = &queue; + if (cache_group == 2) q = (fft.shape.fft_type == FFT323161 || fft.shape.fft_type == FFT3161) ? &queue : &auxQueues[0]; + if (cache_group == 3) q = &auxQueues[0]; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + replay_one(kern, cache_group, arg, q); + arg = replay_next_arg(kern, arg); } - - if (kern == KTAILSQUARE) { - Buffer *buf = recorded_kernel_args[arg++]; - // If not in place, the output is to the scratch buffer - Buffer *in = buf; - Buffer *out = in_place ? buf : &buf3; - if (!tail_single_kernel) { - if (cache_group == 1) ktailSquareZero(*out, *in); - if (cache_group == 2) ktailSquareZeroGF31(*out, *in); - if (cache_group == 3) ktailSquareZeroGF61(*out, *in); + } + // Using multiple command queues, go back to a single command queue + mergeQueue(); + } + + // The next case is L2 striping in one command queue. The hope is two stripe groups plus one stripe are small enough to fit in the L2 caches + // for the fftMiddleIn, tailSquare, and fftMiddleOut kernels. + // Sadly, testing thusfar shows the extra overhead of more kernel launches and events/syncs outweighs the benefit of more L2 cache hits. +#ifdef ORIGINAL_VERSION // Very readable, replaced by version below which merges the teo base_lo and base_hi kernel calls into one combined kernel call + else if (!multi_q && l2_striping) { + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Allow larger caches to do several L2 stripes in a single "stripe group" at a time (increases occupancy, reduces kernel launch costs). + u32 stripe_group_size = l2_striping; + +//For now, only support INPLACE with its 16x16 transpose + + // Loop over all the stripes for this data type. Let's define a stripe as one "column" of data processed by fftMiddleIn producing 16*MIDDLE tailSquare lines. + // Since tailSquare operates on Hermetian pairs of lines, fftMiddleIn alternates operating on low stripes and high stripes. + u32 num_stripes = fft.shape.width / 16; + u32 num_stripe_groups = num_stripes / stripe_group_size; + for (u32 i = 0; i < num_stripe_groups / 2; ++i) { + // Base_lo refers to the starting fftMiddleIn column number (x coordinate). When fftMiddleIn uses a 16x16 transpose, base_lo advances 16 at a time. + // Base_hi refers to the fftMiddleIn column number that outputs the lines needed for Hermetian matching in tailSquare. + u32 base_lo = i * stripe_group_size * 16; + u32 base_hi = (num_stripe_groups - 1 - i) * stripe_group_size * 16; + bool last_block = (base_hi == fft.shape.width / 2); + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + + if (kern == KMIDIN) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + // If MIDDLE is odd, the first fftMiddleIn call must be preceeded by a fftMiddleIn call to produce the special N/2 tailSquare line from the WIDTH/2 column. + if (base_lo == 0 && (fft.shape.middle & 1)) { + replay_one(kern, cache_group, arg, &queue, fft.shape.width / 2, oneStripeKernelsToExecute); + } + + // Produce one stripe group. The last base value must take into account that the N/2 stripe has already been done if MIDDLE is odd. + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + + u32 base = base_hi; // Read full base_hi stripe groups (usually) + if (last_block && (fft.shape.middle & 1)) base += 16, kernelsToExecute -= oneStripeKernelsToExecute; // Skip first stripe for last block if MIDDLE is odd + if (kernelsToExecute) replay_one(kern, cache_group, arg, &queue, base, kernelsToExecute); + } + + else if (kern == KFFTHIN) { + // fftMiddleIn produces 16 * MIDDLE lines + replay_one(kern, cache_group, arg, &queue, base_lo, stripe_group_size * 16 * fft.shape.middle); + replay_one(kern, cache_group, arg, &queue, base_hi, stripe_group_size * 16 * fft.shape.middle); + } + + else if (kern == KTAILSQUARE || kern == KTAILMUL || kern == KTAILMULLOW) { + // fftMiddleIn produces 16 * MIDDLE lines in base_lo and base_hi. tailSquare kernel processes 2 lines linked by Hermetian symmetry. + // Tail kernels use two dimensions, the X coordinate bumps line number by one, the Y coordinate bumps line number by WIDTH. + u32 kernelsToExecuteX = stripe_group_size * 16; + u32 kernelsToExecuteY = (fft.shape.middle + 1) / 2; // For base_lo, round odd middles up. + + // We can now completely process lines from the lower half of the base_lo stripe group (Hermetian mates are mostly in upper half of base_hi stripe group) + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecuteX, kernelsToExecuteY); + + // We can process most lines from the lower half of the base_hi stripe group (Hermetian mates are mostly in upper half of base_lo stripe group) + // The first line in the stripe group is the only line that is not ready for base_hi tail processing. + u32 base = base_hi + 1; // Skip first line in base_hi (usually) + if (base_lo == 0) kernelsToExecuteX--; // Do one fewer line for the first tail call. + if (base_hi == fft.shape.width / 2) base--, kernelsToExecuteX++; // Last tail call does not skip first line + if (fft.shape.middle & 1) kernelsToExecuteY--; // For base_hi, round odd middles down. + replay_one(kern, cache_group, arg, &queue, base, kernelsToExecuteX, kernelsToExecuteY); + } + + else if (kern == KMIDOUT) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + + // We've completely processed lines from the base_lo stripe group + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + + // The first stripe in the base_hi stripe group is not ready for output. + u32 base = base_hi + 16; // Skip first stripe in base_hi (usually) + if (base_lo == 0) kernelsToExecute -= oneStripeKernelsToExecute; // Do one fewer stripe for the first midOut call. + if (base_hi == fft.shape.width / 2) base -= 16, kernelsToExecute += oneStripeKernelsToExecute; // Last midOut call does not skip first stripe + if (kernelsToExecute) replay_one(kern, cache_group, arg, &queue, base, kernelsToExecute); + } + + // Skip other kernels (KFFTW) + else; + + // Advance argument index + arg = replay_next_arg(kern, arg); } - if (cache_group == 1) ktailSquare(*out, *in); - if (cache_group == 2) ktailSquareGF31(*out, *in); - if (cache_group == 3) ktailSquareGF61(*out, *in); } - if (kern == KTAILMUL) { - Buffer *buf = recorded_kernel_args[arg++]; - Buffer *in2 = recorded_kernel_args[arg++]; - // If not in place, the output is to the scratch buffer - Buffer *in1 = buf; - Buffer *out = in_place ? buf : &buf3; - if (cache_group == 1) ktailMul(*out, *in1, *in2); - if (cache_group == 2) ktailMulGF31(*out, *in1, *in2); - if (cache_group == 3) ktailMulGF61(*out, *in1, *in2); + // Iterate over the recorded kernels. Execute any not already executed (KFFTW). + // FFTW cannot benefit from L2 striping, it can only benefit from datatype grouping. + int arg = 0; + for (auto kern : recorded_kernels) { + if (kern == KFFTW) replay_one(kern, cache_group, arg, &queue); + arg = replay_next_arg(kern, arg); } + } + } +#endif - if (kern == KTAILMULLOW) { - Buffer *buf = recorded_kernel_args[arg++]; - Buffer *in2 = recorded_kernel_args[arg++]; - // If not in place, the output is to the scratch buffer - Buffer *in1 = buf; - Buffer *out = in_place ? buf : &buf3; - if (cache_group == 1) ktailMulLow(*out, *in1, *in2); - if (cache_group == 2) ktailMulLowGF31(*out, *in1, *in2); - if (cache_group == 3) ktailMulLowGF61(*out, *in1, *in2); + // The next case is L2 striping in one command queue. The hope is two stripe groups plus one stripe are small enough to fit in the L2 caches + // for the fftMiddleIn, tailSquare, and fftMiddleOut kernels. + // Sadly, testing thusfar shows the extra overhead of more kernel launches and events/syncs outweighs the benefit of more L2 cache hits. + + else if (!multi_q && l2_striping) { + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Allow larger caches to do several L2 stripes in a single "stripe group" at a time (increases occupancy, reduces kernel launch costs). + u32 stripe_group_size = l2_striping; + +//For now, only support INPLACE with its 16x16 transpose + + // Loop over all the stripes for this data type. Let's define a stripe as one "column" of data processed by fftMiddleIn producing 16*MIDDLE tailSquare lines. + // Since tailSquare operates on Hermetian pairs of lines, fftMiddleIn alternates operating on low stripes and high stripes. + u32 num_stripes = fft.shape.width / 16; + u32 num_stripe_groups = num_stripes / stripe_group_size; + for (u32 i = 0; i < num_stripe_groups / 2; ++i) { + bool last_block = (i == num_stripe_groups / 2 - 1); + // Base_lo refers to the starting fftMiddleIn column number (x coordinate). When fftMiddleIn uses a 16x16 transpose, base_lo advances 16 at a time. + // Base_hi refers to the fftMiddleIn column number that outputs the lines needed for Hermetian matching in tailSquare. + u32 base_lo = i * stripe_group_size * 16; + // u32 base_hi = (num_stripe_groups - 1 - i) * stripe_group_size * 16; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + + if (kern == KMIDIN) { + // Do midIn on base_lo and base_hi. If MIDDLE is odd, the first midIn call must be preceeded by a midIn call to produce the special N/2 tailSquare + // line from the WIDTH/2 column. The last base pair must take into account that the N/2 stripe has already been done if MIDDLE is odd. + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + if (base_lo == 0 && (fft.shape.middle & 1)) kernelsToExecute += oneStripeKernelsToExecute; + if (last_block && (fft.shape.middle & 1)) kernelsToExecute -= oneStripeKernelsToExecute; + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + } + + else if (kern == KFFTHIN) { + // fftMiddleIn produces 2 * stripe_group_size * 16 * MIDDLE lines + replay_one(kern, cache_group, arg, &queue, base_lo, 2 * stripe_group_size * 16 * fft.shape.middle); + } + + else if (kern == KTAILSQUARE || kern == KTAILMUL || kern == KTAILMULLOW) { + // We can now completely process lines from the lower half of the base_lo stripe group (Hermetian mates are mostly in upper half of base_hi stripe group) + u32 half_size = (fft.shape.middle + 1) / 2; // For base_lo, round odd middles up. + u32 stripeGroupLines = stripe_group_size * 16; + u32 kernelsToExecute = half_size * stripeGroupLines; + // We can process most lines from the lower half of the base_hi stripe group (Hermetian mates are mostly in upper half of base_lo stripe group) + // The first line in the stripe group is the only line that is not ready for base_hi tail processing. + if (base_lo == 0) stripeGroupLines--; // Do one fewer line for the first tail call. + if (last_block) stripeGroupLines++; // Last tail call does not skip first line + if (fft.shape.middle & 1) half_size--; // For base_hi, round odd middles down. + kernelsToExecute += half_size * stripeGroupLines; + // Call the kernel + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + } + + else if (kern == KMIDOUT) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + // We've completely processed lines from the base_lo stripe group. + // The first stripe in the base_hi stripe group is not ready for output. + // The last stripe group will does an extra stripe. + if (base_lo == 0) kernelsToExecute -= oneStripeKernelsToExecute; // Do one fewer stripe for the first midOut call + if (last_block) kernelsToExecute += oneStripeKernelsToExecute; // Last midOut call does not skip first stripe + replay_one(kern, cache_group, arg, &queue, base_lo, kernelsToExecute); + } + + // Skip other kernels (KFFTW) + else; + + // Advance argument index + arg = replay_next_arg(kern, arg); + } } - if (kern == KMIDOUT) { - Buffer *buf = recorded_kernel_args[arg++]; - // If not in place, the input is from the scratch buffer - Buffer *in = in_place ? buf : &buf3; - Buffer *out = buf; - if (cache_group == 1) kfftMidOut(*out, *in); - if (cache_group == 2) kfftMidOutGF31(*out, *in); - if (cache_group == 3) kfftMidOutGF61(*out, *in); + // Iterate over the recorded kernels. Execute any not already executed (KFFTW). + // FFTW cannot benefit from L2 striping, it can only benefit from datatype grouping. + int arg = 0; + for (auto kern : recorded_kernels) { + if (kern == KFFTW) replay_one(kern, cache_group, arg, &queue); + arg = replay_next_arg(kern, arg); } + } + } - if (kern == KFFTW) { - Buffer *out = recorded_kernel_args[arg++]; - Buffer *in = recorded_kernel_args[arg++]; - if (cache_group == 1) kfftW(*out, *in); - if (cache_group == 2) kfftWGF31(*out, *in); - if (cache_group == 3) kfftWGF61(*out, *in); + // The last case is L2 striping in two command queues. The hope is four stripe groups plus two stripes are small enough to fit in the L2 caches for the + // fftMiddleIn, tailSquare, and fftMiddleOut kernels. The hope also is the dual queue approach hides the overhead introduced by more kernel launches. + // Sadly, testing thusfar shows the extra overhead of more kernel launches and events/syncs outweighs the benefit of more L2 cache hits. + + else if (multi_q && l2_striping) { + Queue *queues[2] = {&queue, &auxQueues[0]}; + EventHolder midInEvents[2]; + EventHolder tailEvents[2]; + + splitQueue(); + for (int cache_group = 1; cache_group <= NUM_CACHE_GROUPS; ++cache_group) { + bool has_unexecuted_kernels = false; + + // Check for irrelevant cache group + if (cache_group == 1 && !(fft.FFT_FP64 || fft.FFT_FP32)) continue; + if (cache_group == 2 && !fft.NTT_GF31) continue; + if (cache_group == 3 && !fft.NTT_GF61) continue; + + // Allow larger caches to do several L2 stripes at a time (increases occupancy, reduces kernel launch costs). + u32 stripe_group_size = l2_striping; + +//For now, only support INPLACE with its 16x16 transpose + + // Loop over all the stripes for this data type. Let's define a stripe as one "column" of data processed by fftMiddleIn producing 16*MIDDLE tailSquare lines. + // Since tailSquare operates on Hermetian pairs of lines, fftMiddleIn alternates operating on low stripes and high stripes. + u32 num_stripes = fft.shape.width / 16; + u32 num_stripe_groups = num_stripes / stripe_group_size; + for (u32 i = 0; i < num_stripe_groups / 2; ++i) { + int q = (i & 1); // Index into which command queue to use + u32 i_within_queue = i >> 1; + bool last_i_within_queue = (i_within_queue == num_stripe_groups / 4 - 1); + // Base_lo refers to the starting fftMiddleIn column number (x coordinate). When fftMiddleIn uses a 16x16 transpose, base_lo advances 16 at a time. + // Base_hi refers to the fftMiddleIn column number that outputs the lines needed for Hermetian matching in tailSquare. + u32 base_lo = (i_within_queue + q * num_stripe_groups / 4) * stripe_group_size * 16; + //u32 base_hi = fft.shape.width - stripe_group_size * 16 - base_lo; + + // Iterate over the recorded kernels. Execute each. + int arg = 0; + for (auto kern : recorded_kernels) { + + if (kern == KMIDIN) { + // Do midIn on base_lo and base_hi. If MIDDLE is odd, the first midIn call must be preceeded by a midIn call to produce the special N/2 tailSquare + // line from the WIDTH/2 column. The first midIn call in the second command queue also must be preceeded by a midIn call to produce one L2 stripe. + // The last base_hi in each queue must take into account pre-read stripes in the other queue. +#define q_requires_preread(q) (((q) == 0 && fft.shape.middle & 1) || ((q) == 1)) + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + if (i_within_queue == 0 && q_requires_preread(q)) kernelsToExecute += oneStripeKernelsToExecute; + if (last_i_within_queue && q_requires_preread(!q)) kernelsToExecute -= oneStripeKernelsToExecute; + replay_one(kern, cache_group, arg, queues[q], base_lo, kernelsToExecute); + if (i_within_queue == 0 && q_requires_preread(q)) midInEvents[q] = queues[q]->createSyncEvent(); + // The last block must sync with a pre-read from the other command queue + // BUG - if block is both i_within_queue == 0 and last_block_in_queue, then midInEvents[1] does not exist! + // Sanity checking L2_STRIPING setting to a max of WIDTH/128 at startup eliminates this bug. + if (last_i_within_queue && q_requires_preread(!q)) queues[q]->waitForSyncEvent(&midInEvents[!q]); + } + + else if (kern == KFFTHIN) { + // fftMiddleIn produces 2 * stripe_group_size * 16 * MIDDLE lines + replay_one(kern, cache_group, arg, queues[q], base_lo, 2 * stripe_group_size * 16 * fft.shape.middle); + } + + else if (kern == KTAILSQUARE || kern == KTAILMUL || kern == KTAILMULLOW) { + // We can now completely process lines from the lower half of the base_lo stripe group (Hermetian mates are mostly in upper half of base_hi stripe group) + u32 half_size = (fft.shape.middle + 1) / 2; // For base_lo, round odd middles up. + u32 stripeGroupLines = stripe_group_size * 16; + u32 kernelsToExecute = half_size * stripeGroupLines; + // We can process most lines from the lower half of the base_hi stripe group (Hermetian mates are mostly in upper half of base_lo stripe group) + // The first line in the stripe group is the only line that is not ready for base_hi tail processing. + if (i == 0) stripeGroupLines--; // Do one fewer line for the first tail call. + if (last_i_within_queue && q == 1) stripeGroupLines++; // Last tail call does not skip first line + half_size = fft.shape.middle / 2; // For base_hi, round odd middles down. + kernelsToExecute += half_size * stripeGroupLines; + // Call the kernel + replay_one(kern, cache_group, arg, queues[q], base_lo, kernelsToExecute); + if (i_within_queue == 0 && q_requires_preread(q)) tailEvents[q] = queues[q]->createSyncEvent(); +// We could eliminate two events and syncs by having the last midIn wait on the first tailsquare. It exposes a little less parallellism, but perhaps that is irrelevant. +// For even middles, we only save one event and sync. + // The last block must sync with the first tailSquare in the other command queue + // BUG - if block is both i_within_queue == 0 and last_block_in_queue, then tailEvents[1] does not exist! + // Sanity checking L2_STRIPING setting to a max of WIDTH/128 at startup eliminates this bug. + if (last_i_within_queue && q_requires_preread(!q)) queues[q]->waitForSyncEvent(&tailEvents[!q]); + } + + else if (kern == KMIDOUT) { + u32 oneStripeKernelsToExecute = fft.shape.height / 16; + u32 kernelsToExecute = 2 * stripe_group_size * oneStripeKernelsToExecute; + // We've completely processed lines from the base_lo stripe group. + // The first stripe in the base_hi stripe group is not ready for output. + // The last stripe group does an extra stripe. + if (i_within_queue == 0) kernelsToExecute -= oneStripeKernelsToExecute; // Do one fewer stripe for the first midOut call + if (last_i_within_queue) kernelsToExecute += oneStripeKernelsToExecute; // Last midOut call does not skip first stripe + replay_one(kern, cache_group, arg, queues[q], base_lo, kernelsToExecute); + } + + // Skip other kernels (KFFTW) + else + has_unexecuted_kernels = true; + + // Advance argument index + arg = replay_next_arg(kern, arg); + } + } + + // Iterate over the recorded kernels again. Execute any not already executed (KFFTW). + // FFTW cannot benefit from L2 striping, it can only benefit from datatype grouping. + if (has_unexecuted_kernels) { + int arg = 0; + mergeQueue(); + for (auto kern : recorded_kernels) { + if (kern == KFFTW) replay_one(kern, cache_group, arg, &queue); + arg = replay_next_arg(kern, arg); + } + splitQueue(); } } + mergeQueue(); } // Empty the recorded kernels queue recorded_kernels.clear(); recorded_kernel_args.clear(); +} + +// Replay one recorded kernel on the specified queue, with specified base and kernelsToExecute. Two dimensional work groups are supported for some kernels. +// A kernelsToExecuteX of zero is permitted - used for the default kernelsToExecute (a.k.a. workSize) set at kernel creation that operates on all the FFT data. +void Gpu::replay_one(enum BOTTOM_HALF_KERNELS kern, int cache_group, int arg, Queue *q, int base, int kernelsToExecuteX, int kernelsToExecuteY) { + + // Call the appropriate kernel + if (kern == KMIDIN) { + Buffer *buf = recorded_kernel_args[arg++]; + // If not in place, the input is from the scratch buffer + Buffer *in = in_place ? buf : &buf3; + Buffer *out = buf; + if (cache_group == 1) { kfftMidIn.setQueue(q); kfftMidIn.setKernelsToExecute(kernelsToExecuteX); kfftMidIn(*out, *in, base); } + if (cache_group == 2) { kfftMidInGF31.setQueue(q); kfftMidInGF31.setKernelsToExecute(kernelsToExecuteX); kfftMidInGF31(*out, *in, base); } + if (cache_group == 3) { kfftMidInGF61.setQueue(q); kfftMidInGF61.setKernelsToExecute(kernelsToExecuteX); kfftMidInGF61(*out, *in, base); } + } + + if (kern == KFFTHIN) { + Buffer *out = recorded_kernel_args[arg++]; + Buffer *in = recorded_kernel_args[arg++]; + if (cache_group == 1) { kfftHin.setQueue(q); kfftHin.setKernelsToExecute(kernelsToExecuteX); kfftHin(*out, *in, base); } + if (cache_group == 2) { kfftHinGF31.setQueue(q); kfftHinGF31.setKernelsToExecute(kernelsToExecuteX); kfftHinGF31(*out, *in, base); } + if (cache_group == 3) { kfftHinGF61.setQueue(q); kfftHinGF61.setKernelsToExecute(kernelsToExecuteX); kfftHinGF61(*out, *in, base); } + } + + if (kern == KTAILSQUARE) { + Buffer *buf = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer *in = buf; + Buffer *out = in_place ? buf : &buf3; + if (!tail_single_kernel && base == 0) { + if (cache_group == 1) { ktailSquareZero.setQueue(q); ktailSquareZero(*out, *in); } + if (cache_group == 2) { ktailSquareZeroGF31.setQueue(q); ktailSquareZeroGF31(*out, *in); } + if (cache_group == 3) { ktailSquareZeroGF61.setQueue(q); ktailSquareZeroGF61(*out, *in); } + if (kernelsToExecuteX) kernelsToExecuteX--; + } + if (cache_group == 1) { ktailSquare.setQueue(q); ktailSquare.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailSquare(*out, *in, base); } + if (cache_group == 2) { ktailSquareGF31.setQueue(q); ktailSquareGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailSquareGF31(*out, *in, base); } + if (cache_group == 3) { ktailSquareGF61.setQueue(q); ktailSquareGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailSquareGF61(*out, *in, base); } + } + + if (kern == KTAILMUL) { + Buffer *buf = recorded_kernel_args[arg++]; + Buffer *in2 = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer *in1 = buf; + Buffer *out = in_place ? buf : &buf3; + if (cache_group == 1) { ktailMul.setQueue(q); ktailMul.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMul(*out, *in1, *in2, base); } + if (cache_group == 2) { ktailMulGF31.setQueue(q); ktailMulGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulGF31(*out, *in1, *in2, base); } + if (cache_group == 3) { ktailMulGF61.setQueue(q); ktailMulGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulGF61(*out, *in1, *in2, base); } + } + + if (kern == KTAILMULLOW) { + Buffer *buf = recorded_kernel_args[arg++]; + Buffer *in2 = recorded_kernel_args[arg++]; + // If not in place, the output is to the scratch buffer + Buffer *in1 = buf; + Buffer *out = in_place ? buf : &buf3; + if (cache_group == 1) { ktailMulLow.setQueue(q); ktailMulLow.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLow(*out, *in1, *in2, base); } + if (cache_group == 2) { ktailMulLowGF31.setQueue(q); ktailMulLowGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLowGF31(*out, *in1, *in2, base); } + if (cache_group == 3) { ktailMulLowGF61.setQueue(q); ktailMulLowGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLowGF61(*out, *in1, *in2, base); } + } + + if (kern == KMIDOUT) { + Buffer *buf = recorded_kernel_args[arg++]; + // If not in place, the input is from the scratch buffer + Buffer *in = in_place ? buf : &buf3; + Buffer *out = buf; + if (cache_group == 1) { kfftMidOut.setQueue(q); kfftMidOut.setKernelsToExecute(kernelsToExecuteX); kfftMidOut(*out, *in, base); } + if (cache_group == 2) { kfftMidOutGF31.setQueue(q); kfftMidOutGF31.setKernelsToExecute(kernelsToExecuteX); kfftMidOutGF31(*out, *in, base); } + if (cache_group == 3) { kfftMidOutGF61.setQueue(q); kfftMidOutGF61.setKernelsToExecute(kernelsToExecuteX); kfftMidOutGF61(*out, *in, base); } + } + + if (kern == KFFTW) { + Buffer *out = recorded_kernel_args[arg++]; + Buffer *in = recorded_kernel_args[arg++]; + if (cache_group == 1) { kfftW.setQueue(q); kfftW(*out, *in); } + if (cache_group == 2) { kfftWGF31.setQueue(q); kfftWGF31(*out, *in); } + if (cache_group == 3) { kfftWGF61.setQueue(q); kfftWGF61(*out, *in); } + } +} + +// Advance the index into the array of kernel arguments +int Gpu::replay_next_arg(enum BOTTOM_HALF_KERNELS kern, int arg) { + + if (kern == KMIDIN || kern == KTAILSQUARE || kern == KMIDOUT) { + return arg + 1; + } - // If using multiple command queues, go back to a single command queue - mergeQueue(); + else { //if (kern == KFFTHIN || kern == KTAILMUL || kern == KTAILMULLOW || kern == KFFTW) { + return arg + 2; + } } // Call the appropriate kernels to support hybrid FFTs and NTTs void Gpu::fftP(Buffer& buf, Buffer& in) { + // Work around a troublesome oddball case. ModMul calls fftP and fftMidIn on one multiplication argument. If !in_place, fftP writes to buf3, and fftMidIn is queued. + // Modmul then calls fftP on the other multiplication argument. If we don't replay now, fftP overwrite buf3. + replay(); // If not in place, instead write the output to the scratch buffer Buffer *out = in_place ? &buf : &buf3; kfftP(*out, in); @@ -1237,7 +1545,7 @@ void Gpu::fftW(Buffer& out, Buffer& in) { recorded_kernel_args.push_back(&out); recorded_kernel_args.push_back(&in); // This kernel always ends the "bottom half". Replay the recorded kernel calls. - replay(); + endBottomHalf(); } void Gpu::carryA(Buffer& out, Buffer& in) { @@ -1258,29 +1566,29 @@ void Gpu::carryLL(Buffer& out, Buffer& in) { void Gpu::carryFused(Buffer& buf) { // This kernel always ends the "bottom half". Replay the recorded kernel calls. - replay(); - assert(roePos <= ROE_SIZE); + endBottomHalf(); // Like fftP, if not in place write the output to the scratch buffer Buffer *in = &buf; Buffer *out = in_place ? &buf : &buf3; + assert(roePos <= ROE_SIZE); roePos < wantROE ? kCarryFusedROE(*out, *in, roePos++) : kCarryFused(*out, *in, updateCarryPos(1 << 0)); } void Gpu::carryFusedMul(Buffer& buf) { // This kernel always ends the "bottom half". Replay the recorded kernel calls. - replay(); - assert(roePos <= ROE_SIZE); + endBottomHalf(); // Like fftP, if not in place write the output to the scratch buffer Buffer *in = &buf; Buffer *out = in_place ? &buf : &buf3; + assert(roePos <= ROE_SIZE); roePos < wantROE ? kCarryFusedMulROE(*out, *in, roePos++) : kCarryFusedMul(*out, *in, updateCarryPos(1 << 1)); } void Gpu::carryFusedLL(Buffer& buf) { // This kernel always ends the "bottom half". Replay the recorded kernel calls. - replay(); + endBottomHalf(); // Like fftP, if not in place write the output to the scratch buffer Buffer *in = &buf; Buffer *out = in_place ? &buf : &buf3; @@ -1323,11 +1631,19 @@ vector> Gpu::makeBufVector(u32 size) { pair Gpu::readROE() { assert(roePos <= ROE_SIZE); if (roePos) { - vector roe = bufROE.read(roePos); - assert(roe.size() == roePos); - bufROE.zero(roePos); - roePos = 0; + vector roe = bufROE.read(roePos + 2); + assert(roe.size() == roePos + 2); + // Split the roe buffer into two. One for squarings and one for multiplications. This is likely overkill as the multiplication ROE is not used - though + // it could be useful for debugging (in which case we could support getting roe for squarings or multipplications, but not both). auto [squareRoe, mulRoe] = split(roe, mulRoePos); + // Delete first two used to calculate roePos on the GPU. Do this after splitting the vector (mulRoePos recorded indices in "+ 2" format). + u32 squareRoeSize = squareRoe.size() - 2; + roe[0] = squareRoe[squareRoeSize]; + roe[1] = squareRoe[squareRoeSize+1]; + squareRoe.resize(squareRoeSize); + // Clear the ROE buffer and mulRoePos vector + bufROE.zero(roePos + 2); + roePos = 0; mulRoePos.clear(); return {roeStat(squareRoe), roeStat(mulRoe)}; } else { @@ -1338,9 +1654,14 @@ pair Gpu::readROE() { RoeInfo Gpu::readCarryStats() { assert(carryPos <= CARRY_SIZE); if (carryPos == 0) { return {}; } - vector carry = bufStatsCarry.read(carryPos); - assert(carry.size() == carryPos); - bufStatsCarry.zero(carryPos); + vector carry = bufStatsCarry.read(carryPos + 2); + assert(carry.size() == carryPos + 2); + // Delete first two used to calculate carryPos on the GPU. + carry[0] = carry[carryPos]; + carry[1] = carry[carryPos+1]; + carry.resize(carryPos); + // Clear the GPU buffer + bufStatsCarry.zero(carryPos + 2); carryPos = 0; RoeInfo ret = roeStat(carry); @@ -1412,7 +1733,7 @@ void Gpu::mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, bool fftW(buf3, tmp1); // Register the current ROE pos as multiplication (vs. a squaring) - if (mulRoePos.empty() || mulRoePos.back() < roePos) { mulRoePos.push_back(roePos); } + if (mulRoePos.empty() || mulRoePos.back() < roePos) { mulRoePos.push_back(roePos + 2); } if (mul3) { carryM(ioA, buf3); } else { carryA(ioA, buf3); } carryB(ioA); @@ -1427,7 +1748,6 @@ void Gpu::modMul(Buffer& ioA, Buffer& inB, bool mul3) { void Gpu::modMul(Buffer& ioA, Buffer& inB, enum LEAD_TYPE leadInB, bool mul3) { if (leadInB == LEAD_NONE) fftP(buf1, inB); if (leadInB != LEAD_MIDDLE) fftMidIn(buf1); - replay(); // Work around an odd bug. The above executed fftP writing to buf3 if !in_place and queued fftMidIn. If we don't replay now, mul will call fftP again overwriting buf3. mul(ioA, buf1, buf2, mul3); }; @@ -1631,6 +1951,27 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu // LL does not do Mul3 assert(!(doMul3 && doLL)); + // Use CUDA graphs for some common squarings + // NOTE: assumes that if doLL is set, it will always be set + bool graph_recording = false; + Graph *graph = NULL; + if (use_graphs && (&out == &bufData || &out == &bufAux) && &in == &out && leadIn == LEAD_WIDTH && leadOut == LEAD_WIDTH && !doMul3) { + // We have one graph for ROE and one for no-ROE and one for bufData and one for bufAux + bool roe = (roePos < wantROE); + bool srcData = (&out == &bufData); + graph = &graph_square[2 * roe + srcData]; + // Execute an already recorded graph + if (graph->isRecorded()) { + graph->launch(&queue); + queue.incSquareCount(); + if (roe) roePos++; // WARNING: If we ever graph Gpu::Mul, we'll need to also maintain mulRoePos vector. + return; + } + // Otherwise, record a new graph + graph->beginRecording(&queue); + graph_recording = true; + } + // In place FFTs use buf1. Not in place FFTs also use buf3. // If leadIn is LEAD_NONE, in contains the input data, squaring starts at fftP // If leadIn is LEAD_WIDTH, buf1 (or buf3 if not in place) contains the input data, squaring starts at fftMidIn @@ -1664,6 +2005,12 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu carryFused(buf1); } } + + // End CUDA graph recording (and execute the just recorded graph) + if (graph_recording) { + graph->endRecording(&queue); + graph->launch(&queue); + } } u32 Gpu::squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3) { diff --git a/src/Gpu.h b/src/Gpu.h index e7a2b810..676bd430 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -87,7 +87,7 @@ class Gpu { Background* background; public: - const Args& args; + Args& args; private: std::unique_ptr> saver; @@ -223,10 +223,16 @@ class Gpu { vector recorded_kernels; vector *> recorded_kernel_args; + bool use_graphs; + Graph graph_square[4]; + const int NUM_CACHE_GROUPS = 3; void splitQueue(void); void mergeQueue(void); + void endBottomHalf(void); void replay(void); + void replay_one(enum BOTTOM_HALF_KERNELS kern, int cache_group, int arg, Queue *q = NULL, int base = 0, int kernelsToExecuteX = 0, int kernelsToExecuteY = 1); + int replay_next_arg(enum BOTTOM_HALF_KERNELS kern, int arg); void fftP(Buffer& out, Buffer& in) { fftP(out, reinterpret_cast&>(in)); } void fftP(Buffer& out, Buffer& in); diff --git a/src/Kernel.cpp b/src/Kernel.cpp index 5da47088..fe1cc462 100644 --- a/src/Kernel.cpp +++ b/src/Kernel.cpp @@ -15,7 +15,8 @@ Kernel::Kernel(string_view name, KernelCompiler* compiler, TimeInfo* timeInfo, Q defines{defines}, timeInfo{timeInfo}, queue{queue}, - workSize{workSize} + workSizeX{workSize}, + workSizeY{1} {} Kernel::~Kernel() = default; @@ -33,12 +34,27 @@ void Kernel::finishLoad() { assert(kernel); groupSize = getWorkGroupSize(kernel.get(), deviceId, name.c_str()); assert(groupSize); - assert(workSize % groupSize == 0); + assert(workSizeX % groupSize == 0); for (auto [pos, arg] : pendingArgs) { setArgs(pos, arg); } } +void Kernel::setKernelsToExecute(size_t nX, size_t nY) { // The rare two-dimensional kernel execution + if (nX == 0) return; // Use the default work size set at object creation. + + // Make sure kernel is loaded so that we have the groupSize + if (!kernel) { + startLoad(compiler); + finishLoad(); + } + if (!kernel) { throw std::runtime_error("OpenCL kernel "s + name + " not found"); } + + // For 2D kernels, we only support groupSizeY of 1. Setting workSizeY to more than one indicates a 2D kernel execution. + workSizeX = nX * groupSize; + workSizeY = nY; +} + void Kernel::run() { assert(kernel); - queue->run(kernel.get(), groupSize, workSize, timeInfo); + queue->run(kernel.get(), groupSize, workSizeX, workSizeY, timeInfo); } diff --git a/src/Kernel.h b/src/Kernel.h index 67a7bf50..9f9dc8b6 100644 --- a/src/Kernel.h +++ b/src/Kernel.h @@ -24,7 +24,8 @@ class Kernel { TimeInfo *timeInfo; Queue* queue; - size_t workSize; + size_t workSizeX; + size_t workSizeY; u32 groupSize = 0; KernelHolder kernel{}; @@ -44,10 +45,14 @@ class Kernel { void finishLoad(); // Change which queue is used to run a kernel - void setQueue(Queue *q) { queue = q; } + void setQueue(Queue *q) { if (q != NULL) queue = q; } + + // Change number of kernels to execute. Usually this is set by Gpu.cpp at object creation (by setting the total number of work-items in workSizeX). + // L2 striping requires the ability to change this setting on-the-fly. One and two dimensional kernels are supported. + void setKernelsToExecute(size_t nX, size_t nY = 1); template void setFixedArgs(int pos, const Args &...tail) { setArgs(pos, tail...); } - + template void operator()(const Args &...args) { if (!kernel) { startLoad(compiler); diff --git a/src/Queue.cpp b/src/Queue.cpp index 0b2d0baf..be2f8044 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -25,9 +25,10 @@ Queue::Queue(const Context& context, bool profile, bool auxQueue) : markerEvent{}, markerQueued(false), queueCount(0), + squareCount(0), squareTime(50), - squareKernels(4), - firstSetTime(true) + firstSetTime(true), + graphRecording(false) { // Formerly a constant (thus the CAPS). nVidia is 3% CPU load at 400 or 500, and 35% load at 800 on my Linux machine. // AMD is just over 2% load at 1600 and 3200 on the same Linux machine. Marginally better timings(?) at 3200. @@ -56,9 +57,9 @@ void Queue::print() { void Queue::add(EventHolder&& e, TimeInfo* ti) { if (hasEvents) { events.emplace_back(std::move(e), ti); } - if (isAuxQueue) return; + if (isAuxQueue || graphRecording) return; queueCount++; - if (queueCount == MAX_QUEUE_COUNT) queueMarkerEvent(); + if (queueCount >= MAX_QUEUE_COUNT) queueMarkerEvent(); } void Queue::readSync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo) { @@ -74,20 +75,21 @@ void Queue::copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo) { add(::copyBuf(get(), {}, src, dst, size, hasEvents), tInfo); } -void Queue::run(cl_kernel kernel, size_t groupSize, size_t workSize, TimeInfo* tInfo) { - add(::run(get(), kernel, groupSize, workSize, {}, tInfo->name, hasEvents), tInfo); +void Queue::run(cl_kernel kernel, size_t groupSizeX, size_t workSizeX, size_t workSizeY, TimeInfo* tInfo) { + add(::run(get(), kernel, groupSizeX, workSizeX, workSizeY, {}, tInfo->name, hasEvents), tInfo); } void Queue::finish() { - assert (!isAuxQueue); + assert(!isAuxQueue); waitForMarkerEvent(); ::finish(get()); events.synced(); queueCount = 0; + squareCount = 0; } void Queue::queueMarkerEvent() { - assert (!isAuxQueue); + assert(!isAuxQueue); waitForMarkerEvent(); if (queueCount) { // AMD GPUs have no trouble waiting for a finish without a CPU busy wait. So, instead of markers and events, simply run finish every now and then. @@ -99,24 +101,26 @@ void Queue::queueMarkerEvent() { markerEvent = enqueueMarker(get()); markerQueued = true; queueCount = 0; + squareCount = 0; } } } void Queue::waitForMarkerEvent() { - assert (!isAuxQueue); + assert(!isAuxQueue); if (!markerQueued) return; // By default, nVidia finish causes a CPU busy wait. Instead, sleep for a while. Since we know how many items are enqueued after the marker we can make an // educated guess of how long to sleep to keep CPU overhead low. while (getEventInfo(markerEvent.get()) != CL_COMPLETE) { - // There are 4, 7, or 10 kernels per squaring. Don't overestimate sleep time. Divide by much more than the number of kernels. - std::this_thread::sleep_for(std::chrono::microseconds(1 + queueCount * squareTime / squareKernels / 2)); + // There are usually 4, 7, or 10 kernels per squaring. Use a rolling average to create a very accurate kernel count. + // Don't overestimate sleep time. Divide by much more than the number of kernels. + std::this_thread::sleep_for(std::chrono::microseconds((squareCount + 1) * squareTime / 2)); } markerQueued = false; } void Queue::setSquareTime(int time) { - assert (!isAuxQueue); + assert(!isAuxQueue); if (firstSetTime) { // Ignore first setSquareTime call. First measured times are wrong because of startup costs firstSetTime = false; return; diff --git a/src/Queue.h b/src/Queue.h index 3e0b3ab0..3066e1c6 100644 --- a/src/Queue.h +++ b/src/Queue.h @@ -45,26 +45,52 @@ class Queue : public QueueHolder { template void fillBuf(cl_mem buf, T pattern, u32 size, TimeInfo* tInfo) { fillBufTE(buf, sizeof(T), &pattern, size, tInfo); } - void run(cl_kernel kernel, size_t groupSize, size_t workSize, TimeInfo* tInfo); + void run(cl_kernel kernel, size_t groupSizeX, size_t workSizeX, size_t workSizeY, TimeInfo* tInfo); void readSync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo); void readAsync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo); void copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo); void finish(); - EventHolder createSyncEvent(void) { return enqueueMarker(get()); } // Enqueue a synchronization event. Used to sync work among multiple queues. - void waitForSyncEvent(EventHolder* e) { enqueueMarkerWithWaits(get(), {e->get()}); } // Wait for a synchronization event to complete. + EventHolder createSyncEvent(void) { if (!isAuxQueue && !graphRecording) queueCount++; return enqueueMarker(get()); } // Enqueue a synchronization event. Used to sync work among multiple queues. + void waitForSyncEvent(EventHolder* e) { if (!isAuxQueue && !graphRecording) queueCount++; enqueueMarkerWithWaits(get(), {e->get()}); } // Wait for a synchronization event to complete. + void incSquareCount(int n = 1) { squareCount += n; } void setSquareTime(int); // Update the time to do one squaring (in microseconds) - void setSquareKernels(int n) { squareKernels = n; firstSetTime = true; } + + void beginRecording(void) { graphRecording = true; CHECK1(clGraphBeginRecording(get())); } + void endRecording(cl_graph *graph) { graphRecording = false; CHECK1(clGraphEndRecording(get(), graph)); } + void playRecording(cl_graph graph) { CHECK1(clGraphLaunch(graph)); add(EventHolder{}, NULL); } private: // This replaces the "call queue->finish every 400 squarings" code in Gpu.cpp. Solves the busy wait on nVidia GPUs. int MAX_QUEUE_COUNT; // Queue size before a marker will be enqueued. Typically, 100 to 1000 squarings. EventHolder markerEvent; // Event associated with an enqueued marker placed in the queue every MAX_QUEUE_COUNT entries and before r/w operations. bool markerQueued; // TRUE if a marker and event have been queued int queueCount; // Count of items added to the queue since last marker + int squareCount; // Count of squarings/multiplies since last marker queued int squareTime; // Time to do one squaring (in microseconds) - int squareKernels; // Number of kernels in one squaring bool firstSetTime; // Flag so we can ignore first setSquareTime call (which is inaccurate because of all the initial openCL compiles) + bool graphRecording; // Graph recording in progress. waitForMarkerEvent and enqueueMarker must be avoided. void queueMarkerEvent(); // Queue the marker event void waitForMarkerEvent(); // Wait for marker event to complete }; + + + +// Wrapper class for our OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA graphs feature + +class Graph { + +public: + Graph() : graph{} {} + ~Graph() { if (graph) release(graph); } + + bool isSupported(cl_device_id id) { return clIsGraphSupported(id); } + void beginRecording(Queue *q) { q->beginRecording(); } + void endRecording(Queue *q) { q->endRecording(&graph); } + bool isRecorded() { return graph != NULL; } + void launch(Queue *q) { q->playRecording(graph); } + +private: + cl_graph graph; +}; + diff --git a/src/cl/base.cl b/src/cl/base.cl index ae20149a..4add30e8 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -200,6 +200,14 @@ G_H "group height" == SMALL_HEIGHT / NH #define ZEROHACK_H 1 #endif +#if !defined(MULTI_Q) +#define MULTI_Q 0 +#endif + +#if !defined(L2_STRIPING) +#define L2_STRIPING 0 +#endif + // Expected defines: EXP the exponent. // WIDTH, SMALL_HEIGHT, MIDDLE. diff --git a/src/cl/carry.cl b/src/cl/carry.cl index fe2ce3d3..a8ba21c1 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -41,8 +41,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) + local u32 lds[G_W]; updateStats(bufROE, posROE, carryMax); #endif } @@ -98,9 +100,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(F2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } @@ -171,10 +175,12 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF31) in, u32 posROE, P(CarryABM) carryOut, P carryOut[G_W * g + me] = carry; #if ROE + local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) M31; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, H, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } @@ -245,10 +251,12 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF61) in, u32 posROE, P(CarryABM) carryOut, P carryOut[G_W * g + me] = carry; #if ROE + local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) (M61 >> 32); // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, H, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } @@ -324,9 +332,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } @@ -407,9 +417,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } @@ -490,9 +502,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } @@ -580,10 +594,12 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, P(u carryOut[G_W * g + me] = carry; #if ROE + local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) 0x1FFFFFFF; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, H, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } @@ -680,9 +696,11 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big carryOut[G_W * g + me] = carry; #if ROE - updateStats(bufROE, posROE, roundMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) - updateStats(bufROE, posROE, carryMax); + local u32 lds[G_W]; + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 9a887a76..8f70e16a 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -198,12 +198,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } frac_bits = starting_frac_bits; // Restore starting frac_bits for applying weights after carry propagation -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -238,6 +232,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Calculate inverse weights T base = optionalHalve(weights.y); for (u32 i = 0; i < NW; ++i) { @@ -410,12 +410,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut frac_bits += frac_bits_bigstep; } -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -450,6 +444,12 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); @@ -627,13 +627,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - float fltRoundMax = (float) roundMax / (float) M31; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -668,6 +661,13 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry __asm("s_setprio 0"); #endif +#if ROE + float fltRoundMax = (float) roundMax / (float) M31; // For speed, roundoff was computed as 32-bit integer. Convert to float. + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, fltRoundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); @@ -849,13 +849,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - float fltRoundMax = (float) roundMax / (float) (M61 >> 32); // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(bufROE, posROE, fltRoundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -890,6 +883,13 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry __asm("s_setprio 0"); #endif +#if ROE + float fltRoundMax = (float) roundMax / (float) (M61 >> 32); // For speed, roundoff was computed as 32-bit integer. Convert to float. + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, fltRoundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); @@ -1087,12 +1087,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -1127,6 +1121,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Calculate inverse weights T base = optionalHalve(weights.y); for (u32 i = 0; i < NW; ++i) { @@ -1351,12 +1351,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -1391,6 +1385,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) ldsF2, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) ldsF2, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Shuffle carries up shufl_carries_up(ldsF2, carry, me, lowMe); @@ -1611,12 +1611,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } combo_counter = starting_combo_counter; // Restore starting counter for applying weights after carry propagation -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -1651,6 +1645,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); @@ -1865,13 +1865,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut m31_combo_counter = m31_starting_combo_counter; // Restore starting counter for applying weights after carry propagation m61_combo_counter = m61_starting_combo_counter; -#if ROE - float fltRoundMax = (float) roundMax / (float) 0x1FFFFFFF; // For speed, roundoff was computed as 32-bit integer. Convert to float - divide by M61. - updateStats(bufROE, posROE, fltRoundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -1906,6 +1899,13 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif +#if ROE + float fltRoundMax = (float) roundMax / (float) 0x1FFFFFFF; // For speed, roundoff was computed as 32-bit integer. Convert to float - divide by M61. + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, fltRoundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); @@ -2153,12 +2153,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut m31_combo_counter = m31_starting_combo_counter; // Restore starting counter for applying weights after carry propagation m61_combo_counter = m61_starting_combo_counter; -#if ROE - updateStats(bufROE, posROE, roundMax); -#elif STATS & (1 << MUL3) - updateStats(bufROE, posROE, carryMax); -#endif - // Write out our carries for the last line in this group. Only groups 0 to H/WMUL-1 need to write carries out. // Group H/WMUL is a duplicate of group 0 (producing the same results) so we don't care about that group writing out, // but it's fine either way. @@ -2193,6 +2187,12 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut __asm("s_setprio 0"); #endif +#if ROE + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, roundMax); +#elif STATS & (1 << MUL3) + updateStats((local u32 *) lds61, G_W * WMUL, H / WMUL, bufROE, posROE, carryMax); +#endif + // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 57bf5adb..64c83a11 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -135,14 +135,48 @@ float OVERLOAD boundCarry(i32 c) { return ldexp(fabs((float) c), -32); } float OVERLOAD boundCarry(i64 c) { return ldexp(fabs((float) (i32) (c >> 8)), -24); } #if STATS || ROE -void updateStats(global uint *bufROE, u32 posROE, float roundMax) { +void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *bufROE, u32 posROE, float roundMax) { assert(roundMax >= 0); - // work_group_reduce_max() allocates an additional 256Bytes LDS for a 64lane workgroup, so avoid it. - // u32 groupRound = work_group_reduce_max(as_uint(roundMax)); - // if (get_local_id(0) == 0) { atomic_max(bufROE + posROE, groupRound); } - // Do the reduction directly over global mem. - atomic_max(bufROE + posROE, as_uint(roundMax)); + // This barrier may be needed by carryFused because this code does not partition lds memory the same way shufl does + bar(); + // Reduce to a single roundMax value + u32 me = get_local_id(0); + u32 u32RoundMax = as_uint(roundMax); + while (num_threads > 1) { + // Write roundMax for high half of threads to local memory. Ignore threads not participating in the reduction. + if (num_threads >= WAVEFRONT) bar(); + __asm("bar.sync 0;"); + if (me >= num_threads / 2 && me < num_threads) lds[me - num_threads / 2] = u32RoundMax; + if (num_threads > WAVEFRONT) { + bar(); // work around a weird CUDA NVCC bug where two bar() calls are required???! (Titan V, CUDA 13.0, WMUL=2) + bar(); + } + // Low half of threads do a max + if (me < num_threads / 2) { + u32 highHalfMax = lds[me]; + if (u32RoundMax < highHalfMax) u32RoundMax = highHalfMax; + } + // Cut num threads in half, loop + num_threads /= 2; + } + +// We could use shfl_down_sync (and AMD's equivalent) instead of LDS memory once num_threads < WAVEFRONT (see https://github.com/mahmoudmaftah/MaxReduction-Cuda/blob/main/code/reduction_benchmarks.cu) +// We could instead write the reduced ROE sequentially to bufROE and then do a max_reduction after last atomic_add + + // Merge this max with others + if (me == 0) { + // The bufROE entry to update is stored in the first bufROE entry. This value used to be passed into carryFused as an argument. + // CUDA graphs don't allow arguments to change. Thus, calculating posROE and storing it in bufROE workd better. + posROE = bufROE[0]; + atomic_max(bufROE + posROE + 2, u32RoundMax); + // The second bufRoe entry is a count of the number atomic_maxes performed. When the last atomic_max is done, increment posROE and clear the counter. + u32 old_value = atomic_add(bufROE + 1, 1); + if (old_value == num_blocks - 1) { + bufROE[0] = posROE + 1; + bufROE[1] = 0; + } + } } #endif @@ -685,7 +719,7 @@ Word OVERLOAD carryStepSignedSloppy(i96 x, i64 *outCarry, bool isBigWord) { const u32 bigwordBits = EXP / NWORDS + 1; u32 nBits = bitlen(isBigWord); #if EXP / NWORDS >= 32 // nBits is 32 or more - return carryStep(x, outCarry, isBigWord); // Should be just as fast as code below + return carryStep(x, outCarry, isBigWord); // Should be just as fast as code below // u32 xmid_topbit = i96_mid32(x) & (1 << (bigwordBits - 32 - 1)); // i32 whi = ulowFixedBits(i96_mid32(x), bigwordBits - 32 - 1) - xmid_topbit; // i64 xhi = i96_hi64(x) + xmid_topbit; @@ -696,7 +730,7 @@ Word OVERLOAD carryStepSignedSloppy(i96 x, i64 *outCarry, bool isBigWord) { *outCarry = (i96_hi64(x) + (w < 0)) << (32 - nBits); return w; #else // nBits less than 32 - return carryStep(x, outCarry, isBigWord); // Should be faster than code below + return carryStep(x, outCarry, isBigWord); // Should be faster than code below // i32 w = lowFixedBits(i96_lo32(x), bigwordBits); // *outCarry = (as_long((int2)(xtract32(i96_lo64(x), bigwordBits), xtract32(i96_hi64(x), bigwordBits))) + (w < 0)) << (bigwordBits - nBits); // return w; diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index 02d388d3..52c39322 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -4,27 +4,52 @@ #include "fftheight.cl" #include "middle.cl" +// If not doing L2 stripes, process the lines in any order. +// If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 2 * stripe_group_size * 16 * MIDDLE tailSquare lines. +u32 get_line_number(u32 base_lo) { + u32 g = get_group_id(0); +#if L2_STRIPING + // Old, simple L2 striping code + // return g / (L2_STRIPING * 16) * WIDTH + base + g % (L2_STRIPING * 16); + + // Process stripe group base_lo or base_hi + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + u32 linesInOneStripe = 16 * MIDDLE; + u32 stripe_group_size = L2_STRIPING; + u32 linesInOneStripeGroup = stripe_group_size * linesInOneStripe; + u32 base; + if (g linesInOneStripeGroup) base = base_lo; + else base = base_hi, g -= linesInOneStripeGroup; + return g / (L2_STRIPING * 16) * WIDTH + base + g % (L2_STRIPING * 16); +#else + return g; +#endif +} + #if FFT_FP64 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES / sizeof(T2)]; + const u32 H = ND / SMALL_HEIGHT; T2 u[NH]; - u32 g = get_group_id(0); + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(in, u, g, me); + readTailFusedLine(in, u, line, me); -#if NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif fft_HEIGHT(lds, u, smallTrig, w, 1, me); - write(G_H, NH, u, out, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, out, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif @@ -37,28 +62,31 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { #if FFT_FP32 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES / sizeof(F2)]; + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; TrigFP32 smallTrigF2 = (TrigFP32) smallTrig; F2 u[NH]; - u32 g = get_group_id(0); + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(inF2, u, g, me); + readTailFusedLine(inF2, u, line, me); -#if NH == 8 - F2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + F2 w = fancyTrig_N(H * me); #else - F2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + F2 w = slowTrig_N(H * me, ND / NH); #endif fft_HEIGHT(lds, u, smallTrigF2, 1, me); - write(G_H, NH, u, outF2, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, outF2, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif @@ -71,7 +99,7 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF31 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES / sizeof(GF31)]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -79,14 +107,14 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); GF31 u[NH]; - u32 g = get_group_id(0); + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(in31, u, g, me); + readTailFusedLine(in31, u, line, me); fft_HEIGHT(lds, u, smallTrig31, 1, me); - write(G_H, NH, u, out31, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, out31, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif @@ -99,7 +127,7 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF61 // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) -KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES / sizeof(GF61)]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -107,14 +135,14 @@ KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, Trig smallTrig) { TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); GF61 u[NH]; - u32 g = get_group_id(0); + u32 line = get_line_number(base); u32 me = get_local_id(0); - readTailFusedLine(in61, u, g, me); + readTailFusedLine(in61, u, line, me); fft_HEIGHT(lds, u, smallTrig61, 1, me); - write(G_H, NH, u, out61, SMALL_HEIGHT * transPos(g, MIDDLE, WIDTH)); + write(G_H, NH, u, out61, SMALL_HEIGHT * transPos(line, MIDDLE, WIDTH)); } #endif diff --git a/src/cl/fftmiddlein.cl b/src/cl/fftmiddlein.cl index 1c97e612..a7c34538 100644 --- a/src/cl/fftmiddlein.cl +++ b/src/cl/fftmiddlein.cl @@ -8,7 +8,7 @@ #if FFT_FP64 -KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { T2 u[MIDDLE]; u32 SIZEY = IN_WG / IN_SIZEX; @@ -65,7 +65,7 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { F2 u[MIDDLE]; CP(F2) inF2 = (CP(F2)) in; @@ -126,7 +126,7 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF31 u[MIDDLE]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -187,7 +187,7 @@ KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF61 u[MIDDLE]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -197,7 +197,7 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { u32 SIZEY = IN_WG / IN_SIZEX; u32 N = WIDTH / IN_SIZEX; - + u32 g = get_group_id(0); u32 gx = g % N; u32 gy = g / N; @@ -244,18 +244,68 @@ KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, Trig trig) { - +// fftMiddleIn processes lines output by fftP or carryFused. Call this the x coordinate with range 0..WIDTH-1. The y coordinate ranges from 0..MIDDLE*SMALL_HEIGHT-1. +// In place transpose processeses blocks of 16 x coordinates by 16 y coordinates. fftMiddleIn processes processes MIDDLE blocks at a time. +// fftMiddleIn outputs lines for tailSquare. The y coordinate from 0..SMALL_HEIGHT-1 is transposed into the x coordinate for tailSquare. +// +// fftMiddleIn can work on all the FFT data, in which case we can process blocks in any order. Sequentially through memory by increasing x coordinates first might be best. +// More interestingly, fftMiddleIn can be configured to work on smaller amounts of FFT data in hopes that the data will stay in the L2 cache during the tailSquare and +// fftMiddleOut kernels. I call this L2 striping. In this case we process "columns of FFT data" by processing all the y coordinates to create complete lines for tailSquare. +// We must also output lines x and N-x for tailSquare handling of Hermetian symmetry. #else // in place transpose +// L2 striping processes both base_lo and base_hi (see Gpu.cpp) in one kernel call to reduce kernel launch overhead. There is also some special handling required for the +// first and last stripe groups. This results in a more complicated map group_id to compute the startx and starty coordinates. +#if L2_STRIPING +void map_striping_group_id(u32 base_lo, u32 g, u32 *startx, u32 *starty) { + // Old, simple L2 striping code + // u32 N = SMALL_HEIGHT / 16; + // u32 starty = g % N * 16; + // u32 startx = base + g / N * 16; + + // If MIDDLE is odd, the first fftMiddleIn must process the special N/2 tailSquare line from the WIDTH/2 stripe. + // If MULTI_Q, the first fftMiddleIn in the second queue must also process the 3*WIDTH/4 stripe. + u32 oneStripeKernelsToExecute = SMALL_HEIGHT / 16; + if ((base_lo == 0 && (MIDDLE & 1)) || (MULTI_Q && base_lo == WIDTH / 4)) { + if (g < oneStripeKernelsToExecute) { + *startx = base_lo + WIDTH / 2; + *starty = g * 16; + return; + } + g -= oneStripeKernelsToExecute; + } + + // Process stripe group base_lo. + u32 stripe_group_size = L2_STRIPING; + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + if (g < kernelsToExecute) { + *startx = base_lo + g / oneStripeKernelsToExecute * 16; + *starty = g % oneStripeKernelsToExecute * 16; + return; + } + g -= kernelsToExecute; + + // Process stripe group base_hi. The last group must take into account that the first stripe may have already been processed. + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + if ((base_hi == WIDTH / 2 && (MIDDLE & 1)) || (MULTI_Q && base_hi == 3 * WIDTH / 4)) base_hi += 16; + *startx = base_hi + g / oneStripeKernelsToExecute * 16; + *starty = g % oneStripeKernelsToExecute * 16; +} +#endif + #if FFT_FP64 -KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); T2 u[MIDDLE]; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = (MIDDLE >= 16) ? 0 : g / 131072; // Rocm optimizer goes bonkers if zerohack used when MIDDLE=16 +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; @@ -301,7 +351,7 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); F2 u[MIDDLE]; @@ -310,7 +360,11 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { TrigFP32 trigF2 = (TrigFP32) trig; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = 0; // Need to test if g / 131072 is of any benefit +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; @@ -356,7 +410,7 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF31 u[MIDDLE]; @@ -365,7 +419,11 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { TrigGF31 trig31 = (TrigGF31) (trig + DISTMTRIGGF31); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = 0; // Need to test if g / 131072 is of any benefit +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; @@ -411,7 +469,7 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF61 u[MIDDLE]; @@ -420,7 +478,11 @@ KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, Trig trig) { TrigGF61 trig61 = (TrigGF61) (trig + DISTMTRIGGF61); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); + u32 zerohack = 0; // Need to test if g / 131072 is of any benefit +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 starty = g % N * 16; u32 startx = g / N * 16; diff --git a/src/cl/fftmiddleout.cl b/src/cl/fftmiddleout.cl index b7c486cb..b3c18d96 100644 --- a/src/cl/fftmiddleout.cl +++ b/src/cl/fftmiddleout.cl @@ -8,7 +8,7 @@ #if FFT_FP64 -KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { T2 u[MIDDLE]; u32 SIZEY = OUT_WG / OUT_SIZEX; @@ -73,7 +73,7 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { F2 u[MIDDLE]; CP(F2) inF2 = (CP(F2)) in; @@ -139,7 +139,7 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF31 u[MIDDLE]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -202,7 +202,7 @@ KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { +KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF61 u[MIDDLE]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -260,16 +260,58 @@ KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, Trig trig) { +// fftMiddleOut processes lines output by tailSquare or tailMul. Call this the x coordinate with range 0..SMALL_HEIGHT-1 +// fftMiddleOut outputs lines for carryFused or fftW. Call this the y coordinate with range 0..WIDTH-1 +// In place transpose processeses blocks of 16 x coordinates by 16 y coordinates. +// fftMiddleOut processes processes MIDDLE blocks at a time. +// +// fftMiddleOut can work on all the FFT data, in which case we can process blocks in any order. Sequentially through memory by increasing x coordinates first might be best. +// More interestingly, fftMiddleOut can work on smaller amounts of FFT data in hopes that the data has stayed in the L2 cache during the fftMiddleIn/tailSquare/fftMiddleOut kernels. +// In this case we must work through all the x coordinates to read complete lines from tailSquare. We must also read y and N-y due to Hermetian symmetry. + + #else // in place transpose +// L2 striping processes both base_lo and base_hi (see Gpu.cpp) in one kernel call to reduce kernel launch overhead. There is also some special handling required for the +// first and last stripe groups. This results in some more complicated to map group_id into startx and starty coordinates. +#if L2_STRIPING +void map_striping_group_id(u32 base_lo, u32 g, u32 *startx, u32 *starty) { + // Old, simple L2 striping code + // u32 N = SMALL_HEIGHT / 16; + // u32 startx = g % N * 16; + // u32 starty = base + g / N * 16; + + // Process stripe group from base_lo. + u32 oneStripeKernelsToExecute = SMALL_HEIGHT / 16; + u32 stripe_group_size = L2_STRIPING; + u32 kernelsToExecute = stripe_group_size * oneStripeKernelsToExecute; + if (g < kernelsToExecute) { + *startx = g % oneStripeKernelsToExecute * 16; + *starty = base_lo + g / oneStripeKernelsToExecute * 16; + return; + } + g -= kernelsToExecute; + + // Process stripe group from base_hi. The first stripe in the base_hi stripe group is not ready for output (except in the last group). + // The last group processes the stripe that was skipped in the first base_hi group. + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + u32 base = (base_hi == WIDTH / 2 || (MULTI_Q && base_hi == 3 * WIDTH / 4)) ? base_hi : base_hi + 16; // Skip first stripe in base_hi (usually) + *startx = g % oneStripeKernelsToExecute * 16; + *starty = base + g / oneStripeKernelsToExecute * 16; +} +#endif + #if FFT_FP64 -KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); T2 u[MIDDLE]; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; @@ -317,7 +359,7 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { #if FFT_FP32 -KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); F2 u[MIDDLE]; @@ -326,7 +368,10 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { TrigFP32 trigF2 = (TrigFP32) trig; u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; @@ -371,7 +416,7 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF31 -KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF31 u[MIDDLE]; @@ -380,7 +425,10 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { TrigGF31 trig31 = (TrigGF31) (trig + DISTMTRIGGF31); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; @@ -422,7 +470,7 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, Trig trig) { #if NTT_GF61 -KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { +KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF61 u[MIDDLE]; @@ -431,7 +479,10 @@ KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, Trig trig) { TrigGF61 trig61 = (TrigGF61) (trig + DISTMTRIGGF61); u32 g = get_group_id(0); -#if INPLACE == 1 // nVidia friendly padding +#if L2_STRIPING + u32 startx, starty; + map_striping_group_id(base, g, &startx, &starty); +#elif INPLACE == 1 // nVidia friendly padding u32 N = SMALL_HEIGHT / 16; u32 startx = g % N * 16; u32 starty = g / N * 16; diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index decc0e61..957aace9 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -5,6 +5,33 @@ #include "tailutil.cl" #include "middle.cl" +// If not doing L2 stripes, process the lines in any order. +// If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 16 * MIDDLE tailSquare lines. +u32 get_line_number(u32 base) { + u32 g = get_group_id(0); +#if L2_STRIPING + // Old, simple L2 striping code + // return get_group_id(1) * WIDTH + base + g; + + // Process all lines from low half of base_lo stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + u32 base_lo = base; + u32 stripe_group_size = L2_STRIPING; + u32 half_size = (MIDDLE + 1) / 2; // For base_lo, round odd middles up. + u32 kernelsToExecute = half_size * stripe_group_size * 16; + if (g < kernelsToExecute) return g / (stripe_group_size * 16) * WIDTH + base_lo + g % (stripe_group_size * 16); + g -= kernelsToExecute; + + // Process lines from low half of base_hi stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + // The first line in the base_hi stripe group is not ready for processing (except for the last group). + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + if (base_hi != WIDTH / 2) base_hi++; // Skip first line in base_hi (usually) + half_size = MIDDLE / 2; // For base_hi, round odd middles up. + return g % half_size * WIDTH + base_hi + g / half_size; +#else + return g; +#endif +} + #if FFT_FP64 // Handle the final multiplication step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -48,15 +75,14 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s } } -KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES / sizeof(T2)]; + const u32 H = ND / SMALL_HEIGHT; T2 u[NH], v[NH]; T2 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); @@ -70,9 +96,9 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if FFT_VARIANT_H != 0 T2 w; #elif NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif #if MUL_LOW @@ -163,8 +189,9 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s } } -KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES / sizeof(F2)]; + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; CP(F2) aF2 = (CP(F2)) a; @@ -174,9 +201,7 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { F2 u[NH], v[NH]; F2 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); @@ -274,8 +299,9 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar } } -KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES / sizeof(GF31)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); @@ -285,9 +311,7 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { GF31 u[NH], v[NH]; GF31 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); @@ -404,8 +428,9 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar } } -KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES / sizeof(GF61)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); @@ -415,9 +440,7 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { GF61 u[NH], v[NH]; GF61 p[NH], q[NH]; - u32 H = ND / SMALL_HEIGHT; - - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 80dc0260..8d2a9a7e 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -5,6 +5,40 @@ #include "tailutil.cl" #include "middle.cl" +// If not doing L2 stripes, process the lines in any order. +// If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 16 * MIDDLE tailSquare lines. +u32 get_line_number(u32 base) { + u32 g = get_group_id(0); +#if !SINGLE_KERNEL +#if L2_STRIPING + if (base == 0) g = g + 1; +#else + g = g + 1; +#endif +#endif +#if L2_STRIPING + // Old, simple L2 striping code + // return get_group_id(1) * WIDTH + base + g; + + // Process all lines from low half of base_lo stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + u32 base_lo = base; + u32 stripe_group_size = L2_STRIPING; + u32 half_size = (MIDDLE + 1) / 2; // For base_lo, round odd middles up. + u32 kernelsToExecute = half_size * stripe_group_size * 16; + if (g < kernelsToExecute) return g / (stripe_group_size * 16) * WIDTH + base_lo + g % (stripe_group_size * 16); + g -= kernelsToExecute; + + // Process lines from low half of base_hi stripe group. One stripe group is stripe_group_size * 16 * MIDDLE lines. + // The first line in the base_hi stripe group is not ready for processing (except for the last group). + u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; + if (base_hi != WIDTH / 2) base_hi++; // Skip first line in base_hi (usually) + half_size = MIDDLE / 2; // For base_hi, round odd middles up. + return g % half_size * WIDTH + base_hi + g / half_size; +#else + return g; +#endif +} + #if FFT_FP64 // Handle the final squaring step on a pair of complex numbers. Swap real and imaginary results for the inverse FFT. @@ -58,7 +92,7 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { local T2 lds[LDS_BYTES / sizeof(T2)]; T2 u[NH]; - u32 H = ND / SMALL_HEIGHT; + const u32 H = ND / SMALL_HEIGHT; // This kernel in executed in two workgroups. u32 which = get_group_id(0); @@ -75,9 +109,9 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if FFT_VARIANT_H != 0 T2 w; #elif NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif T2 trig = slowTrig_N(line + me * H, ND / NH); @@ -94,20 +128,14 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES / sizeof(T2)]; + const u32 H = ND / SMALL_HEIGHT; T2 u[NH], v[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); @@ -121,9 +149,9 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { #if FFT_VARIANT_H != 0 T2 w; #elif NH == 8 - T2 w = fancyTrig_N(ND / SMALL_HEIGHT * me); + T2 w = fancyTrig_N(H * me); #else - T2 w = slowTrig_N(ND / SMALL_HEIGHT * me, ND / NH); + T2 w = slowTrig_N(H * me, ND / NH); #endif u32 zerohack = ZEROHACK_H * (u32) get_group_id(0) / 131072; @@ -207,21 +235,14 @@ void OVERLOAD pairSq2_special(T2 *u, T2 base_squared) { } } -KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local T2 lds[2 * LDS_BYTES / sizeof(T2)]; + const u32 H = ND / SMALL_HEIGHT; T2 u[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). @@ -348,7 +369,7 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { local F2 lds[LDS_BYTES / sizeof(F2)]; F2 u[NH]; - u32 H = ND / SMALL_HEIGHT; + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -380,8 +401,9 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES / sizeof(F2)]; + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -389,15 +411,8 @@ KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { F2 u[NH], v[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); @@ -489,8 +504,9 @@ void OVERLOAD pairSq2_special(F2 *u, F2 base_squared) { } } -KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local F2 lds[2 * LDS_BYTES / sizeof(F2)]; + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -498,16 +514,8 @@ KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, Trig smallTrig) { F2 u[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). @@ -629,13 +637,13 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { local GF31 lds[LDS_BYTES / sizeof(GF31)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); GF31 u[NH]; - u32 H = ND / SMALL_HEIGHT; // This kernel in executed in two workgroups. u32 which = get_group_id(0); @@ -680,8 +688,9 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES / sizeof(GF31)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -689,15 +698,8 @@ KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 u[NH], v[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); @@ -784,8 +786,9 @@ void OVERLOAD pairSq2_special(GF31 *u, GF31 base_squared) { } } -KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF31 lds[2 * LDS_BYTES / sizeof(GF31)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -793,16 +796,8 @@ KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, Trig smallTrig) { GF31 u[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). @@ -880,10 +875,10 @@ void OVERLOAD onePairSq(GF61* pa, GF61* pb, GF61 t_squared, const u32 t_squared_ // This code should be faster (saves at least one wide mul) but the CUDA compiler makes poorer decisions regarding register usage resulting in local memory usage #if ENABLE_BETTER_ONEPAIRSQ X2qconjb(&a, &b); // X2(a, conjugate(b)). a.x range is 0..2+, a.y range is -1-..1+, b.x range is -1-..1+, b.y range is 0..2+ - a.y += 2*M61; // a range is 0..2+ / 1-..3+ - b.x += 2*M61; // b range is 1-..3+ / 0..2+ + a.y += 2*M61; // a range is 0..2+ / 1-..3+ + b.x += 2*M61; // b range is 1-..3+ / 0..2+ - ab = addq(a, b); // Compute 2ab as (a + b)^2 - a^2 - b^2. ab range is 1-..5+ + ab = addq(a, b); // Compute 2ab as (a + b)^2 - a^2 - b^2. ab range is 1-..5+ a2 = csqq(a, 3, 4); // a2 = a^2, a2 range is 0..2+ b2 = csq(b, 4, 3); // b2 = b^2, b2 range is 0..1+ @@ -978,13 +973,13 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { local GF61 lds[LDS_BYTES / sizeof(GF61)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); GF61 u[NH]; - u32 H = ND / SMALL_HEIGHT; // This kernel in executed in two workgroups. u32 which = get_group_id(0); @@ -1029,8 +1024,9 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES / sizeof(GF61)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -1038,15 +1034,8 @@ KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 u[NH], v[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line1 = get_group_id(0); + u32 line1 = get_line_number(base); u32 line2 = line1 ? H - line1 : (H / 2); -#else - u32 line1 = get_group_id(0) + 1; - u32 line2 = H - line1; -#endif u32 memline1 = transPos(line1, MIDDLE, WIDTH); u32 memline2 = transPos(line2, MIDDLE, WIDTH); @@ -1133,8 +1122,9 @@ void OVERLOAD pairSq2_special(GF61 *u, GF61 base_squared) { } } -KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF61 lds[2 * LDS_BYTES / sizeof(GF61)]; + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); @@ -1142,16 +1132,8 @@ KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, Trig smallTrig) { GF61 u[NH]; - u32 H = ND / SMALL_HEIGHT; - -#if SINGLE_KERNEL - u32 line_u = get_group_id(0); + u32 line_u = get_line_number(base); u32 line_v = line_u ? H - line_u : (H / 2); -#else - u32 line_u = get_group_id(0) + 1; - u32 line_v = H - line_u; -#endif - u32 me = get_local_id(0); u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). diff --git a/src/tune.cpp b/src/tune.cpp index dea4f864..17d16526 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -707,7 +707,8 @@ void Tune::tune() { args->flags["STORES"] = to_string(stores); } - // Find best FAST_BARRIER setting + // Find best FAST_BARRIER setting. This setting does nothing when using the CUDA backend. +#ifndef CUDA_BACKEND if (1 /*AMDGPU*/) { // FAST_BARRIER now works for nVidia GPUs too (from what I've seen) FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); @@ -726,6 +727,7 @@ void Tune::tune() { configsUpdate(current_cost, best_cost, 0.000, "FAST_BARRIER", best_fast_barrier, newConfigKeyVals, suggestedConfigKeyVals); args->flags["FAST_BARRIER"] = to_string(best_fast_barrier); } +#endif // Find best TAIL_KERNELS setting if (1) { @@ -1033,12 +1035,53 @@ void Tune::tune() { if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_wmul = wmul; } } log("Best WMUL is %u. Default WMUL is 2.\n", best_wmul); - configsUpdate(current_cost, best_cost, 0.003, "WMUL", best_wmul, newConfigKeyVals, suggestedConfigKeyVals); + configsUpdate(current_cost, best_cost, 0.000, "WMUL", best_wmul, newConfigKeyVals, suggestedConfigKeyVals); args->flags["WMUL"] = to_string(best_wmul); } + // Find best MULTI_Q setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_multi_q = 0; + u32 current_multi_q = args->value("MULTI_Q", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 multi_q : {0, 1}) { + args->flags["MULTI_Q"] = to_string(multi_q); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using MULTI_Q=%u is %6.1f\n", fft.spec().c_str(), multi_q, cost); + if (multi_q == current_multi_q) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_multi_q = multi_q; } + } + log("Best MULTI_Q is %u. Default MULTI_Q is 0.\n", best_multi_q); + configsUpdate(current_cost, best_cost, 0.000, "MULTI_Q", best_multi_q, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["MULTI_Q"] = to_string(best_multi_q); + } + // Find best CUDA compiler options #if CUDA_BACKEND + // Find best GRAPHS setting + if (1) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_graphs = 0; + u32 current_graphs = args->value("GRAPHS", 1); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 graphs : {0, 1}) { + args->flags["GRAPHS"] = to_string(graphs); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using GRAPHS=%u is %6.1f\n", fft.spec().c_str(), graphs, cost); + if (graphs == current_graphs) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_graphs = graphs; } + } + log("Best GRAPHS is %u. Default GRAPHS is 1.\n", best_graphs); + configsUpdate(current_cost, best_cost, 0.000, "GRAPHS", best_graphs, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["GRAPHS"] = to_string(best_graphs); + } + + // See if disabling our default register usage makes sense if (0) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); From 8de0c2194fb7585aa8d1c1c287fc0cfcd320b8a1 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 20 Jul 2026 18:15:45 +0000 Subject: [PATCH 089/214] Fixed issues from merging tdulcet's Clang fixes. --- src/Gpu.cpp | 10 +++++----- src/clwrap.cpp | 2 +- src/cuda/clwrap_cuda.cpp | 6 +++--- src/log.cpp | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index a1c78c27..dd4e11b0 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -1636,7 +1636,7 @@ for (u32 i = 0; i < size; ++i) { r.emplace_back(timeBufVect, &queue, N); } pair Gpu::readROE() { assert(roePos <= ROE_SIZE); if (roePos) { - vector const roe = bufROE.read(roePos + 2); + vector roe = bufROE.read(roePos + 2); assert(roe.size() == roePos + 2); // Split the roe buffer into two. One for squarings and one for multiplications. This is likely overkill as the multiplication ROE is not used - though // it could be useful for debugging (in which case we could support getting roe for squarings or multipplications, but not both). @@ -1651,15 +1651,15 @@ pair Gpu::readROE() { roePos = 0; mulRoePos.clear(); return {roeStat(squareRoe), roeStat(mulRoe)}; - } + } else { return {}; - + } } RoeInfo Gpu::readCarryStats() { assert(carryPos <= CARRY_SIZE); if (carryPos == 0) { return {}; } - vector const carry = bufStatsCarry.read(carryPos + 2); + vector carry = bufStatsCarry.read(carryPos + 2); assert(carry.size() == carryPos + 2); // Delete first two used to calculate carryPos on the GPU. carry[0] = carry[carryPos]; @@ -2131,7 +2131,7 @@ bool Gpu::equals9(const Words& a) { return true; } -static int ulps(double a, double b) { +[[maybe_unused]] static int ulps(double a, double b) { if (a == 0 && b == 0) { return 0; } u64 const aa = as(a); diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 8b6a506e..aab635c5 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -448,7 +448,7 @@ cl_context getQueueContext(cl_command_queue q) { return ret; } -static cl_device_id getQueueDevice(cl_command_queue q) { +[[maybe_unused]] static cl_device_id getQueueDevice(cl_command_queue q) { cl_device_id id; CHECK1(clGetCommandQueueInfo(q, CL_QUEUE_DEVICE, sizeof(id), &id, nullptr)); return id; diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 2bc53626..ceb91038 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -707,7 +707,7 @@ cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id // ---- Enqueue operations ---- -int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned /*workDim*/, +int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, const size_t* /*globalOffset*/, const size_t* globalSize, const size_t* localSize, unsigned /*nWaits*/, const cl_event* /*waits*/, cl_event* event) { @@ -751,7 +751,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned /*workDim* if (!pStart) { cuEventCreate(&pStart, CU_EVENT_DEFAULT); cuEventCreate(&pEnd, CU_EVENT_DEFAULT); } cuEventRecord(pStart, q->stream); - CUresult cosnt r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); + CUresult const r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); cuEventRecord(pEnd, q->stream); cuEventSynchronize(pEnd); float ms = 0; @@ -1127,7 +1127,7 @@ int clSetKernelArgSVMPointer(cl_kernel k, unsigned pos, const void* ptr) { // Computes the minimum address span covering all buffers, then sets one access policy // window with hitRatio sized so that only the actual buffer bytes get persisting treatment, // not the gaps between non-contiguous allocations. -static void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) { +[[maybe_unused]] static void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) { if (!q) return; // Find address span and total data size diff --git a/src/log.cpp b/src/log.cpp index 0f80be05..a5639251 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -20,7 +20,7 @@ void initLog(const char *logName) { logFile = File::openAppend(logName); } -static string longTimeStr() { return timeStr("%Y-%m-%d %H:%M:%S %Z"); } +[[maybe_unused]] static string longTimeStr() { return timeStr("%Y-%m-%d %H:%M:%S %Z"); } string shortTimeStr() { return timeStr("%Y%m%d %H:%M:%S"); } static char logBuf[32 * 1024]; From a5c96af835174d2bdd185efcd12bf9e584445d10 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 20 Jul 2026 18:55:55 +0000 Subject: [PATCH 090/214] Fixed issue with CUDA graps on CUDA 11 (cuGraphInstantiate interface changes from CUDA 11 to 12) --- src/cuda/clwrap_cuda.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index ceb91038..f8f35fa2 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -1202,7 +1202,11 @@ int clGraphEndRecording(cl_command_queue q, cl_graph* graph) { auto* g = new _cl_graph; g->queue = q; CUresult r = cuStreamEndCapture(q->stream, &g->graph); +#if CUDA_VERSION >= 12000 if (r == CUDA_SUCCESS) r = cuGraphInstantiate(&g->graphExec, g->graph, 0); +#else + if (r == CUDA_SUCCESS) r = cuGraphInstantiateWithFlags(&g->graphExec, g->graph, 0); +#endif *graph = g; return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; } From 1bf3bd630f3825e6cd1c42acc2a8ee5f026b2e2c Mon Sep 17 00:00:00 2001 From: george Date: Tue, 21 Jul 2026 00:04:46 +0000 Subject: [PATCH 091/214] Fixed a lot of Windows MSVC compiler warning messages. Lots of size_t/u32 and automatic conversion from larger to smaller data type issues. Not all warnings fixed --- src/Buffer.h | 2 +- src/FFTConfig.h | 2 +- src/File.cpp | 2 +- src/File.h | 26 +++++++++++++------------- src/Gpu.cpp | 34 +++++++++++++++++----------------- src/Gpu.h | 4 ++-- src/Hash.h | 10 +++++----- src/KernelCompiler.cpp | 6 +++--- src/Proof.cpp | 28 ++++++++++++++-------------- src/Queue.cpp | 8 ++++---- src/Queue.h | 10 +++++----- src/TrigBufCache.cpp | 10 +++++----- src/TuneEntry.cpp | 2 +- src/clwrap.cpp | 18 +++++++++--------- src/cuda/clwrap_cuda.cpp | 14 +++++++------- src/shared.h | 5 +++-- src/state.cpp | 4 +--- src/state.h | 2 +- src/tinycl.h | 2 +- src/tune.cpp | 36 ++++++++++++++++++------------------ src/typeName.h | 2 +- 21 files changed, 113 insertions(+), 114 deletions(-) diff --git a/src/Buffer.h b/src/Buffer.h index 8e76dca1..b7796261 100644 --- a/src/Buffer.h +++ b/src/Buffer.h @@ -35,7 +35,7 @@ class Buffer { , tInfo{tInfo} {} - void fill(T value, u32 sizeOrFull = 0) { + void fill(T value, size_t sizeOrFull = 0) { assert(sizeOrFull <= size); auto fillSize = sizeOrFull ? sizeOrFull : size; queue->fillBuf(get(), value, fillSize * sizeof(T), tInfo); diff --git a/src/FFTConfig.h b/src/FFTConfig.h index 5bc719e7..3863ec19 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -92,7 +92,7 @@ struct FFTConfig { FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry); [[nodiscard]] std::string spec() const; - [[nodiscard]] u64 size() const { return shape.size(); } + [[nodiscard]] u32 size() const { return shape.size(); } [[nodiscard]] u64 maxExp() const { return u64(maxBpw() * shape.size()); } [[nodiscard]] float minBpw() const { return shape.minBpw(); } diff --git a/src/File.cpp b/src/File.cpp index 56d258b1..fd04256e 100644 --- a/src/File.cpp +++ b/src/File.cpp @@ -36,7 +36,7 @@ File::~File() { f = nullptr; } -i64 File::size(const fs::path &name) { +u64 File::size(const fs::path &name) { error_code dummy; return filesystem::file_size(name, dummy); } diff --git a/src/File.h b/src/File.h index fdc3e3ac..03c7a86e 100644 --- a/src/File.h +++ b/src/File.h @@ -8,7 +8,7 @@ #include #include #include -#ifndef _MSC_VER // unistd.h does not exist for MSVC +#ifndef _MSC_VER // unistd.h does not exist for MSVC #include #endif #include @@ -60,9 +60,9 @@ class File { File(const fs::path &path, const string& mode, bool throwOnError); - bool readNoThrow(void* data, u32 nBytes) const { return fread(data, nBytes, 1, this->get()); } + bool readNoThrow(void* data, size_t nBytes) const { return fread(data, nBytes, 1, this->get()); } - void read(void* data, u32 nBytes) const { + void read(void* data, size_t nBytes) const { if (!readNoThrow(data, nBytes)) { throw ReadError{name}; } } @@ -89,7 +89,7 @@ class File { public: const std::string name; - static i64 size(const fs::path& name); + static u64 size(const fs::path& name); static File openRead(const fs::path& name) { return File{name, "rb", false}; } static File openReadThrow(const fs::path& name) { return File{name, "rb", true}; } @@ -142,7 +142,7 @@ class File { template void write(const T& x) const { write(&x, sizeof(T)); } - void write(const void* data, u32 nBytes) const { + void write(const void* data, size_t nBytes) const { if (!fwrite(data, nBytes, 1, this->get())) { throw WriteError{name}; } } @@ -177,7 +177,7 @@ class File { void write(const string& s) { write(string_view(s)); } void write(const char* s) { write(string_view(s)); } - void write(string_view s) { write(s.data(), u32(s.size())); } + void write(string_view s) { write(s.data(), s.size()); } operator bool() const { return f != nullptr; } [[nodiscard]] FILE* get() const { return f; } @@ -223,7 +223,7 @@ class File { } template - [[nodiscard]] [[nodiscard]] std::vector read(u32 nWords) const { + [[nodiscard]] std::vector read(size_t nWords) const { vector ret; ret.resize(nWords); read(ret.data(), nWords * sizeof(T)); @@ -231,7 +231,7 @@ class File { } template - [[nodiscard]] std::vector readChecked(u32 nWords) const { + [[nodiscard]] std::vector readChecked(size_t nWords) const { u32 const expectedCRC = read(1)[0]; return readWithCRC(nWords, expectedCRC); } @@ -243,7 +243,7 @@ class File { } template - [[nodiscard]] std::vector readWithCRC(u32 nWords, u32 crc) const { + [[nodiscard]] std::vector readWithCRC(size_t nWords, u32 crc) const { auto data = read(nWords); if (crc != crc32(data)) { log("File '%s' : CRC: expected %u, actual %u\n", name.c_str(), crc, crc32(data)); @@ -252,18 +252,18 @@ class File { return data; } - std::vector readBytesLE(u32 nBytes) { + std::vector readBytesLE(size_t nBytes) { assert(nBytes > 0); - u32 const nWords = (nBytes - 1) / 4 + 1; + size_t const nWords = (nBytes - 1) / 4 + 1; vector data(nWords); read(data.data(), nBytes); return data; } - u32 readUpTo(void* data, u32 nUpToBytes) { return u32(fread(data, 1, nUpToBytes, this->get())); } + size_t readUpTo(void* data, size_t nUpToBytes) { return fread(data, 1, nUpToBytes, this->get()); } string readAll() { - u32 const sz = u32(size()); + u64 const sz = size(); return {read(sz).data(), sz}; } }; diff --git a/src/Gpu.cpp b/src/Gpu.cpp index dd4e11b0..5c01d6e6 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -73,19 +73,19 @@ double invWeightM1(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { double boundUnderOne(double x) { return std::min(x, nexttoward(1, 0)); } float weight32(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N); + return float(exp2((double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } float invWeight32(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return exp2(-(double)(extra(N, E, kAt(H, line, col) + rep)) / N); + return float(exp2(-(double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } float weightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return expm1(M_LN2 * (double)(extra(N, E, kAt(H, line, col) + rep)) / N); + return float(expm1(M_LN2 * (double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } float invWeightM132(u32 N, u64 E, u32 H, u32 line, u32 col, u32 rep) { - return expm1(M_LN2 * - (double)(extra(N, E, kAt(H, line, col) + rep)) / N); + return float(expm1(M_LN2 * - (double)(extra(N, E, kAt(H, line, col) + rep)) / N)); } Weights genWeights(FFTConfig fft, u64 E, u32 W, u32 H, u32 nW, bool nvidiaGpu) { @@ -353,8 +353,8 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector>{ - {"EXP", E}, {"WIDTH", fft.shape.width}, {"SMALL_HEIGHT", fft.shape.height}, {"MIDDLE", fft.shape.middle}, @@ -498,7 +498,7 @@ RoeInfo roeStat(const vector& roe) { sumRoe += x; sum2Roe += x * x; } - u32 const n = roe.size(); + u32 const n = u32(roe.size()); double const sdRoe = sqrt(n * sum2Roe - sumRoe * sumRoe) / n; double const meanRoe = sumRoe / n; @@ -514,7 +514,7 @@ class IterationTimer { explicit IterationTimer(u64 kStart) : kStart(kStart) { } float reset(u64 k) { - float const secs = timer.reset(); + float const secs = float(timer.reset()); u64 const its = max(u64(1), k - kStart); kStart = k; @@ -1642,7 +1642,7 @@ pair Gpu::readROE() { // it could be useful for debugging (in which case we could support getting roe for squarings or multipplications, but not both). auto [squareRoe, mulRoe] = split(roe, mulRoePos); // Delete first two used to calculate roePos on the GPU. Do this after splitting the vector (mulRoePos recorded indices in "+ 2" format). - u32 squareRoeSize = squareRoe.size() - 2; + u32 squareRoeSize = u32(squareRoe.size()) - 2; roe[0] = squareRoe[squareRoeSize]; roe[1] = squareRoe[squareRoeSize+1]; squareRoe.resize(squareRoeSize); @@ -2018,7 +2018,7 @@ void Gpu::square(Buffer& out, Buffer& in, enum LEAD_TYPE leadIn, enu } } -u32 Gpu::squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3) { +u64 Gpu::squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3) { assert(from < to); enum LEAD_TYPE leadIn = LEAD_NONE; for (u64 k = from; k < to; ++k) { @@ -2076,7 +2076,7 @@ static string formatETA(u32 secs) { return string(buf); } -static string getETA(u32 step, u32 total, float secsPerStep) { +static string getETA(u64 step, u64 total, float secsPerStep) { u32 const etaSecs = max(0u, u32((total - step) * secsPerStep)); return formatETA(etaSecs); } @@ -2119,7 +2119,7 @@ void Gpu::doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u3 RoeInfo const carryStats = readCarryStats(); if (carryStats.N > 2) { - u32 const m = ldexp(carryStats.max, 32); + u32 const m = u32(ldexp(carryStats.max, 32)); double const z = carryStats.z(); log("Carry: %x Z(%u)=%.1f\n", m, carryStats.N, z); } @@ -2137,7 +2137,7 @@ bool Gpu::equals9(const Words& a) { u64 const aa = as(a); u64 const bb = as(b); bool const sameSign = (aa >> 63) == (bb >> 63); - int const delta = sameSign ? bb - aa : bb + aa; + int const delta = int(sameSign ? bb - aa : bb + aa); return delta; } @@ -2196,7 +2196,7 @@ void Gpu::selftestTrig() { [[maybe_unused]] i64 const prev = 0; u64 min = -1; u64 sum = 0; - for (long const x : times) { + for (i64 const x : times) { #if 0 if (x != prev) { @@ -2227,12 +2227,12 @@ static void doDiv3(u64 E, Words& words) { assert(topBits > 0 && topBits < 32); { u64 const w = (u64(r) << topBits) + words.back(); - words.back() = w / 3; + words.back() = u32(w / 3); r = w % 3; } for (auto it = words.rbegin() + 1, end = words.rend(); it != end; ++it) { u64 const w = (u64(r) << 32) + *it; - *it = w / 3; + *it = u32(w / 3); r = w % 3; } } @@ -2645,7 +2645,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { log(" %9" PRIu64 " %016" PRIx64 " %s\n", k, res, formatSecsPerIter(secsPerIt).c_str()); RoeInfo const carryStats = readCarryStats(); if (carryStats.N) { - u32 const m = ldexp(carryStats.max, 32); + u32 const m = u32(ldexp(carryStats.max, 32)); double const z = carryStats.z(); log("Carry: %x Z(%u)=%.1f\n", m, carryStats.N, z); } @@ -2832,7 +2832,7 @@ array Gpu::isCERT(const Task& task) { if (k >= kEnd) { fs::remove (fname); - return std::move(SHA3{}.update(data.data(), (E-1)/8+1)).finish(); + return std::move(SHA3{}.update(data.data(), u32((E-1)/8+1))).finish(); } if (doStop) { throw "stop requested"; } diff --git a/src/Gpu.h b/src/Gpu.h index 107732c0..ba350c71 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -265,8 +265,8 @@ class Gpu { void squareCERT(Buffer& io, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut) { square(io, io, leadIn, leadOut, false, false); } void squareLL(Buffer& io, enum LEAD_TYPE leadIn, enum LEAD_TYPE leadOut) { square(io, io, leadIn, leadOut, false, true); } - u32 squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3); - u32 squareLoop(Buffer& io, u64 from, u64 to) { return squareLoop(io, io, from, to, false); } + u64 squareLoop(Buffer& out, Buffer& in, u64 from, u64 to, bool doTailMul3); + u64 squareLoop(Buffer& io, u64 from, u64 to) { return squareLoop(io, io, from, to, false); } bool isEqual(Buffer& bufCheck, Buffer& bufAux); u64 bufResidue(Buffer& buf); diff --git a/src/Hash.h b/src/Hash.h index 5144db17..87a9aa25 100644 --- a/src/Hash.h +++ b/src/Hash.h @@ -20,15 +20,15 @@ class Hash { Hash& update(const void* data, u32 size) { h.update(data, size); return *this; } template - Hash&& update(const array& v) && { h.update(v.data(), N * sizeof(T)); return std::move(*this); } + Hash&& update(const array& v) && { h.update(v.data(), u32(N * sizeof(T))); return std::move(*this); } - void update(u32 x) { h.update(&x, sizeof(x)); } - void update(u64 x) { h.update(&x, sizeof(x)); } + void update(u32 x) { h.update(&x, u32(sizeof(x))); } + void update(u64 x) { h.update(&x, u32(sizeof(x))); } template - void update(const vector& v) { h.update(v.data(), v.size() * sizeof(T)); } + void update(const vector& v) { h.update(v.data(), u32(v.size() * sizeof(T))); } - void update(const string& s) {h.update(s.c_str(), s.size()); } + void update(const string& s) {h.update(s.c_str(), u32(s.size())); } auto finish() && { return std::move(h).finish(); } }; diff --git a/src/KernelCompiler.cpp b/src/KernelCompiler.cpp index a5885227..e5d22520 100644 --- a/src/KernelCompiler.cpp +++ b/src/KernelCompiler.cpp @@ -45,7 +45,7 @@ KernelCompiler::KernelCompiler(const Args& args, const Context* context, const s auto& clNames = getClFileNames(); auto& clFiles = getClFiles(); assert(clNames.size() == clFiles.size()); - int const n = clNames.size(); + int const n = int(clNames.size()); for (int i = 0; i < n; ++i) { auto &src = clFiles[i]; files.emplace_back(clNames[i], src); @@ -68,12 +68,12 @@ Program KernelCompiler::compile(const string& fileName, const string& extraArgs) } #ifdef CUDA_BACKEND int err = clCompileProgram(p1.get(), 1, &deviceId, args.c_str(), - clSources.size(), (const cl_program*) (clSources.data()), getClFileNames().data(), + u32(clSources.size()), (const cl_program*) (clSources.data()), getClFileNames().data(), nullptr, nullptr); #else // Skip first file (opencl_compat.cuh) if this is a standard openCL application rather than a CUDA translation int err = clCompileProgram(p1.get(), 1, &deviceId, args.c_str(), - clSources.size()-1, (const cl_program*) (clSources.data()+1), getClFileNames().data()+1, + u32(clSources.size())-1, (const cl_program*) (clSources.data()+1), getClFileNames().data()+1, nullptr, nullptr); #endif if (string const mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } diff --git a/src/Proof.cpp b/src/Proof.cpp index 1d395954..e8b3b81a 100644 --- a/src/Proof.cpp +++ b/src/Proof.cpp @@ -21,19 +21,19 @@ namespace proof { array hashWords(u64 E, const Words& words) { - return std::move(SHA3{}.update(words.data(), (E-1)/8+1)).finish(); + return std::move(SHA3{}.update(words.data(), u32((E-1)/8+1))).finish(); } array hashWords(u64 E, array prefix, const Words& words) { - return std::move(SHA3{}.update(prefix).update(words.data(), (E-1)/8+1)).finish(); + return std::move(SHA3{}.update(prefix).update(words.data(), u32((E-1)/8+1))).finish(); } string fileHash(const fs::path& filePath) { File fi = File::openReadThrow(filePath); char buf[64 * 1024]; MD5 h; - u32 size = 0; - while ((size = fi.readUpTo(buf, sizeof(buf)))) { h.update(buf, size); } + size_t size = 0; + while ((size = fi.readUpTo(buf, sizeof(buf)))) { h.update(buf, u32(size)); } return std::move(h).finish(); } @@ -56,13 +56,13 @@ ProofInfo getInfo(const fs::path& proofFile) { fs::path Proof::file(const fs::path& proofDir) const { string const strE = to_string(E); - u32 const power = middles.size(); + u32 const power = u32(middles.size()); return proofDir / (strE + '-' + to_string(power) + ".proof"); } void Proof::save(const fs::path& proofFile) const { File const fo = File::openWrite(proofFile); - u32 const power = middles.size(); + u32 const power = u32(middles.size()); fo.printf(HEADER_v2, power, E, '\n'); fo.write(B.data(), (E-1)/8+1); for (const Words& w : middles) { fo.write(w.data(), (E-1)/8+1); } @@ -77,7 +77,7 @@ Proof Proof::load(const fs::path& path) { log("Proof file '%s' has invalid header\n", path.string().c_str()); throw "Invalid proof header"; } - u32 const nBytes = (E - 1) / 8 + 1; + u32 const nBytes = u32((E - 1) / 8 + 1); Words const B = fi.readBytesLE(nBytes); vector middles; middles.reserve(power); @@ -89,7 +89,7 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { // log("B %016" PRIx64 "\n", res64(B)); // for (u32 i = 0; i < middles.size(); ++i) { log("Middle[%u] %016" PRIx64 "\n", i, res64(middles[i])); } - u32 const power = middles.size(); + u32 const power = u32(middles.size()); assert(power > 0); bool const isPrime = (B == makeWords(E, 9)); @@ -118,7 +118,7 @@ bool Proof::verify(Gpu *gpu, const vector& hashes) const { } log("proof verification: doing %" PRIu64 " iterations\n", span); - A = gpu->expExp2(A, span); + A = gpu->expExp2(A, u32(span)); bool const ok = (A == B); if (ok) { @@ -149,18 +149,18 @@ ProofSet::ProofSet(u64 E, u32 power) u32 p; u64 span; for (p = 0, span = (E + 1) / 2; p < power; ++p, span = (span + 1) / 2) { - for (u32 i = 0, end = points.size(); i < end; ++i) { + for (u32 i = 0, end = u32(points.size()); i < end; ++i) { points.push_back(points[i] + span); } } - assert(points.size() == (1u << power)); + assert(u32(points.size()) == (1u << power)); assert(points.front() == 0); points.front() = E; std::ranges::sort(points); - assert(points.size() == (1u << power)); + assert(u32(points.size()) == (1u << power)); assert(points.back() == E); points.push_back(u32(-1)); // guard element @@ -200,7 +200,7 @@ u32 ProofSet::bestPower(u64 E) { assert(E > 0); // log2(x)/2 is log4(x) - int const power = 10 + floor(log2(E / 60e6) / 2); + int const power = int(10 + floor(log2(double(E) / 60e6) / 2)); assert(power >= 2); return power; } @@ -222,7 +222,7 @@ u32 ProofSet::effectivePower(u64 E, u32 power, u64 currentK) { } bool ProofSet::fileExists(u64 k) const { - return File::size(proofPath(E) / to_string(k)) == i64(E / 32 + 2) * 4; + return File::size(proofPath(E) / to_string(k)) == (E / 32 + 2) * 4; } bool ProofSet::isValidTo(u64 limitK) const { diff --git a/src/Queue.cpp b/src/Queue.cpp index 56338157..1fbd79d6 100644 --- a/src/Queue.cpp +++ b/src/Queue.cpp @@ -39,7 +39,7 @@ void Queue::writeTE(cl_mem buf, u64 size, const void* data, TimeInfo* tInfo) { events.synced(); } -void Queue::fillBufTE(cl_mem buf, u32 patSize, const void* pattern, u64 size, TimeInfo* tInfo) { +void Queue::fillBufTE(cl_mem buf, size_t patSize, const void* pattern, size_t size, TimeInfo* tInfo) { add(::fillBuf(get(), {}, buf, pattern, patSize, size, hasEvents), tInfo); } @@ -61,16 +61,16 @@ void Queue::add(EventHolder&& e, TimeInfo* ti) { if (queueCount >= MAX_QUEUE_COUNT) queueMarkerEvent(); } -void Queue::readSync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo) { +void Queue::readSync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo) { add(read(get(), {}, true, buf, size, out, hasEvents), tInfo); events.synced(); } -void Queue::readAsync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo) { +void Queue::readAsync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo) { add(read(get(), {}, false, buf, size, out, hasEvents), tInfo); } -void Queue::copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo) { +void Queue::copyBuf(cl_mem src, cl_mem dst, size_t size, TimeInfo* tInfo) { add(::copyBuf(get(), {}, src, dst, size, hasEvents), tInfo); } diff --git a/src/Queue.h b/src/Queue.h index 603a6c3c..65643251 100644 --- a/src/Queue.h +++ b/src/Queue.h @@ -26,7 +26,7 @@ class Queue : public QueueHolder { bool isAuxQueue; void writeTE(cl_mem buf, u64 size, const void* data, TimeInfo *tInfo); - void fillBufTE(cl_mem buf, u32 patSize, const void* pattern, u64 size, TimeInfo* tInfo); + void fillBufTE(cl_mem buf, size_t patSize, const void* pattern, size_t size, TimeInfo* tInfo); void flush(); void print(); void add(EventHolder &&e, TimeInfo* ti); @@ -43,12 +43,12 @@ class Queue : public QueueHolder { void write(cl_mem buf, const vector& v, TimeInfo* tInfo) { writeTE(buf, v.size() * sizeof(T), v.data(), tInfo); } template - void fillBuf(cl_mem buf, T pattern, u32 size, TimeInfo* tInfo) { fillBufTE(buf, sizeof(T), &pattern, size, tInfo); } + void fillBuf(cl_mem buf, T pattern, size_t size, TimeInfo* tInfo) { fillBufTE(buf, sizeof(T), &pattern, size, tInfo); } void run(cl_kernel kernel, size_t groupSizeX, size_t workSizeX, size_t workSizeY, TimeInfo* tInfo); - void readSync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo); - void readAsync(cl_mem buf, u32 size, void* out, TimeInfo* tInfo); - void copyBuf(cl_mem src, cl_mem dst, u32 size, TimeInfo* tInfo); + void readSync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo); + void readAsync(cl_mem buf, size_t size, void* out, TimeInfo* tInfo); + void copyBuf(cl_mem src, cl_mem dst, size_t size, TimeInfo* tInfo); void finish(); EventHolder createSyncEvent() { if (!isAuxQueue && !graphRecording) queueCount++; return enqueueMarker(get()); } // Enqueue a synchronization event. Used to sync work among multiple queues. diff --git a/src/TrigBufCache.cpp b/src/TrigBufCache.cpp index 933e3116..d46d6082 100644 --- a/src/TrigBufCache.cpp +++ b/src/TrigBufCache.cpp @@ -303,9 +303,9 @@ static float trigError(float c, float s) { return abs(trigNorm(c, s) - 1.0f); } // Round trig double to float as to satisfy c^2 + s^2 == 1 as best as possible static float2 roundTrig(double lc, double ls) { - float const c1 = lc; + float const c1 = float(lc); float const c2 = nexttoward(c1, lc); - float const s1 = ls; + float const s1 = float(ls); float const s2 = nexttoward(s1, ls); float c = c1; @@ -858,7 +858,7 @@ static vector genMiddleTrigGF61(u32 smallH, u32 middle, u32 width) { static vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { vector tab; - u32 tabsize; + size_t tabsize; if (fft.FFT_FP64) { tab = genSmallTrigFP64(size, radix); @@ -897,7 +897,7 @@ static vector genSmallTrig(FFTConfig fft, u32 size, u32 radix) { static vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u32 middle, u32 size, u32 radix, bool tail_single_wide) { vector tab; - u32 tabsize; + size_t tabsize; if (fft.FFT_FP64) { tab = genSmallTrigComboFP64(args, width, middle, size, radix, tail_single_wide); @@ -936,7 +936,7 @@ static vector genSmallTrigCombo(Args *args, FFTConfig fft, u32 width, u static vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 width) { vector tab; - u32 tabsize; + size_t tabsize; if (fft.FFT_FP64) { tab = genMiddleTrigFP64(smallH, middle, width); diff --git a/src/TuneEntry.cpp b/src/TuneEntry.cpp index e1a0900b..1ad420ea 100644 --- a/src/TuneEntry.cpp +++ b/src/TuneEntry.cpp @@ -10,7 +10,7 @@ bool TuneEntry::update(vector& results) const { u64 const maxExp = fft.maxExp(); [[maybe_unused]] bool didErase = false; - int i{}; + size_t i{}; for (i = results.size() - 1; i >= 0 && results[i].cost > cost; --i) { if (results[i].fft.maxExp() <= maxExp) { results.erase(std::next(results.begin(), i)); diff --git a/src/clwrap.cpp b/src/clwrap.cpp index aab635c5..3305f6c2 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -144,7 +144,7 @@ float getGpuRamGB(cl_device_id id) { try { u64 totSize = 0; GET_INFO(id, CL_DEVICE_GLOBAL_MEM_SIZE, totSize); - return ldexp(totSize, -30); // to GB + return float(ldexp(totSize, -30)); // to GB } catch (const gpu_error& err) { } return 0; @@ -333,7 +333,7 @@ EventHolder run(cl_queue queue, cl_kernel kernel, size_t workSizes[2] = {workSizeX, workSizeY}; size_t groupSizes[2] = {groupSizeX, 1}; CHECK2(clEnqueueNDRangeKernel(queue, kernel, workSizeY == 1 ? 1 : 2, nullptr, workSizes, groupSizes, - waits.size(), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr), + u32(waits.size()), waits.empty() ? 0 : waits.data(), genEvent ? &event : nullptr), name.c_str()); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -343,7 +343,7 @@ EventHolder read(cl_queue queue, vector&& waits, size_t const start = 0; cl_event event{}; CHECK1(clEnqueueReadBuffer(queue, buf, blocking, start, size, data, - waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -352,7 +352,7 @@ EventHolder write(cl_queue queue, vector&& waits, size_t const start = 0; cl_event event{}; CHECK1(clEnqueueWriteBuffer(queue, buf, blocking, start, size, data, - waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -360,7 +360,7 @@ EventHolder copyBuf(cl_queue queue, vector&& waits, const cl_mem src, cl_mem dst, size_t size, bool genEvent) { cl_event event{}; CHECK1(clEnqueueCopyBuffer(queue, src, dst, 0, 0, size, - waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -369,7 +369,7 @@ EventHolder fillBuf(cl_queue q, vector&& waits, assert(size); cl_event event{}; CHECK1(clEnqueueFillBuffer(q, buf, pat, patSize, 0 /*start*/, size, - waits.size(), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); + u32(waits.size()), waits.empty() ? nullptr : waits.data(), genEvent ? &event : nullptr)); return genEvent ? EventHolder{event} : EventHolder{}; } @@ -381,13 +381,13 @@ EventHolder enqueueMarker(cl_queue q) { EventHolder enqueueMarkerWithWaits(cl_queue q, vector&& waits) { cl_event event{}; - CHECK1(clEnqueueMarkerWithWaitList(q, waits.size(), waits.empty() ? nullptr : waits.data(), &event)); + CHECK1(clEnqueueMarkerWithWaitList(q, u32(waits.size()), waits.empty() ? nullptr : waits.data(), &event)); return EventHolder{event}; } void waitForEvents(vector&& waits) { if (!waits.empty()) { - CHECK1(clWaitForEvents(waits.size(), waits.data())); + CHECK1(clWaitForEvents(u32(waits.size()), waits.data())); } } @@ -400,7 +400,7 @@ int getKernelNumArgs(cl_kernel k) { int getWorkGroupSize(cl_kernel k, cl_device_id device, const char *name) { size_t size[3]; CHECK2(clGetKernelWorkGroupInfo(k, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, sizeof(size), &size, nullptr), name); - return size[0]; + return int(size[0]); } std::string getKernelArgName(cl_kernel k, int pos) { diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index f8f35fa2..9d0f6f23 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -112,7 +112,7 @@ unsigned clGetPlatformIDs(unsigned num, cl_platform_id* platforms, unsigned* num int clGetDeviceIDs(cl_platform_id, cl_device_type, unsigned num, cl_device_id* devices, unsigned* numRet) { enumerateDevices(); - unsigned const n = g_devices.size(); + unsigned const n = u32(g_devices.size()); if (numRet) *numRet = n; if (devices) { for (unsigned i = 0; i < min(num, n); i++) { @@ -714,12 +714,12 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, if (!q || !k) return CL_INVALID_VALUE; ensureContextCurrent(); - size_t const gsX = globalSize[0]; - size_t const lsX = localSize ? localSize[0] : 256; - size_t const numBlocksX = (gsX + lsX - 1) / lsX; - size_t const gsY = (workDim > 1) ? globalSize[1] : 1; - size_t const lsY = (workDim > 1) ? localSize[1] : 1; - size_t const numBlocksY = (gsY + lsY - 1) / lsY; + unsigned int const gsX = u32(globalSize[0]); + unsigned int const lsX = u32(localSize ? localSize[0] : 256); + unsigned int const numBlocksX = (gsX + lsX - 1) / lsX; + unsigned int const gsY = u32((workDim > 1) ? globalSize[1] : 1); + unsigned int const lsY = u32((workDim > 1) ? localSize[1] : 1); + unsigned int const numBlocksY = (gsY + lsY - 1) / lsY; // Build args array void* argPtrs[_cl_kernel::MAX_ARGS]; diff --git a/src/shared.h b/src/shared.h index 33ff1113..92899b23 100644 --- a/src/shared.h +++ b/src/shared.h @@ -1,4 +1,5 @@ // included from both C++ and OpenCL. -inline u32 bitposToWord(u64 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } -inline u32 wordToBitpos(u64 E, u32 N, u32 word) { return (word * ( E) + (N - 1)) / N; } +// NO LONGER USED -- and if it were would need recoding for 64-bit exponents +//inline u32 bitposToWord(u64 E, u32 N, u32 offset) { return offset * ((u64) N) / E; } +//inline u32 wordToBitpos(u64 E, u32 N, u32 word) { return (word * (E) + (N - 1)) / N; } diff --git a/src/state.cpp b/src/state.cpp index 8ac49307..637aa1cf 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -1,8 +1,6 @@ // Copyright 2017 Mihai Preda. #include "state.h" -#include "shared.h" - #include static i64 lowBits(i64 u, int bits) { return (u << (64 - bits)) >> (64 - bits); } @@ -10,7 +8,7 @@ static i64 lowBits(i64 u, int bits) { return (u << (64 - bits)) >> (64 - bits); std::vector compactBits(const vector &dataVect, u64 E) { if (dataVect.empty()) { return {}; } // Indicating all zero - u32 const N = dataVect.size(); + u32 const N = u32(dataVect.size()); const Word *data = dataVect.data(); std::vector out; diff --git a/src/state.h b/src/state.h index 02fab3fb..7254e05c 100644 --- a/src/state.h +++ b/src/state.h @@ -14,4 +14,4 @@ vector expandBits(const vector &compactBits, u32 N, u64 E); constexpr u32 step(u32 N, u64 E) { return N - (E % N); } constexpr u32 extra(u32 N, u64 E, u32 k) { return u64(step(N, E)) * k % N; } constexpr bool isBigWord(u32 N, u64 E, u32 k) { return extra(N, E, k) + step(N, E) < N; } -constexpr u32 bitlen(u32 N, u64 E, u32 k) { return E / N + isBigWord(N, E, k); } +constexpr u32 bitlen(u32 N, u64 E, u32 k) { return u32(E / N) + isBigWord(N, E, k); } diff --git a/src/tinycl.h b/src/tinycl.h index e4ee49b2..bcca2634 100644 --- a/src/tinycl.h +++ b/src/tinycl.h @@ -326,7 +326,7 @@ typedef union // native openCL builds, these extension routines basicly do nothing. struct _cl_graph {}; -typedef _cl_graph* cl_graph; +using cl_graph = struct _cl_graph *; bool clIsGraphSupported(cl_device_id); int clGraphBeginRecording(cl_command_queue); int clGraphEndRecording(cl_command_queue, cl_graph*); diff --git a/src/tune.cpp b/src/tune.cpp index 0231f2e7..37144d71 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -41,7 +41,7 @@ namespace { vector permute(const vector>>& params) { vector configs; - int const n = params.size(); + int const n = int(params.size()); vector vpos(n); while (true) { TuneConfig config; @@ -115,7 +115,7 @@ float Tune::maxBpw(FFTConfig fft) { // This doesn't need to be a very accurate estimate. // This estimate comes from analyzing a 4M FFT and a 7.5M FFT. // The 4M FFT needed a .015 step, the 7.5M FFT needed a .012 step. - float bpw_step = .015 + (log2(fft.size()) - log2(4.0*1024*1024)) / (log2(7.5*1024*1024) - log2(4.0*1024*1024)) * (.012 - .015); + float bpw_step = float(.015 + (log2(fft.size()) - log2(4.0*1024*1024)) / (log2(7.5*1024*1024) - log2(4.0*1024*1024)) * (.012 - .015)); // Pick a bpw that might be close to Z=34, it is best to err on the high side of Z=34 float bpw1 = fft.maxBpw() - 9 * bpw_step; // Old bpw gave Z=28, we want Z=34 (or more) @@ -173,11 +173,11 @@ printf ("Reguess bpw for %s is %.2f first Z22 is %.2f\n", fft.spec().c_str(), bp } float Tune::zForBpw(float bpw, FFTConfig fft, u32 count) { - u64 exponent = (count == 1) ? primes.prevPrime(fft.size() * bpw) : primes.nextPrime(fft.size() * bpw); + u64 exponent = (count == 1) ? primes.prevPrime(u64(fft.size() * bpw)) : primes.nextPrime(u64(fft.size() * bpw)); float total_z = 0.0f; for (u32 i = 0; i < count; i++, exponent = primes.nextPrime (exponent + 1)) { auto [ok, res, roeSq, roeMul] = Gpu::make(exponent, shared, fft, {}, false)->measureROE(true); - float const z = roeSq.z(); + float const z = float(roeSq.z()); total_z += z; log("Zforbpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); if (!ok) { log("Error at bpw %.2f (z %.2f) : %s\n", bpw, z, fft.spec().c_str()); continue; } @@ -249,15 +249,15 @@ void Tune::carryTune() { double m = 0; const float mid = fft.shape.carry32BPW(); for (float const bpw : {mid - 0.05f, mid + 0.05f}) { - u64 const exponent = primes.nearestPrime(fft.size() * bpw); + u64 const exponent = primes.nearestPrime(u64(fft.size() * bpw)); auto [ok, carry] = Gpu::make(exponent, shared, fft, {}, false)->measureCarry(); m = carry.max; if (!ok) { log("Error %s at %f\n", fft.spec().c_str(), bpw); } - zv.push_back(carry.z()); + zv.push_back(float(carry.z())); } float const avg = (zv[0] + zv[1]) / 2; - u64 const exponent = fft.shape.carry32BPW() * fft.size(); + u64 const exponent = u64(fft.shape.carry32BPW() * fft.size()); double const pErr100 = -expm1(-exp(-avg) * exponent * 100); log("%14s %.3f : %.3f (%.3f %.3f) %f %.0f%%\n", fft.spec().c_str(), mid, avg, zv[0], zv[1], m, pErr100 * 100); fo.printf("%f %f\n", log2(fft.size()), avg); @@ -367,7 +367,7 @@ void Tune::tune() { if (s == "inplace") time_inplace_only = true; auto keyVal = split(s, '='); if (keyVal.size() == 2) { - if (keyVal.front() == "quick") quick = stod(keyVal.back()); + if (keyVal.front() == "quick") quick = stoi(keyVal.back()); if (keyVal.front() == "minexp") min_exponent = stoull(keyVal.back()); if (keyVal.front() == "maxexp") max_exponent = stoull(keyVal.back()); } @@ -594,7 +594,7 @@ void Tune::tune() { u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_fft_load = 0; double best_cost = -1.0; - for (u32 const fft_load : {0, 1, 2, 3, 4}) { + for (u32 const fft_load : {0, 1, 2, 3, 4}) { if (fft_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10 * 10 + fft_load); double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); @@ -612,7 +612,7 @@ void Tune::tune() { u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_fft_store = 0; double best_cost = -1.0; - for (u32 const fft_store : {0, 1, 2, 3, 4}) { + for (u32 const fft_store : {0, 1, 2, 3, 4}) { if (fft_store >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["STORES"] = to_string(stores / 10 * 10 + fft_store); double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); @@ -630,10 +630,10 @@ void Tune::tune() { u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_cs_load = 0, best_cs_store = 0; double best_cost = -1.0; - for (u32 const cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load with L2 store + for (u32 const cs : {0, 1, 2}) { // Test three combinations: Default load/store, non-temporal, last-use load with L2 store if (cs >= 2 && (!NVIDIAGPU || NO_ASM)) continue; - u32 const cs_load = cs == 0 ? 0 : cs == 1 ? 1 : 4; - u32 const cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; + u32 const cs_load = cs == 0 ? 0 : cs == 1 ? 1 : 4; + u32 const cs_store = cs == 0 ? 0 : cs == 1 ? 1 : 2; args->flags["LOADS"] = to_string(loads / 100 * 100 + cs_load * 10 + loads % 10); args->flags["STORES"] = to_string(stores / 100 * 100 + cs_store * 10 + stores % 10); double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); @@ -653,7 +653,7 @@ void Tune::tune() { u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_trig_load = 0; double best_cost = -1.0; - for (u32 const trig_load : {0, 5}) { + for (u32 const trig_load : {0, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 1000 * 1000 + trig_load * 100 + loads % 100); double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); @@ -671,7 +671,7 @@ void Tune::tune() { u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_trig_load = 0; double best_cost = -1.0; - for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { + for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(loads / 10000 * 10000 + trig_load * 1000 + loads % 1000); double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); @@ -689,7 +689,7 @@ void Tune::tune() { u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_trig_load = 0; double best_cost = -1.0; - for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { + for (u32 const trig_load : {0, 1, 2, 3, 4, 5}) { if (trig_load >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["LOADS"] = to_string(trig_load * 10000 + loads % 10000); double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); @@ -798,7 +798,7 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u64 const exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 const exponent = primes.prevPrime(u64(fft.maxBpw() * 0.95 * fft.shape.size())); // Back off the maxExp as different settings will have different maxBpw u32 best_tail_trigs = 0; u32 const current_tail_trigs = args->value("TAIL_TRIGS32", 2); double best_cost = -1.0; @@ -881,7 +881,7 @@ void Tune::tune() { if (time_NTTs && time_FP32) { FFTConfig fft{defaultNTTShape, 202, CARRY_AUTO}; if (!fft.FFT_FP32) fft = FFTConfig(FFTShape(FFT3261, 512, 8, 512), 202, CARRY_AUTO); - u64 const exponent = primes.prevPrime(fft.maxBpw() * 0.95 * fft.shape.size()); // Back off the maxExp as different settings will have different maxBpw + u64 const exponent = primes.prevPrime(u64(fft.maxBpw() * 0.95 * fft.shape.size())); // Back off the maxExp as different settings will have different maxBpw u32 best_tabmul_chain = 0; u32 const current_tabmul_chain = args->value("TABMUL_CHAIN32", 0); double best_cost = -1.0; diff --git a/src/typeName.h b/src/typeName.h index 6ca21042..457d878b 100644 --- a/src/typeName.h +++ b/src/typeName.h @@ -9,7 +9,7 @@ const char* typeName(T&& v) { const char* ret = typeid(v).name(); try { size_t pos = 0; - std::stoi(ret, &pos); + (void)std::stoi(ret, &pos); return ret + pos; } catch (...) { return ret; From 47463badd732a4234ea9c6ed4df9ca94a739e714 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 21 Jul 2026 00:31:01 +0000 Subject: [PATCH 092/214] Removed CUDA debugging code accidently left in ROE calculations. --- src/cl/carryutil.cl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 64c83a11..8c614d2b 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -146,7 +146,6 @@ void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *b while (num_threads > 1) { // Write roundMax for high half of threads to local memory. Ignore threads not participating in the reduction. if (num_threads >= WAVEFRONT) bar(); - __asm("bar.sync 0;"); if (me >= num_threads / 2 && me < num_threads) lds[me - num_threads / 2] = u32RoundMax; if (num_threads > WAVEFRONT) { bar(); // work around a weird CUDA NVCC bug where two bar() calls are required???! (Titan V, CUDA 13.0, WMUL=2) From 0f03b5b07b2dc92bca19704ced4297315d6f75b5 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 21 Jul 2026 01:35:59 +0000 Subject: [PATCH 093/214] Successfully ran 2000 iterations of exponent 4.3B (more than 32 bits). --- src/Gpu.cpp | 2 +- src/Proof.cpp | 9 +++++---- src/Proof.h | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 5c01d6e6..35b8c1fb 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -353,7 +353,7 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector>{ {"WIDTH", fft.shape.width}, {"SMALL_HEIGHT", fft.shape.height}, diff --git a/src/Proof.cpp b/src/Proof.cpp index e8b3b81a..d10aa8dd 100644 --- a/src/Proof.cpp +++ b/src/Proof.cpp @@ -135,7 +135,7 @@ ProofSet::ProofSet(u64 E, u32 power) : E{E}, power{power} { assert(E & 1); // E is supposed to be prime - if (power <= 0 || power > 12) { + if (power <= 0 || power > 13) { log("Invalid proof power: %u\n", power); throw "Invalid proof power"; } @@ -163,7 +163,7 @@ ProofSet::ProofSet(u64 E, u32 power) assert(u32(points.size()) == (1u << power)); assert(points.back() == E); - points.push_back(u32(-1)); // guard element + points.push_back(u64(-1LL)); // guard element cacheIt = points.begin(); for ([[maybe_unused]] u64 const p : points) { @@ -188,7 +188,7 @@ bool ProofSet::isInPoints(u64 E, u32 power, u64 k) { } bool ProofSet::canDo(u64 E, u32 power, u64 currentK) { - assert(power > 0 && power <= 12); + assert(power > 0 && power <= 13); return ProofSet{E, power}.isValidTo(currentK); } @@ -200,7 +200,8 @@ u32 ProofSet::bestPower(u64 E) { assert(E > 0); // log2(x)/2 is log4(x) - int const power = int(10 + floor(log2(double(E) / 60e6) / 2)); + int power = int(10 + floor(log2(double(E) / 60e6) / 2)); + if (power > 13) power = 13; assert(power >= 2); return power; } diff --git a/src/Proof.h b/src/Proof.h index bac5d295..68fc1f4d 100644 --- a/src/Proof.h +++ b/src/Proof.h @@ -58,7 +58,7 @@ class ProofSet { u32 power; private: - vector points; + vector points; bool isValidTo(u64 limitK) const; From 84b5b9cefac7850ad0c5b17cdbea2f127af04909 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 22 Jul 2026 01:49:47 +0000 Subject: [PATCH 094/214] Allow tuning FFT6431 (FFT type 51). TitanV may find this useful. --- src/FFTConfig.cpp | 2 +- src/tune.cpp | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index 83603861..120bc287 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -83,7 +83,7 @@ vector FFTShape::multiSpec(const string& iniSpec) { vector FFTShape::allShapes(u32 sizeFrom, u32 sizeTo) { vector configs; - for (enum FFT_TYPES const type : {FFT64, FFT3161, FFT3261, FFT61, FFT323161}) { + for (enum FFT_TYPES const type : {FFT64, FFT6431, FFT3161, FFT3261, FFT61, FFT323161}) { for (u32 const width : {256, 512, 1024, 4096}) { for (u32 const height : {256, 512, 1024}) { if (width == 256 && height == 1024) { continue; } // Skip because we prefer width >= height diff --git a/src/tune.cpp b/src/tune.cpp index 37144d71..3789860f 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -351,8 +351,9 @@ void Tune::tune() { bool time_FFTs = false; bool time_NTTs = false; bool time_FP32 = true; + bool time_FFT6431 = false; bool time_inplace_only = NVIDIAGPU; // Default is nVidia is better off with INPLACE=1, AMD GPUs need to time extra options used when INPLACE=0 - int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) + int quick = 7; // Run config from slowest (quick=1) to fastest (quick=10) u64 min_exponent = 75000000; u64 max_exponent = 350000000; if (!args->fftSpec.empty()) { min_exponent = 0; max_exponent = 1000000000000ull; } @@ -363,7 +364,8 @@ void Tune::tune() { if (s == "noconfig") tune_config = false; if (s == "fp64") time_FFTs = true; if (s == "ntt") time_NTTs = true; - if (s == "nofp32") time_FP32 = false; + if (s == "fp6431") time_FFT6431 = true; // It is rare to have a GPU good at both FP64 and integer ops. TitanV is one. Allow tuning FFT6431. + if (s == "nofp32") time_FP32 = false; // Workaround bug in some openCL compilers that cannot compile our FP32 openCL code if (s == "inplace") time_inplace_only = true; auto keyVal = split(s, '='); if (keyVal.size() == 2) { @@ -1183,7 +1185,8 @@ skip_1K_256 = false; // Skip some FFTs and NTTs if (shape.fft_type == FFT64 && !time_FFTs) continue; - if (shape.fft_type != FFT64 && !time_NTTs) continue; + if (shape.fft_type == FFT6431 && !time_FFT6431) continue; + if (shape.fft_type != FFT64 && shape.fft_type != FFT6431 && !time_NTTs) continue; if ((shape.fft_type == FFT3261 || shape.fft_type == FFT323161 || shape.fft_type == FFT3231 || shape.fft_type == FFT32) && !time_FP32) continue; // Time an exponent that's good for all variants and carry-config. @@ -1195,8 +1198,9 @@ skip_1K_256 = false; // Loop through all possible variants for (u32 variant = 0; variant <= LAST_VARIANT; variant = next_variant (variant)) { - // Only FP64 code supports variants + // Only FP64 code supports variants. For FFT6431, we've not worked out how variant_M = 1 affects max exp. if (variant != 202 && !FFTConfig{shape, variant, CARRY_AUTO}.FFT_FP64) continue; + if (shape.fft_type == FFT6431 && variant_M(variant) == 1) continue; // Only AMD GPUs support variant zero (BCAST) and only if width <= 1024. CLANG doesn't support builtins. Let NO_ASM bypass variant zero. if (variant_W(variant) == 0) { From 037067be74311274937b24c18e4a8638e797ef90 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 22 Jul 2026 16:56:40 +0000 Subject: [PATCH 095/214] Fix (hopefully) errors using CUDA graphs and workers=2 --- src/cuda/clwrap_cuda.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 9d0f6f23..83cf6fcd 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -1193,7 +1193,7 @@ bool clIsGraphSupported(cl_device_id dev) { int clGraphBeginRecording(cl_command_queue q) { ensureContextCurrent(); - CUresult r = cuStreamBeginCapture(q->stream, CU_STREAM_CAPTURE_MODE_GLOBAL); + CUresult r = cuStreamBeginCapture(q->stream, CU_STREAM_CAPTURE_MODE_THREAD_LOCAL); return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; } From 4ff6c108e8353215454866eedb775fdd3fd7bfc4 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 22 Jul 2026 18:36:22 +0000 Subject: [PATCH 096/214] Removed unused halfbar. Fixed mis-typecast in shufl32. --- src/cl/base.cl | 5 ----- src/cl/fftbase.cl | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 4add30e8..d2a5297f 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -772,11 +772,6 @@ void OVERLOAD bar(const u32 WG) { } } -// A half-barrier is only needed when half-a-workgroup needs a barrier. -// This is used e.g. by the double-wide tailSquare, where LDS is split between the halves. -void halfBar() { if (get_enqueued_local_size(0) / 2 > WAVEFRONT) { bar(); } } - - // nVidia GPUs (Hopper architecture sm 9.0 and later) support Programatic Dependent Launch where the tail end execution of one kernel can overlap // with the beginning of the next kernel. This requires a special launch kernel command that is only available in CUDA 12.0 and later. // These routines let us take advantage of this CUDA feature. These routines do nothing in OpenCL. diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index e6c3c3a3..220d4527 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -1575,7 +1575,7 @@ void OVERLOAD fft_common(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 low #if NTT_GF31 void OVERLOAD shufl(local GF31 *lds, GF31 *u, u32 f, u32 numWG, u32 lowMe) { - shufl32((local F2 *) lds, (local F2 *) u, f, numWG, lowMe); + shufl32((local F2 *) lds, (F2 *) u, f, numWG, lowMe); } void OVERLOAD fft_RADIX(GF31 *u) { From 11059471aef2849c2f8a3b09069dfe7ff8b8b534 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 22 Jul 2026 23:25:23 +0000 Subject: [PATCH 097/214] Tune the NOREG option. I'm seeing better timings with NOREG on TitanV CUDA 13.0. --- src/tune.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tune.cpp b/src/tune.cpp index 3789860f..40529793 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -1085,7 +1085,7 @@ void Tune::tune() { } // See if disabling our default register usage makes sense - if (0) { + if (true) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); u32 best_noreg = 0; From da522b3315405f3a29cc39e1bf66a0956592548a Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Thu, 23 Jul 2026 04:54:58 -0700 Subject: [PATCH 098/214] Updated CI to add MSVC builds. --- .github/workflows/ci.yml | 90 +++++++++++++----- Makefile | 2 +- PRPLL.sln | 33 +++++++ PRPLL.vcxproj | 194 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 297 insertions(+), 22 deletions(-) create mode 100644 PRPLL.sln create mode 100644 PRPLL.vcxproj diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31d60e10..ed44d7c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: env: CXX: ${{ matrix.cxx }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install run: | sudo apt-get update -y @@ -30,15 +30,15 @@ jobs: $CXX --version - name: Script run: | - make -O -j "$(nproc)" - cd build-release + make DEBUG=1 -O -j "$(nproc)" + cd build-debug rm -f -- *.o ./prpll -h - uses: actions/upload-artifact@v7 if: always() with: name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_prpll - path: ${{ github.workspace }} + path: build-debug/ Linux-CUDA: name: Linux CUDA @@ -55,7 +55,7 @@ jobs: env: CXX: ${{ matrix.cxx }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install run: | sudo apt-get update -y @@ -63,22 +63,22 @@ jobs: $CXX --version - name: Script run: | - make CUDA=1 -O -j "$(nproc)" - cd build-cuda + make DEBUG=1 CUDA=1 -O -j "$(nproc)" + cd build-debug rm -f -- *.o ./prpll -h - uses: actions/upload-artifact@v7 if: always() with: name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_cuda_prpll - path: ${{ github.workspace }} + path: build-debug/ Cppcheck: name: Cppcheck runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install run: | sudo apt-get update -y @@ -91,7 +91,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Script run: clang-tidy -checks='bugprone-*,-bugprone-reserved-identifier,cert-*,-cert-dcl37-c,-cert-dcl51-cpp,clang-analyzer-*,concurrency-*,misc-*,-misc-no-recursion,modernize-*,-modernize-use-trailing-return-type,performance-*,portability-*,readability-const-return-type,readability-container-*,readability-duplicate-include,readability-else-after-return,readability-make-member-function-cons,readability-non-const-parameter,readability-redundant-*,readability-simplify-*,readability-string-compare,readability-use-*' -header-filter='.*' src/*.cpp -- -Wall -O3 -std=gnu++20 continue-on-error: true @@ -101,13 +101,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Script run: shopt -s globstar; shellcheck -o avoid-nullary-conditions,check-set-e-suppressed,deprecate-which,quote-safe-variables,require-double-brackets -s bash **/*.sh continue-on-error: true - Windows: - name: Windows + Windows-MSYS2: + name: Windows MSYS2 runs-on: ${{ matrix.os }} strategy: @@ -119,7 +119,7 @@ jobs: CXX: ${{ matrix.cxx }} PACKAGE_PREFIX: mingw-w64-${{ endsWith(matrix.os, '-arm') && 'clang-aarch64' || 'x86_64' }}- steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Before Install run: | echo "C:\msys64\mingw64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append @@ -137,15 +137,63 @@ jobs: & $env:CXX --version - name: Script run: | - make -O -j $env:NUMBER_OF_PROCESSORS - cd build-release + make DEBUG=1 -O -j $env:NUMBER_OF_PROCESSORS + cd build-debug rm *.o .\prpll.exe -h - uses: actions/upload-artifact@v7 if: always() with: name: win_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_prpll - path: ${{ github.workspace }} + path: build-debug/ + + Windows-MSVC-OpenCL: + name: Windows MSVC OpenCL + + runs-on: windows-2022 + steps: + - uses: actions/checkout@v7 + - uses: step-security/msvc-dev-cmd@v1 + - name: Install OpenCL + run: | + vcpkg install opencl + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always --match 'v/prpll/*')")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=OpenCL-Debug /p:OpenCLRoot=C:\vcpkg\installed\x64-windows + & .\build-msvc\OpenCL\Debug\prpll.exe -h + - uses: actions/upload-artifact@v7 + if: always() + with: + name: win_x86_msvc_opencl_prpll + path: build-msvc/ + + Windows-MSVC-CUDA: + name: Windows MSVC CUDA + + runs-on: windows-2022 + steps: + - uses: actions/checkout@v7 + - uses: step-security/msvc-dev-cmd@v1 + - name: Install CUDA Toolkit + uses: N-Storm/cuda-toolkit@v0.2.34 + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always --match 'v/prpll/*')")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=CUDA-Debug + - uses: actions/upload-artifact@v7 + if: always() + with: + name: win_x86_msvc_cuda_prpll + path: build-msvc/ macOS: name: macOS @@ -158,18 +206,18 @@ jobs: env: CXX: g++-15 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install run: | $CXX --version - name: Script run: | - make -j "$(sysctl -n hw.ncpu)" - cd build-release + make DEBUG=1 -j "$(sysctl -n hw.ncpu)" + cd build-debug rm -f -- *.o ./prpll -h - uses: actions/upload-artifact@v7 if: always() with: name: macos_${{ endsWith(matrix.os, '-intel') && 'x86' || 'arm' }}_prpll - path: ${{ github.workspace }} + path: build-debug/ diff --git a/Makefile b/Makefile index 1ba62799..31400c27 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ endif ifeq ($(DEBUG), 1) BIN=build-debug -CXXFLAGS = -g $(COMMON_FLAGS) +CXXFLAGS = -g -Og $(COMMON_FLAGS) STRIP= else diff --git a/PRPLL.sln b/PRPLL.sln new file mode 100644 index 00000000..25c0cdc3 --- /dev/null +++ b/PRPLL.sln @@ -0,0 +1,33 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PRPLL", "PRPLL.vcxproj", "{2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + OpenCL-Debug|x64 = OpenCL-Debug|x64 + OpenCL-Release|x64 = OpenCL-Release|x64 + CUDA-Debug|x64 = CUDA-Debug|x64 + CUDA-Release|x64 = CUDA-Release|x64 + CUDA-Release-non-static|x64 = CUDA-Release-non-static|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Debug|x64.ActiveCfg = OpenCL-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Debug|x64.Build.0 = OpenCL-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Release|x64.ActiveCfg = OpenCL-Release|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Release|x64.Build.0 = OpenCL-Release|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Debug|x64.ActiveCfg = CUDA-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Debug|x64.Build.0 = CUDA-Debug|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release|x64.ActiveCfg = CUDA-Release|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release|x64.Build.0 = CUDA-Release|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release-non-static|x64.ActiveCfg = CUDA-Release-non-static|x64 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release-non-static|x64.Build.0 = CUDA-Release-non-static|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {5185D8CA-7025-4E3B-8190-85F2B7B4A526} + EndGlobalSection +EndGlobal diff --git a/PRPLL.vcxproj b/PRPLL.vcxproj new file mode 100644 index 00000000..d63a2737 --- /dev/null +++ b/PRPLL.vcxproj @@ -0,0 +1,194 @@ + + + + + OpenCL-Debug + x64 + + + OpenCL-Release + x64 + + + CUDA-Debug + x64 + + + CUDA-Release + x64 + + + CUDA-Release-non-static + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + true + + + + 17.0 + {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A} + Win32Proj + PRPLL + PRPLL + 10.0 + + + + Application + true + MultiByte + v143 + + + Application + false + true + MultiByte + v143 + + + + + + + + + + $(ProjectDir)build-msvc\OpenCL\Debug\ + $(ProjectDir)build-msvc\obj\OpenCL\Debug\ + prpll + true + + + $(ProjectDir)build-msvc\OpenCL\Release\ + $(ProjectDir)build-msvc\obj\OpenCL\Release\ + prpll + + + $(ProjectDir)build-msvc\CUDA\Debug\ + $(ProjectDir)build-msvc\obj\CUDA\Debug\ + prpll + true + + + $(ProjectDir)build-msvc\CUDA\Release\ + $(ProjectDir)build-msvc\obj\CUDA\Release\ + prpll + + + $(ProjectDir)build-msvc\CUDA\Release-non-static\ + $(ProjectDir)build-msvc\obj\CUDA\Release-non-static\ + prpll + + + + Level3 + WIN32;WIN64;_CONSOLE;_CRT_SECURE_NO_WARNINGS;NOMINMAX;%(PreprocessorDefinitions) + stdcpp20 + Sync + true + + + Console + + + + + Disabled + _DEBUG;%(PreprocessorDefinitions) + MultiThreadedDebugDLL + ProgramDatabase + EnableFastChecks + + + true + + + + + MaxSpeed + true + true + NDEBUG;%(PreprocessorDefinitions) + MultiThreaded + + + true + true + true + + + + + $(OpenCLRoot)\debug\lib;%(AdditionalLibraryDirectories) + OpenCL.lib;%(AdditionalDependencies) + + + + + $(OpenCLRoot)\lib;%(AdditionalLibraryDirectories) + OpenCL.lib;%(AdditionalDependencies) + + + + + CUDA_BACKEND;%(PreprocessorDefinitions) + $(ProjectDir)\src\cuda;$(CUDA_PATH)\include;%(AdditionalIncludeDirectories) + + + $(CUDA_PATH)\lib\x64;%(AdditionalLibraryDirectories) + + + + + cuda.lib;nvrtc.lib;Ws2_32.lib;%(AdditionalDependencies) + + + + + cuda.lib;nvrtc_static.lib;nvrtc-builtins_static.lib;nvptxcompiler_static.lib;Ws2_32.lib;%(AdditionalDependencies) + + + + + From d8143063ad3f52e58b6f0894ca6a18f939c12c68 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Thu, 23 Jul 2026 05:54:56 -0700 Subject: [PATCH 099/214] Removed the legacy PrimeNet script. --- src/Args.cpp | 3 +- tools/primenet.py | 174 ---------------------------------------------- tools/upload.py | 114 ------------------------------ 3 files changed, 1 insertion(+), 290 deletions(-) delete mode 100755 tools/primenet.py delete mode 100755 tools/upload.py diff --git a/src/Args.cpp b/src/Args.cpp index 6e115a16..a11c4748 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -118,8 +118,7 @@ and should be able to run. Worktodo: PRPLL keeps the active tasks in per-worker files worktodo-0.txt, worktodo-1.txt etc in the local directory. These per-worker files are supplied from the global worktodo.txt file if -pool is used. -In turn the global worktodo.txt can be supplied through the primenet.py script, -either the one located at gpuowl/tools/primenet.py or https://download.mersenne.ca/AutoPrimeNet +In turn the work files can be supplied through AutoPrimeNet, located at https://download.mersenne.ca/AutoPrimeNet It is also possible to manually add exponents by adding lines of the form "PRP=118063003" to worktodo-.txt diff --git a/tools/primenet.py b/tools/primenet.py deleted file mode 100755 index e61e1a42..00000000 --- a/tools/primenet.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/python3 - -# Copyright (c) Mihai Preda. -# Inspired by mlucas-primenet.py , part of Mlucas by Ernst W. Mayer. - -import argparse -import time -import urllib -import requests -import os -import upload -import getpass - -from http import cookiejar -from urllib.parse import urlencode -from urllib.request import build_opener -from urllib.request import HTTPCookieProcessor -from datetime import datetime - -baseUrl = "https://www.mersenne.org/" -primenet = build_opener(HTTPCookieProcessor(cookiejar.CookieJar())) - -def login(user, password): - login = {"user_login": user, "user_password": password} - data = urlencode(login).encode('utf-8') - r = primenet.open(baseUrl, data).read().decode("utf-8") - if not user + "
logged in" in r: - print(r) - print("Login failed"); - raise(PermissionError("Login failed")) - -def loadLines(fileName): - try: - with open(fileName, 'r') as fi: - return set((line.strip().strip('\n') for line in fi)) - except FileNotFoundError as e: - return set() - -def sendOne(line): - print("Sending result: ", line) - data = urlencode({"data": line}).encode('utf-8') - res = primenet.open(baseUrl + "manual_result/default.php", data).read().decode("utf-8") - if "Error code" in res: - begin = res.find("Error code") - end = res.find("", begin) - text = res[begin:end] - print(text) - already = text.startswith('Error code: 40, error text: This computer has already sent in this PRP result') - if already: - print('Already sent, will not retry') - return already - else: - begin = res.find("CPU credit is") - end = res.find("", begin); - if begin >= 0 and end >= 0: - print(res[begin:end], '\n') - return True - else: - return False - -def appendLine(fileName, line): - with open(fileName, 'a') as fo: print(line, file = fo, end = '\n') - -def sendResults(results, sent, sentName, retryName): - for result in results: - ok = sendOne(result) - sent.add(result) - appendLine(sentName if ok else retryName, result) - -def fetch(what): - assignment = {"cores":1, "num_to_get":1, "pref":what} - # res = primenet.open(baseUrl + "manual_assignment/?" + urlencode(assignment)).read().decode("utf-8") - res = primenet.open(baseUrl + "manual_assignment/", data=urlencode(assignment).encode()).read().decode("utf-8") - # print(res) - - BEGIN_MARK = "" - # begin = res.find(BEGIN_MARK) - begin = res.find(">PRP=") - if begin == -1: begin = res.find(">LL=") - if begin == -1: - print(res) - raise(AssertionError("assignment no BEGIN mark")) - begin += 1 - # begin += len(BEGIN_MARK) - end = res.find("= end) - return True - -def getTask(userId): - url = f'http://mersenne.org/oneAssignment/&UserID={userId}&workpref=150' - print(url) - r = requests.get(url) - print(r) - print(r.json()) - -def uploadProof(userId, fileName, verbose=False): - exponent = headerExponent(fileName) - print(f'Uploading M{exponent} from "{fileName}"') - data = fileBytes(fileName) - return upload(userId, exponent, data, verbose) - -if __name__ == '__main__': - if len(sys.argv) < 3: - print(f'Usage: {sys.argv[0]} ') - exit(1) - - userId = sys.argv[1] - fileName = sys.argv[2] - if uploadProof(userId, fileName, verbose=True): - print('Success') - else: - exit(1) From dc18f72513173a31f19604198a7c74a3031db102 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Jul 2026 16:45:33 +0000 Subject: [PATCH 100/214] Require a clear advantage before overriding the default GRAPHS=1 setting. The default setting uses less CPU time (1.7% vs. 8% on my machine). --- src/tune.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tune.cpp b/src/tune.cpp index 40529793..bfef0050 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -938,7 +938,7 @@ void Tune::tune() { if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_modm31 = modm31; } } log("Best MODM31 is %u. Default MODM31 is 0.\n", best_modm31); - configsUpdate(current_cost, best_cost, 0.003, "MODM31", best_modm31, newConfigKeyVals, suggestedConfigKeyVals); + configsUpdate(current_cost, best_cost, 0.000, "MODM31", best_modm31, newConfigKeyVals, suggestedConfigKeyVals); args->flags["MODM31"] = to_string(best_modm31); } @@ -1064,7 +1064,7 @@ void Tune::tune() { // Find best CUDA compiler options #if CUDA_BACKEND - // Find best GRAPHS setting + // Find best GRAPHS setting. Require a clear advantage to override the default GRAPHS setting. GRAPHS=1 will use less CPU time. if (true) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); @@ -1080,7 +1080,7 @@ void Tune::tune() { if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_graphs = graphs; } } log("Best GRAPHS is %u. Default GRAPHS is 1.\n", best_graphs); - configsUpdate(current_cost, best_cost, 0.000, "GRAPHS", best_graphs, newConfigKeyVals, suggestedConfigKeyVals); + configsUpdate(current_cost, best_cost, 0.003, "GRAPHS", best_graphs, newConfigKeyVals, suggestedConfigKeyVals); args->flags["GRAPHS"] = to_string(best_graphs); } From ce05c9ff7c20147e82913e1deff4eae1c8a87c64 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Jul 2026 17:06:57 +0000 Subject: [PATCH 101/214] Tweaked help text --- src/Args.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/Args.cpp b/src/Args.cpp index a11c4748..256fb5af 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -167,16 +167,11 @@ named "config.txt" in the prpll run directory. -cache : use binary kernel cache; useful with repeated use of -roeTune and -tune -roe : measure the Round-Off Error (Z) for more iterations (slow) --use : comma separated list of defines for configuring gpuowl.cl, such as: +-use : comma separated list of defines for configuring openCL code, such as: -use FAST_BARRIER: on AMD Radeon VII and older AMD GPUs, use a faster barrier(). This option may not work on Nvidia GPUs or on RDNA AMD GPUs where it produces errors (which are nevertheless detected). -use NO_ASM : do not use __asm() blocks (inline assembly) - -use STATS= : enable carry statistics collection & logging, for the kernel according to : - 1 = CarryFused - 2 = CarryFusedMul - 4 = CarryA - 8 = CarryMul -use TAIL_KERNELS= : change how tailSquare operates according to : 0 = single wide, single kernel 1 = single wide, two kernels @@ -196,18 +191,21 @@ named "config.txt" in the prpll run directory. 1 = All trig values are pre-computed and read from memmory. -use DEBUG : enable asserts in OpenCL kernels (slow, developers) + -use STATS= : enable carry statistics collection & logging (developers), for the kernel according to : + 1 = CarryFused, 2 = CarryFusedMul, 4 = CarryA, 8 = CarryMul -tune : Looks for best settings to include in config.txt. Times many FFTs to find fastest one to test exponents -- written to tune.txt. An -fft can be given on the command line to limit which FFTs are timed. Options are not required. If present, the options are a comma separated list from below. noconfig - Skip timings to find best config.txt settings. - inplace - Skip timings for not-in-place FFTs and NTTs. All nVidia GPUs seem to prefer in-place FFTs and NTTs. - fp64 - Tune for settings that affect FP64 FFTs. Time FP64 FFTs for tune.txt. + inplace - Skip timings for not-in-place FFTs and NTTs. All nVidia GPUs seem to prefer in-place FFTs and NTTs. + fp64 - Tune for settings that affect FP64 FFTs. Time FP64 FFTs for tune.txt. ntt - Tune for settings that affect integer NTTs. Time integer NTTs for tune.txt. nofp32 - Do not tune for settings that affect FP32 FFTs. Some openCL compilers have trouble with FP32. minexp= - Time FFTs to find the best one for exponents greater than . maxexp= - Time FFTs to find the best one for exponents less than . - quick= - Higher values equals a quicker, potentially less accurate tune. Val ranges from 1 to 10. + fp6431 - Time FP64+M31 FFTs for tune.txt. Only GPUs with great FP64 performance will find this beneficial. + quick= - Use higher values for a quicker, potentially less accurate tune. Val ranges from 1 to 10. -device : select the GPU at position N in the list of devices -uid : select the GPU with the given UID (on ROCm/AMDGPU, Linux) -pci : select the GPU with the given PCI BDF, e.g. "0c:00.0" From 9270110508b377dd5cb05ca6409118466df48eaa Mon Sep 17 00:00:00 2001 From: george Date: Thu, 23 Jul 2026 21:49:00 +0000 Subject: [PATCH 102/214] Output GPU name at start up. --- src/Args.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Args.cpp b/src/Args.cpp index 256fb5af..05103425 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -420,7 +420,9 @@ void Args::parse(const string& line) { void Args::setDefaults() { uid = getUidFromPos(device); - log("device %d, OpenCL %s, unique id '%s'\n", device, getDriverVersionByPos(device).c_str(), uid.c_str()); + cl_device_id dev = getDevice(device); + log("device %d, OpenCL %s, %s, unique id '%s'\n", device, getDriverVersionByPos(device).c_str(), + isAmdGpu(dev) ? getBoardName(dev).c_str() : getDeviceName(dev).c_str(), uid.c_str()); if (!masterDir.empty()) { assert(masterDir.is_absolute()); From c577f9834bbd5eee0f7b376a336a6234a591737f Mon Sep 17 00:00:00 2001 From: george Date: Fri, 24 Jul 2026 21:05:35 +0000 Subject: [PATCH 103/214] Allow changing the L1 cache config when using CUDA --- src/Gpu.cpp | 6 ++++++ src/clwrap.h | 3 +++ src/cuda/clwrap_cuda.cpp | 9 +++++++++ src/tune.cpp | 22 +++++++++++++++++++++- 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 35b8c1fb..dbc72092 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -992,6 +992,12 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo // Set flag indicating we're going to use CUDA graphs use_graphs = graph_square[0].isSupported(shared.context->deviceId()) && args.value("GRAPHS", 1); + // Set L1 cache configuration. Really we should only do this once rather than once per worker. + // However, the current way PRPLL is organized would then make this option hard to tune. +#if CUDA_BACKEND + cudaSetL1Config(args.value("L1CUDA", 0)); +#endif + // Process the queue. I don't know if this is really needed. queue.finish(); } diff --git a/src/clwrap.h b/src/clwrap.h index b42c490e..3ef3fa20 100644 --- a/src/clwrap.h +++ b/src/clwrap.h @@ -133,6 +133,9 @@ u32 getEventInfo(cl_event event); cl_context getQueueContext(cl_command_queue q); #ifdef CUDA_BACKEND +// Set L1 cache configuration - 4 possibilities +void cudaSetL1Config(int x); + // Set L2 cache persistence for multiple read-only buffers on the given stream. // Computes the address span covering all buffers and sets a single access policy window. // Buffers that are nullptr or zero-size are skipped. diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 83cf6fcd..4f5700ea 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -1123,6 +1123,15 @@ int clSetKernelArgSVMPointer(cl_kernel k, unsigned pos, const void* ptr) { // C++ linkage — must be outside the extern "C" block above. +// Set L1 cache configuration +void cudaSetL1Config(int x) { + ensureContextCurrent(); + cuCtxSetCacheConfig (x == 0 ? CU_FUNC_CACHE_PREFER_NONE : // no preference for shared memory or L1 (default) + x == 1 ? CU_FUNC_CACHE_PREFER_SHARED : // prefer larger shared memory and smaller L1 cache + x == 2 ? CU_FUNC_CACHE_PREFER_L1 : // prefer larger L1 cache and smaller shared memory + CU_FUNC_CACHE_PREFER_EQUAL); // prefer equal sized L1 cache and shared memory +} + // Set L2 cache persistence for multiple read-only buffers on the given stream. // Computes the minimum address span covering all buffers, then sets one access policy // window with hitRatio sized so that only the actual buffer bytes get persisting treatment, diff --git a/src/tune.cpp b/src/tune.cpp index bfef0050..9011b572 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -1064,6 +1064,26 @@ void Tune::tune() { // Find best CUDA compiler options #if CUDA_BACKEND + // Find best L1CUDA setting. + if (true) { + FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; + u64 exponent = primes.prevPrime(fft.maxExp()); + u32 best_l1cuda = 0; + u32 current_l1cuda = args->value("L1CUDA", 0); + double best_cost = -1.0; + double current_cost = -1.0; + for (u32 l1cuda : {0, 1, 2, 3}) { + args->flags["L1CUDA"] = to_string(l1cuda); + double cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); + log("Time for %12s using L1CUDA=%u is %6.1f\n", fft.spec().c_str(), l1cuda, cost); + if (l1cuda == current_l1cuda) current_cost = cost; + if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_l1cuda = l1cuda; } + } + log("Best L1CUDA is %u. Default L1CUDA is 1.\n", best_l1cuda); + configsUpdate(current_cost, best_cost, 0.000, "L1CUDA", best_l1cuda, newConfigKeyVals, suggestedConfigKeyVals); + args->flags["L1CUDA"] = to_string(best_l1cuda); + } + // Find best GRAPHS setting. Require a clear advantage to override the default GRAPHS setting. GRAPHS=1 will use less CPU time. if (true) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; @@ -1084,7 +1104,7 @@ void Tune::tune() { args->flags["GRAPHS"] = to_string(best_graphs); } - // See if disabling our default register usage makes sense + // See if disabling the default register usage makes sense if (true) { FFTConfig fft{*defaultShape, variant, CARRY_AUTO}; u64 exponent = primes.prevPrime(fft.maxExp()); From b4716e9797145afbae486bc85bc42db73fd9dfe6 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 24 Jul 2026 22:24:17 +0000 Subject: [PATCH 104/214] Added L!CUDA to list of recognized -use options --- src/Gpu.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index dbc72092..f313cc06 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -291,7 +291,8 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector Date: Sat, 25 Jul 2026 00:33:14 +0000 Subject: [PATCH 105/214] Output -time timings to more precision (RTX 5090 is bloody fast!). Align output better. --- src/Gpu.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index f313cc06..5e6f99d6 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -1817,8 +1817,8 @@ void Gpu::logTimeKernels() { double const percent = 100.0 / total * p->times[2]; if (!args.verbose && percent < 0.2) { break; } snprintf(buf, sizeof(buf), - args.verbose ? "%s %5.2f%% %-11s : %6.0f us/call x %5d calls (%.3f %.0f)\n" - : "%s %5.2f%% %-11s %4.0f x%6d %.3f %.0f\n", + args.verbose ? "%s %5.2f%% %-18s : %6.1f us/call x %5d calls (%.3f %.0f)\n" + : "%s %5.2f%% %-18s %6.1f x%6d %.3f %.0f\n", logContext().c_str(), percent, p->name.c_str(), p->times[2] * f, n, p->times[0] * (f * 1e-3), p->times[1] * (f * 1e-3)); s += buf; From 2b555a3cd580248ab043acdb6a6714f235bc46ad Mon Sep 17 00:00:00 2001 From: george Date: Sun, 26 Jul 2026 02:04:46 +0000 Subject: [PATCH 106/214] Fixed typo --- src/tune.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tune.cpp b/src/tune.cpp index 9011b572..3b4ee686 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -1079,7 +1079,7 @@ void Tune::tune() { if (l1cuda == current_l1cuda) current_cost = cost; if (best_cost < 0.0 || cost < best_cost) { best_cost = cost; best_l1cuda = l1cuda; } } - log("Best L1CUDA is %u. Default L1CUDA is 1.\n", best_l1cuda); + log("Best L1CUDA is %u. Default L1CUDA is 0.\n", best_l1cuda); configsUpdate(current_cost, best_cost, 0.000, "L1CUDA", best_l1cuda, newConfigKeyVals, suggestedConfigKeyVals); args->flags["L1CUDA"] = to_string(best_l1cuda); } From ee9fe6395e8ac1e00f43d3bc51d72c001f1fe3e2 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Sun, 26 Jul 2026 03:18:04 -0700 Subject: [PATCH 107/214] Added CD workflow to build release executables. --- .github/workflows/cd.yml | 157 +++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 11 ++- Makefile | 6 +- PRPLL.vcxproj | 10 ++- src/cuda/clwrap_cuda.cpp | 6 +- 5 files changed, 183 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/cd.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..017af7cb --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,157 @@ +name: CD + +on: + push: + tags: + - '*' + workflow_dispatch: + +jobs: + Linux-OpenCL: + name: Linux OpenCL + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - name: Script + run: | + docker run --rm -i -v "$GITHUB_WORKSPACE:/workspace" -w /workspace ubuntu:22.04 bash -s <<'EOF' + set -e -o pipefail + apt-get update -y + apt-get install -y build-essential git ocl-icd-opencl-dev + g++ --version + + make -O -j "$(nproc)" + cd build-release + rm -f -- *.o + ./prpll -h + EOF + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_linux_x86_opencl + path: | + README.* + LICENSE + tools/ + build-release/* + + Linux-CUDA: + name: Linux CUDA + + runs-on: ubuntu-latest + strategy: + matrix: + include: + - cuda: '13.2.1' + container: 'ubuntu24.04' + - cuda: '12.9.2' + container: 'ubuntu24.04' + - cuda: '11.8.0' + container: 'ubuntu22.04' + # - cuda: '10.2' + # container: 'ubuntu18.04' + fail-fast: false + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - name: Script + run: | + docker run --rm -i -v "$GITHUB_WORKSPACE:/workspace" -w /workspace "nvcr.io/nvidia/cuda:${{ matrix.cuda }}-devel-${{ matrix.container }}" bash -s <<'EOF' + set -e -o pipefail + apt-get update -y + apt-get install -y build-essential git + g++ --version + + make CUDA=1 ${{ matrix.cuda != '10.2' && 'CUDA_STATIC=1' || '' }} -O -j "$(nproc)" + cd build-cuda + rm -f -- *.o + if [[ "${{ matrix.cuda }}" == "10.2" ]]; then + cp /usr/local/cuda/lib64/libnvrtc.so.10.2 /usr/local/cuda/lib64/libnvrtc-builtins.so.10.2 . + fi + # ./prpll -h + EOF + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_linux_x86_cuda_${{ matrix.cuda }} + path: | + README.* + LICENSE + tools/ + build-cuda/* + + Windows-OpenCL: + name: Windows OpenCL + + runs-on: windows-2022 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - uses: step-security/msvc-dev-cmd@v1 + - name: Install OpenCL + run: | + vcpkg install opencl + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=OpenCL-Release /p:OpenCLRoot=C:\vcpkg\installed\x64-windows + cd build-msvc\OpenCL\Release\ + & .\prpll -h + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_win_x86_opencl + path: | + README.* + LICENSE + tools/ + build-msvc/OpenCL/Release/* + + Windows-CUDA: + name: Windows CUDA + + runs-on: windows-2022 + strategy: + matrix: + cuda: ['13.2.1', '12.9.2', '11.8.0', '10.2.89'] + fail-fast: false + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + fetch-tags: true + - uses: step-security/msvc-dev-cmd@v1 + - name: Install CUDA Toolkit + uses: N-Storm/cuda-toolkit@v0.2.34 + with: + cuda: ${{ matrix.cuda }} + - name: Before Script + shell: bash + run: | + bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc + - name: Script + run: | + msbuild PRPLL.sln /m /p:Configuration=CUDA-${{ matrix.cuda != '10.2.89' && 'Release' || 'Release-non-static' }} + cd build-msvc\CUDA\Release*\ + if ("${{ matrix.cuda }}" -eq "10.2.89") { + Copy-Item "$env:CUDA_PATH\bin\nvrtc64_102_0.dll", "$env:CUDA_PATH\bin\nvrtc-builtins64_102.dll" . + } + # & .\prpll -h + - uses: actions/upload-artifact@v7 + with: + name: PRPLL-NTT_win_x86_cuda_${{ matrix.cuda }} + path: | + README.* + LICENSE + tools/ + build-msvc/CUDA/Release*/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed44d7c5..b6f1c7e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/upload-artifact@v7 if: always() with: - name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_prpll + name: ${{ matrix.os }}_${{ endsWith(matrix.os, '-arm') && 'arm' || 'x86' }}_${{ matrix.cxx }}_opencl_prpll path: build-debug/ Linux-CUDA: @@ -161,11 +161,12 @@ jobs: shell: bash run: | bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp - printf '"%s"\n' "$(basename "$(git describe --tags --long --always --match 'v/prpll/*')")" > src/version.inc + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc - name: Script run: | msbuild PRPLL.sln /m /p:Configuration=OpenCL-Debug /p:OpenCLRoot=C:\vcpkg\installed\x64-windows - & .\build-msvc\OpenCL\Debug\prpll.exe -h + cd build-msvc\OpenCL\Debug\ + & .\prpll -h - uses: actions/upload-artifact@v7 if: always() with: @@ -185,10 +186,12 @@ jobs: shell: bash run: | bash genbundle.sh src/cuda/*.cuh src/cl/*.cl > src/bundle.cpp - printf '"%s"\n' "$(basename "$(git describe --tags --long --always --match 'v/prpll/*')")" > src/version.inc + printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc - name: Script run: | msbuild PRPLL.sln /m /p:Configuration=CUDA-Debug + # cd build-msvc\CUDA\Debug\ + # & .\prpll -h - uses: actions/upload-artifact@v7 if: always() with: diff --git a/Makefile b/Makefile index 31400c27..da1dabaf 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,11 @@ ifeq ($(CUDA), 1) CUDASRCS1 = clwrap_cuda.cpp cudawrap.cpp CUDAFLAGS = -DCUDA_BACKEND -Isrc/cuda -I/usr/local/cuda/include CUDAOBJS = $(CUDASRCS1:%.cpp=$(BIN)/%.o) - OPENCL_LIBS = -L/usr/local/cuda/lib64 -lcuda -lnvrtc + ifeq ($(CUDA_STATIC), 1) + OPENCL_LIBS = -L/usr/local/cuda/lib64 -Wl,--start-group -lnvrtc_static -lnvrtc-builtins_static -lnvptxcompiler_static -Wl,--end-group -lcuda + else + OPENCL_LIBS = -L/usr/local/cuda/lib64 -lnvrtc -lcuda + endif else BIN=build-release CUDAFLAGS = diff --git a/PRPLL.vcxproj b/PRPLL.vcxproj index d63a2737..49cf61fc 100644 --- a/PRPLL.vcxproj +++ b/PRPLL.vcxproj @@ -158,6 +158,14 @@ true + + + WINVER=0x0601;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions) + + + /SUBSYSTEM:CONSOLE,6.01 %(AdditionalOptions) + + $(OpenCLRoot)\debug\lib;%(AdditionalLibraryDirectories) @@ -186,7 +194,7 @@ - cuda.lib;nvrtc_static.lib;nvrtc-builtins_static.lib;nvptxcompiler_static.lib;Ws2_32.lib;%(AdditionalDependencies) + cuda.lib;nvrtc_static.lib;nvrtc-builtins_static.lib;nvptxcompiler_static.lib;User32.lib;Ws2_32.lib;%(AdditionalDependencies) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 4f5700ea..20cd3c21 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -1136,6 +1136,7 @@ void cudaSetL1Config(int x) { // Computes the minimum address span covering all buffers, then sets one access policy // window with hitRatio sized so that only the actual buffer bytes get persisting treatment, // not the gaps between non-contiguous allocations. +#if CUDA_VERSION >= 11000 [[maybe_unused]] static void cudaSetL2Persistent(cl_command_queue q, const std::vector& buffers) { if (!q) return; @@ -1188,6 +1189,7 @@ void cudaSetL1Config(int x) { buffers.size()); } } +#endif // OpenCL-like extensions invented to provide a clean interface to some nVidia CUDA features @@ -1213,8 +1215,10 @@ int clGraphEndRecording(cl_command_queue q, cl_graph* graph) { CUresult r = cuStreamEndCapture(q->stream, &g->graph); #if CUDA_VERSION >= 12000 if (r == CUDA_SUCCESS) r = cuGraphInstantiate(&g->graphExec, g->graph, 0); -#else +#elif CUDA_VERSION >= 11040 if (r == CUDA_SUCCESS) r = cuGraphInstantiateWithFlags(&g->graphExec, g->graph, 0); +#else + if (r == CUDA_SUCCESS) r = cuGraphInstantiate(&g->graphExec, g->graph, nullptr, nullptr, 0); #endif *graph = g; return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; From 8b7d6ce973cc5d49a7bab0b18c766e5719f534f2 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 27 Jul 2026 16:10:49 +0000 Subject: [PATCH 108/214] The selfTest code (for developers only) was causing issues on some GPUs. Changed -verbose from a true/false flag to a level. SelfTest is now activated by -verbose 99. --- src/Args.cpp | 9 +++++---- src/Args.h | 2 +- src/Gpu.cpp | 2 +- src/KernelCompiler.h | 2 +- src/common.h | 4 ++-- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Args.cpp b/src/Args.cpp index 05103425..179bbd83 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -15,8 +15,8 @@ #include #include -// This is a copy of the args.verbose flag. It allows the CUDA wrapper to access the flag. -bool prpll_verbose = false; +// This is a copy of the args.verbose level. It allows the CUDA wrapper to access the value. +int prpll_verbose = 0; int Args::value(const string& key, int valNotFound) const { auto it = flags.find(key); @@ -319,8 +319,9 @@ void Args::parse(const string& line) { } else if (key == "-carryTune") { carryTune = true; } else if (key == "-verbose" || key == "-v") { - verbose = true; - prpll_verbose = true; + if (s.empty()) verbose = 1; + else verbose = stoi(s); + prpll_verbose = verbose; } else if (key == "-time") { profile = true; } else if (key == "-workers") { diff --git a/src/Args.h b/src/Args.h index d876ce79..5f1a1a0c 100644 --- a/src/Args.h +++ b/src/Args.h @@ -58,7 +58,7 @@ class Args { bool safeMath = true; bool clean = true; - bool verbose = false; + int verbose = 0; bool useCache = false; bool profile = false; bool smallest = false; diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 5e6f99d6..e6293f91 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -981,7 +981,7 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo bufStatsCarry.zero(); bufTrue.write({1}); - if (args.verbose) { + if (args.verbose >= 99) { selftestTrig(); } diff --git a/src/KernelCompiler.h b/src/KernelCompiler.h index 19cd9d60..a5588331 100644 --- a/src/KernelCompiler.h +++ b/src/KernelCompiler.h @@ -18,7 +18,7 @@ class KernelCompiler { std::string baseArgs; std::string dump; const bool useCache; - const bool verbose; + const int verbose; std::vector clSources; std::vector> files; diff --git a/src/common.h b/src/common.h index 3da39734..13a010e9 100644 --- a/src/common.h +++ b/src/common.h @@ -6,8 +6,8 @@ #include #include -// This is a copy of the args.verbose flag. It allows the CUDA wrapper to access the flag. -extern bool prpll_verbose; +// This is a copy of the args.verbose level. It allows the CUDA wrapper to access the value. +extern int prpll_verbose; using u8 = uint8_t; using i32 = int32_t; From 4a23f04fc2dacfb1b854540f14418b23e482f580 Mon Sep 17 00:00:00 2001 From: george Date: Wed, 29 Jul 2026 03:22:36 +0000 Subject: [PATCH 109/214] Fixed typo bugs in unused LDSSWIZ code --- src/cl/fftbase.cl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 220d4527..36137055 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -135,7 +135,7 @@ void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). - if (!force_default && f == 1 && n == 4) { + if (!force_default && f == 1 && RADIX == 4) { bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } bar(WG); @@ -312,7 +312,7 @@ void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). - if (!force_default && f == 4 && n == RADIX) { + if (!force_default && f == 4 && RADIX == 4) { bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } bar(WG); From 9d4be5208338a000c9c2625928cdd6c8022f3a26 Mon Sep 17 00:00:00 2001 From: Teal Dulcet Date: Mon, 27 Jul 2026 05:13:03 -0700 Subject: [PATCH 110/214] Updated CD workflow based on feedback. --- .github/workflows/cd.yml | 86 +++++++++++++++++++++------------------- .github/workflows/ci.yml | 9 ++--- Makefile | 45 ++++++++++++--------- PRPLL.sln | 3 -- PRPLL.vcxproj | 32 +++++++-------- 5 files changed, 89 insertions(+), 86 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 017af7cb..06dd8455 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -18,25 +18,28 @@ jobs: fetch-tags: true - name: Script run: | - docker run --rm -i -v "$GITHUB_WORKSPACE:/workspace" -w /workspace ubuntu:22.04 bash -s <<'EOF' + docker run --rm -i -v "$GITHUB_WORKSPACE:/workspace" -w /workspace centos:7 bash -s <<'EOF' set -e -o pipefail - apt-get update -y - apt-get install -y build-essential git ocl-icd-opencl-dev + sed -i -e '/^mirrorlist/d;/^#baseurl=/{s,^#,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + # yum update -y + yum install -y centos-release-scl epel-release + sed -i -e '/^mirrorlist/d;/^# *baseurl=/{s,^# *,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + yum install -y devtoolset-11-gcc-c++ devtoolset-11-libstdc++-static make git ocl-icd-devel + + source /opt/rh/devtoolset-11/enable g++ --version + ldd --version - make -O -j "$(nproc)" + make STATIC_RUNTIME=1 -j "$(nproc)" + cp -vr README.* LICENSE tools/ build-release/ cd build-release rm -f -- *.o ./prpll -h EOF - uses: actions/upload-artifact@v7 with: - name: PRPLL-NTT_linux_x86_opencl - path: | - README.* - LICENSE - tools/ - build-release/* + name: PRPLL-NTT_linux_x64_opencl + path: build-release/* Linux-CUDA: name: Linux CUDA @@ -46,13 +49,13 @@ jobs: matrix: include: - cuda: '13.2.1' - container: 'ubuntu24.04' + container: 'rockylinux8' - cuda: '12.9.2' - container: 'ubuntu24.04' + container: 'rockylinux8' - cuda: '11.8.0' - container: 'ubuntu22.04' - # - cuda: '10.2' - # container: 'ubuntu18.04' + container: 'centos7' + - cuda: '10.2' + container: 'centos7' fail-fast: false steps: - uses: actions/checkout@v7 @@ -63,26 +66,35 @@ jobs: run: | docker run --rm -i -v "$GITHUB_WORKSPACE:/workspace" -w /workspace "nvcr.io/nvidia/cuda:${{ matrix.cuda }}-devel-${{ matrix.container }}" bash -s <<'EOF' set -e -o pipefail - apt-get update -y - apt-get install -y build-essential git + if [[ "${{ matrix.container }}" == centos* ]]; then + sed -i -e '/^mirrorlist/d;/^#baseurl=/{s,^#,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + # yum update -y + yum install -y centos-release-scl epel-release + sed -i -e '/^mirrorlist/d;/^# *baseurl=/{s,^# *,,;s,/mirror,/vault,;}' /etc/yum.repos.d/CentOS*.repo + yum install -y devtoolset-11-gcc-c++ devtoolset-11-libstdc++-static make git + source /opt/rh/devtoolset-11/enable + else + # dnf update -y + dnf install -y gcc-toolset-11-gcc-c++ make git + source /opt/rh/gcc-toolset-11/enable + fi + g++ --version + ldd --version - make CUDA=1 ${{ matrix.cuda != '10.2' && 'CUDA_STATIC=1' || '' }} -O -j "$(nproc)" + make CUDA=1 STATIC_RUNTIME=1 STATIC_CUDA=${{ matrix.cuda != '10.2' && '1' || '0' }} -j "$(nproc)" + cp -vr README.* LICENSE tools/ build-cuda/ cd build-cuda rm -f -- *.o if [[ "${{ matrix.cuda }}" == "10.2" ]]; then - cp /usr/local/cuda/lib64/libnvrtc.so.10.2 /usr/local/cuda/lib64/libnvrtc-builtins.so.10.2 . + cp -v /usr/local/cuda/lib64/{libnvrtc.so.10.2,libnvrtc-builtins.so.10.2} . fi # ./prpll -h EOF - uses: actions/upload-artifact@v7 with: - name: PRPLL-NTT_linux_x86_cuda_${{ matrix.cuda }} - path: | - README.* - LICENSE - tools/ - build-cuda/* + name: PRPLL-NTT_linux_x64_cuda_${{ matrix.cuda }} + path: build-cuda/* Windows-OpenCL: name: Windows OpenCL @@ -104,17 +116,14 @@ jobs: printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc - name: Script run: | - msbuild PRPLL.sln /m /p:Configuration=OpenCL-Release /p:OpenCLRoot=C:\vcpkg\installed\x64-windows + msbuild PRPLL.sln /m /p:Configuration=OpenCL-Release /p:StaticRuntime=true /p:OpenCLRoot=C:\vcpkg\installed\x64-windows + Copy-Item -Recurse README.*, LICENSE, tools\ build-msvc\OpenCL\Release\ cd build-msvc\OpenCL\Release\ & .\prpll -h - uses: actions/upload-artifact@v7 with: - name: PRPLL-NTT_win_x86_opencl - path: | - README.* - LICENSE - tools/ - build-msvc/OpenCL/Release/* + name: PRPLL-NTT_win_x64_opencl + path: build-msvc/OpenCL/Release/* Windows-CUDA: name: Windows CUDA @@ -141,17 +150,14 @@ jobs: printf '"%s"\n' "$(basename "$(git describe --tags --long --always)")" > src/version.inc - name: Script run: | - msbuild PRPLL.sln /m /p:Configuration=CUDA-${{ matrix.cuda != '10.2.89' && 'Release' || 'Release-non-static' }} - cd build-msvc\CUDA\Release*\ + msbuild PRPLL.sln /m /p:Configuration=CUDA-Release /p:StaticRuntime=true /p:StaticCUDA=${{ matrix.cuda != '10.2.89' && 'true' || 'false' }} + Copy-Item -Recurse README.*, LICENSE, tools\ build-msvc\CUDA\Release\ + cd build-msvc\CUDA\Release\ if ("${{ matrix.cuda }}" -eq "10.2.89") { Copy-Item "$env:CUDA_PATH\bin\nvrtc64_102_0.dll", "$env:CUDA_PATH\bin\nvrtc-builtins64_102.dll" . } # & .\prpll -h - uses: actions/upload-artifact@v7 with: - name: PRPLL-NTT_win_x86_cuda_${{ matrix.cuda }} - path: | - README.* - LICENSE - tools/ - build-msvc/CUDA/Release*/* + name: PRPLL-NTT_win_x64_cuda_${{ matrix.cuda }} + path: build-msvc/CUDA/Release/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6f1c7e2..d038df19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: pull_request: schedule: - - cron: '0 0 1 * *' + - cron: '0 0 1 * *' jobs: Linux-OpenCL: @@ -48,9 +48,6 @@ jobs: matrix: os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04] cxx: [g++, clang++] - exclude: - - os: ubuntu-22.04-arm - cxx: clang++ fail-fast: false env: CXX: ${{ matrix.cxx }} @@ -133,7 +130,7 @@ jobs: - name: Install Clang if: ${{ matrix.cxx == 'clang++' }} run: | - pacman -S --noconfirm mingw-w64-x86_64-clang + pacman -S --noconfirm "${env:PACKAGE_PREFIX}clang" & $env:CXX --version - name: Script run: | @@ -204,7 +201,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-15-intel, macos-latest] + os: [macos-26-intel, macos-latest] fail-fast: false env: CXX: g++-15 diff --git a/Makefile b/Makefile index da1dabaf..c41aa5fb 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,11 @@ # On Windows invoke with "make exe" or "make all" +DEBUG = 0 +CUDA = 0 +STATIC_RUNTIME = 0 +STATIC_CUDA = 0 + # Uncomment below as desired to set a particular compiler or force a debug build: # CXX = g++-12 # DEBUG = 1 @@ -12,22 +17,17 @@ HOST_OS = $(shell uname -s) -ifeq ($(HOST_OS), Darwin) -# Real GCC (not clang), needed for 128-bit floats and std::filesystem::path -CXX ?= g++-15 -else CXX ?= g++ -endif ifeq ($(CUDA), 1) BIN=build-cuda CUDASRCS1 = clwrap_cuda.cpp cudawrap.cpp CUDAFLAGS = -DCUDA_BACKEND -Isrc/cuda -I/usr/local/cuda/include CUDAOBJS = $(CUDASRCS1:%.cpp=$(BIN)/%.o) - ifeq ($(CUDA_STATIC), 1) - OPENCL_LIBS = -L/usr/local/cuda/lib64 -Wl,--start-group -lnvrtc_static -lnvrtc-builtins_static -lnvptxcompiler_static -Wl,--end-group -lcuda + ifeq ($(STATIC_CUDA), 1) + OPENCL_LIBS = -L/usr/local/cuda/lib64 -Wl,--start-group -lnvrtc_static -lnvrtc-builtins_static -lnvptxcompiler_static -Wl,--end-group -lcuda -lpthread -ldl else - OPENCL_LIBS = -L/usr/local/cuda/lib64 -lnvrtc -lcuda + OPENCL_LIBS = -L/usr/local/cuda/lib64 -Wl,-rpath,'$$ORIGIN' -lnvrtc -lcuda -lpthread endif else BIN=build-release @@ -36,15 +36,24 @@ else ifeq ($(HOST_OS), Darwin) OPENCL_LIBS = -framework OpenCL else - OPENCL_LIBS = -lOpenCL + OPENCL_LIBS = -lOpenCL -lpthread endif endif -ifneq ($(findstring MINGW, $(HOST_OS)), MINGW) - COMMON_FLAGS = -Wall $(CUDAFLAGS) -std=c++20 -static-libstdc++ -static-libgcc -else +COMMON_FLAGS = -Wall -Wextra $(CUDAFLAGS) -std=c++20 + +ifeq ($(STATIC_RUNTIME),1) + LDFLAGS += -static-libstdc++ -static-libgcc + + ifeq ($(findstring MINGW, $(HOST_OS)), MINGW) # For mingw-64 use this: - COMMON_FLAGS = -Wall $(CUDAFLAGS) -std=c++20 -static-libstdc++ -static-libgcc -static + LDFLAGS += -static + endif +endif + +ifeq ($(findstring MINGW, $(HOST_OS)), MINGW) + CPPFLAGS += -DWINVER=0x0601 -D_WIN32_WINNT=0x0601 + LDFLAGS += -Wl,--subsystem,console:6.01 endif # -fext-numeric-literals @@ -52,12 +61,10 @@ ifeq ($(DEBUG), 1) BIN=build-debug CXXFLAGS = -g -Og $(COMMON_FLAGS) -STRIP= else -CXXFLAGS = -O3 -DNDEBUG $(COMMON_FLAGS) -STRIP=-s +CXXFLAGS = -O3 -flto -DNDEBUG $(COMMON_FLAGS) endif @@ -81,14 +88,14 @@ prpll: $(BIN)/prpll amd: $(BIN)/prpll-amd #$(BIN)/test: $(BIN)/test.o -# $(CXX) $(CXXFLAGS) -o $@ $< $(LIBPATH) ${STRIP} +# $(CXX) $(CXXFLAGS) -o $@ $< $(LIBPATH) $(BIN)/prpll: ${OBJS} - $(CXX) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) $(OPENCL_LIBS) ${STRIP} + $(CXX) $(LDFLAGS) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) $(OPENCL_LIBS) # Instead of linking with libOpenCL, link with libamdocl64 $(BIN)/prpll-amd: ${OBJS} - $(CXX) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) -lamdocl64 -L/opt/rocm/lib ${STRIP} + $(CXX) $(LDFLAGS) $(CXXFLAGS) -o $@ ${OBJS} $(LIBPATH) -lamdocl64 -L/opt/rocm/lib clean: rm -rf build-debug build-release build-cuda diff --git a/PRPLL.sln b/PRPLL.sln index 25c0cdc3..6eaecdd1 100644 --- a/PRPLL.sln +++ b/PRPLL.sln @@ -10,7 +10,6 @@ Global OpenCL-Release|x64 = OpenCL-Release|x64 CUDA-Debug|x64 = CUDA-Debug|x64 CUDA-Release|x64 = CUDA-Release|x64 - CUDA-Release-non-static|x64 = CUDA-Release-non-static|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.OpenCL-Debug|x64.ActiveCfg = OpenCL-Debug|x64 @@ -21,8 +20,6 @@ Global {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Debug|x64.Build.0 = CUDA-Debug|x64 {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release|x64.ActiveCfg = CUDA-Release|x64 {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release|x64.Build.0 = CUDA-Release|x64 - {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release-non-static|x64.ActiveCfg = CUDA-Release-non-static|x64 - {2B3F9C8E-3384-0407-D9E1-D3E86AA8823A}.CUDA-Release-non-static|x64.Build.0 = CUDA-Release-non-static|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PRPLL.vcxproj b/PRPLL.vcxproj index 49cf61fc..743cb33d 100644 --- a/PRPLL.vcxproj +++ b/PRPLL.vcxproj @@ -17,10 +17,6 @@ CUDA-Release x64 - - CUDA-Release-non-static - x64 - @@ -79,7 +75,7 @@ MultiByte v143 - + Application false true @@ -92,7 +88,10 @@ - + + true + false + $(ProjectDir)build-msvc\OpenCL\Debug\ $(ProjectDir)build-msvc\obj\OpenCL\Debug\ @@ -115,14 +114,9 @@ $(ProjectDir)build-msvc\obj\CUDA\Release\ prpll - - $(ProjectDir)build-msvc\CUDA\Release-non-static\ - $(ProjectDir)build-msvc\obj\CUDA\Release-non-static\ - prpll - - Level3 + Level4 WIN32;WIN64;_CONSOLE;_CRT_SECURE_NO_WARNINGS;NOMINMAX;%(PreprocessorDefinitions) stdcpp20 Sync @@ -136,7 +130,8 @@ Disabled _DEBUG;%(PreprocessorDefinitions) - MultiThreadedDebugDLL + MultiThreadedDebug + MultiThreadedDebugDLL ProgramDatabase EnableFastChecks @@ -144,13 +139,14 @@ true - + MaxSpeed true true NDEBUG;%(PreprocessorDefinitions) - MultiThreaded + MultiThreaded + MultiThreadedDLL true @@ -178,7 +174,7 @@ OpenCL.lib;%(AdditionalDependencies) - + CUDA_BACKEND;%(PreprocessorDefinitions) $(ProjectDir)\src\cuda;$(CUDA_PATH)\include;%(AdditionalIncludeDirectories) @@ -187,12 +183,12 @@ $(CUDA_PATH)\lib\x64;%(AdditionalLibraryDirectories) - + cuda.lib;nvrtc.lib;Ws2_32.lib;%(AdditionalDependencies) - + cuda.lib;nvrtc_static.lib;nvrtc-builtins_static.lib;nvptxcompiler_static.lib;User32.lib;Ws2_32.lib;%(AdditionalDependencies) From 8f820d856ea0ca4d46ede27460fb10d738a267df Mon Sep 17 00:00:00 2001 From: george Date: Wed, 29 Jul 2026 17:56:59 +0000 Subject: [PATCH 111/214] Implemented LDS padding for SHUFL_BYTES=4. Not used. Will test if lower shared memory usage in tailSquareGF31 with SHUFL_BYTES=4 is beneficial. --- src/cl/fftbase.cl | 94 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 36137055..704e84dc 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -12,6 +12,10 @@ #define LDS_BYTES ((WG * RADIX + 56) * SHUFL_BYTES) #elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 4 #define LDS_BYTES ((WG * RADIX + 12) * SHUFL_BYTES) +#elif LDSPAD && SHUFL_BYTES == 4 && RADIX == 8 +#define LDS_BYTES ((WG * RADIX + 56) * SHUFL_BYTES) +#elif LDSPAD && SHUFL_BYTES == 4 && RADIX == 4 +#define LDS_BYTES ((WG * RADIX + 12) * SHUFL_BYTES) #else #define LDS_BYTES (WG * RADIX * SHUFL_BYTES) #endif @@ -401,7 +405,7 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i]; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } return; } @@ -430,7 +434,7 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } return; } @@ -514,13 +518,93 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } } - // If SHUFL_BYTES is 4 we split the F2 values into 2 int values. These are written to LDS memory using two instructions. - // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. else if (SHUFL_BYTES == 4) { - // Accessing lds memory as ints might be faster than F2 accesses (halving LDS memory requirements) local F* lds = ((local F*) lds2); if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F); +#if LDSPAD + // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 2, 66..., 3, 67..., 32, 96... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + return; + } + + // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (!force_default && f == 8 && RADIX == 8) { + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + return; + } + + // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 32 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 7.., 32... lds[64..127] = +8 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + return; + } + + // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 1.... ... 8... lds[64..127] = +2 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + return; + } +#endif + bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } bar(WG); From 6f1e5d726fc3af0a705362e9daef42d816c4068f Mon Sep 17 00:00:00 2001 From: george Date: Thu, 30 Jul 2026 18:15:54 +0000 Subject: [PATCH 112/214] Restored a bit of useless code. Having the useless code improves performance by 1% for FP64+M31 FFTs on TitanV CUDA 13.0. I hate optimizer bugs. --- src/cl/fftbase.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 704e84dc..9416c76b 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -405,7 +405,7 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i]; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } return; } From 080df11ebbeb0f973630d12aef1a85e1c2a75c75 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 2 Aug 2026 21:21:56 +0000 Subject: [PATCH 113/214] Fixed more compiler warning messages --- src/Gpu.cpp | 8 ++++---- src/Proof.cpp | 4 ++-- src/Saver.cpp | 4 ++-- src/Saver.h | 2 +- src/clwrap.cpp | 10 +++++----- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index e6293f91..ff20f0dc 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -581,7 +581,7 @@ Gpu::~Gpu() { // Part of GPU initialization is to compute the default number of registers each kernel should target during compilation. // Kernel register usage is critical for maximizing GPU occupancy. The default values can be overrriden with command line arguments. // This feature currently only works for the CUDA compiler. -string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { +string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { #if CUDA_BACKEND int regs = 0; const char *use_override = ""; @@ -1267,7 +1267,7 @@ void Gpu::replay() { } // Skip other kernels (KFFTW) - else; + else {} // Advance argument index arg = replay_next_arg(kern, arg); @@ -2521,7 +2521,7 @@ double Gpu::timePRP(int quick) { // Quick varies from 1 (slowest, longest return secsPerIt * 1e6; } -PRPResult Gpu::isPrimePRP(const Task& task) { +PRPResult Gpu::isPrimePRP([[maybe_unused]] const Task& task) { assert(E == task.exponent); // This timer is used to measure total elapsed time to be written to the savefile. @@ -2704,7 +2704,7 @@ PRPResult Gpu::isPrimePRP(const Task& task) { } } -LLResult Gpu::isPrimeLL(const Task& task) { +LLResult Gpu::isPrimeLL([[maybe_unused]] const Task& task) { assert(E == task.exponent); wantROE = 0; diff --git a/src/Proof.cpp b/src/Proof.cpp index d10aa8dd..6ad2987b 100644 --- a/src/Proof.cpp +++ b/src/Proof.cpp @@ -254,7 +254,7 @@ u64 ProofSet::next(u64 k) const { return *cacheIt; } -void ProofSet::save(u64 E, u32 power, u64 k, const Words& words) { +void ProofSet::save(u64 E, [[maybe_unused]] u32 power, u64 k, const Words& words) { assert(k && k <= E); assert(isInPoints(E, power, k)); @@ -262,7 +262,7 @@ void ProofSet::save(u64 E, u32 power, u64 k, const Words& words) { assert(load(E, power, k) == words); } -Words ProofSet::load(u64 E, u32 power, u64 k) { +Words ProofSet::load(u64 E, [[maybe_unused]] u32 power, u64 k) { assert(k && k <= E); assert(isInPoints(E, power, k)); return File::openReadThrow(proofPath(E) / to_string(k)).readChecked(E/32 + 1); diff --git a/src/Saver.cpp b/src/Saver.cpp index c9663fb5..dfb2b314 100644 --- a/src/Saver.cpp +++ b/src/Saver.cpp @@ -88,7 +88,7 @@ fs::path findLast(const fs::path& dir, const string& prefix, const string& kind) return path; } -PRPState readState(const PRPState& dummy, File fi) { +PRPState readState([[maybe_unused]] const PRPState& dummy, File fi) { u64 exponent{}, k{}; u32 blockSize{}, nErrors{}; u64 res64{}; @@ -109,7 +109,7 @@ PRPState readState(const PRPState& dummy, File fi) { throw BadHeaderError{fi.name}; } -LLState readState(const LLState& dummy, File fi) { +LLState readState([[maybe_unused]] const LLState& dummy, File fi) { u64 exponent{}, k{}; double elapsed{}; diff --git a/src/Saver.h b/src/Saver.h index 671b7025..07fc44a6 100644 --- a/src/Saver.h +++ b/src/Saver.h @@ -19,7 +19,7 @@ struct PRPState { u64 res64; vector check; u32 nErrors; - double elapsed; + double elapsed{}; }; struct LLState { diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 3305f6c2..70a306c7 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -459,9 +459,9 @@ cl_context getQueueContext(cl_command_queue q) { // The dummy implementation below is for the native openCL builds. #ifndef CUDA_BACKEND -bool clIsGraphSupported(cl_device_id dev) { return 0; } -int clGraphBeginRecording(cl_command_queue q) { return CL_INVALID_VALUE; } -int clGraphEndRecording(cl_command_queue q, cl_graph* g) { return CL_INVALID_VALUE; } -int clGraphLaunch(cl_graph g) { return CL_INVALID_VALUE; } -int clReleaseGraph(cl_graph g) { return CL_INVALID_VALUE; } +bool clIsGraphSupported(cl_device_id) { return 0; } +int clGraphBeginRecording(cl_command_queue) { return CL_INVALID_VALUE; } +int clGraphEndRecording(cl_command_queue, cl_graph*) { return CL_INVALID_VALUE; } +int clGraphLaunch(cl_graph) { return CL_INVALID_VALUE; } +int clReleaseGraph(cl_graph) { return CL_INVALID_VALUE; } #endif From 9d3716a543d846374c62b992cb73b362966de989 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 4 Aug 2026 01:15:45 +0000 Subject: [PATCH 114/214] More fixes for compiler warnings --- src/Task.h | 10 +++++----- src/TuneEntry.cpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Task.h b/src/Task.h index 588acd3a..945172a5 100644 --- a/src/Task.h +++ b/src/Task.h @@ -19,12 +19,12 @@ class Task { enum Kind {PRP, VERIFY, LL, CERT}; Kind kind; - u64 exponent; - string AID; // Assignment ID - string line; // the verbatim worktodo line, used in deleteTask(). - u32 squarings; // For CERTs + u64 exponent{}; + string AID{}; // Assignment ID + string line{}; // the verbatim worktodo line, used in deleteTask(). + u32 squarings{}; // For CERTs + string verifyPath{}; // For Verify - string verifyPath; // For Verify void execute(GpuCommon shared, u32 instance); void writeResultPRP(FFTConfig fft, const Args&, u32 instance, bool isPrime, u64 res64, const std::string& res2048, u32 nErrors, const fs::path& proofPath) const; diff --git a/src/TuneEntry.cpp b/src/TuneEntry.cpp index 1ad420ea..da4e6dad 100644 --- a/src/TuneEntry.cpp +++ b/src/TuneEntry.cpp @@ -10,8 +10,8 @@ bool TuneEntry::update(vector& results) const { u64 const maxExp = fft.maxExp(); [[maybe_unused]] bool didErase = false; - size_t i{}; - for (i = results.size() - 1; i >= 0 && results[i].cost > cost; --i) { + int i{}; + for (i = int(results.size()) - 1; i >= 0 && results[i].cost > cost; --i) { if (results[i].fft.maxExp() <= maxExp) { results.erase(std::next(results.begin(), i)); didErase = true; From 7e2421395846a8f461bcbf6c3fd29977f25aa3d8 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 6 Aug 2026 01:29:38 +0000 Subject: [PATCH 115/214] Fixed another compiler warning. --- src/clwrap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clwrap.cpp b/src/clwrap.cpp index 70a306c7..dcc08f44 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -398,7 +398,7 @@ int getKernelNumArgs(cl_kernel k) { } int getWorkGroupSize(cl_kernel k, cl_device_id device, const char *name) { - size_t size[3]; + size_t size[3]{}; CHECK2(clGetKernelWorkGroupInfo(k, device, CL_KERNEL_COMPILE_WORK_GROUP_SIZE, sizeof(size), &size, nullptr), name); return int(size[0]); } From f5e377c6a2706de44ed9207211acbd57de7b048c Mon Sep 17 00:00:00 2001 From: george Date: Thu, 6 Aug 2026 02:57:45 +0000 Subject: [PATCH 116/214] Slight cleanup to max ROE calculations. --- src/cl/carryutil.cl | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 8c614d2b..9f7aa0b7 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -137,15 +137,15 @@ float OVERLOAD boundCarry(i64 c) { return ldexp(fabs((float) (i32) (c >> 8)), -2 #if STATS || ROE void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *bufROE, u32 posROE, float roundMax) { assert(roundMax >= 0); - - // This barrier may be needed by carryFused because this code does not partition lds memory the same way shufl does - bar(); - // Reduce to a single roundMax value u32 me = get_local_id(0); u32 u32RoundMax = as_uint(roundMax); - while (num_threads > 1) { + + // Reduce to a handful of roundMax values + // We could use shfl_down_sync (and AMD's equivalent) instead of LDS memory once num_threads < WAVEFRONT + // (see https://github.com/mahmoudmaftah/MaxReduction-Cuda/blob/main/code/reduction_benchmarks.cu) + while (num_threads > 8) { // Write roundMax for high half of threads to local memory. Ignore threads not participating in the reduction. - if (num_threads >= WAVEFRONT) bar(); + if (num_threads > WAVEFRONT) bar(); if (me >= num_threads / 2 && me < num_threads) lds[me - num_threads / 2] = u32RoundMax; if (num_threads > WAVEFRONT) { bar(); // work around a weird CUDA NVCC bug where two bar() calls are required???! (Titan V, CUDA 13.0, WMUL=2) @@ -160,20 +160,19 @@ void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *b num_threads /= 2; } -// We could use shfl_down_sync (and AMD's equivalent) instead of LDS memory once num_threads < WAVEFRONT (see https://github.com/mahmoudmaftah/MaxReduction-Cuda/blob/main/code/reduction_benchmarks.cu) -// We could instead write the reduced ROE sequentially to bufROE and then do a max_reduction after last atomic_add - - // Merge this max with others - if (me == 0) { - // The bufROE entry to update is stored in the first bufROE entry. This value used to be passed into carryFused as an argument. - // CUDA graphs don't allow arguments to change. Thus, calculating posROE and storing it in bufROE workd better. + // The bufROE entry to update is stored in the first bufROE entry. This value used to be passed into carryFused as an argument. + // CUDA graphs don't allow arguments to change. Thus, calculating posROE and storing it in bufROE works better. + if (me < num_threads) { posROE = bufROE[0]; atomic_max(bufROE + posROE + 2, u32RoundMax); + // The second bufRoe entry is a count of the number atomic_maxes performed. When the last atomic_max is done, increment posROE and clear the counter. - u32 old_value = atomic_add(bufROE + 1, 1); - if (old_value == num_blocks - 1) { - bufROE[0] = posROE + 1; - bufROE[1] = 0; + if (me == 0) { + u32 old_value = atomic_add(bufROE + 1, 1); + if (old_value == num_blocks - 1) { + bufROE[0] = posROE + 1; + bufROE[1] = 0; + } } } } From 6e38ac2341dfe5e5f272e7edd8ffa3148a201200 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 7 Aug 2026 01:01:40 +0000 Subject: [PATCH 117/214] Latest tweaks to CUDA kernel register usage. --- src/Gpu.cpp | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ff20f0dc..70ea4d14 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -581,6 +581,10 @@ Gpu::~Gpu() { // Part of GPU initialization is to compute the default number of registers each kernel should target during compilation. // Kernel register usage is critical for maximizing GPU occupancy. The default values can be overrriden with command line arguments. // This feature currently only works for the CUDA compiler. +// Most kernels have occupancy limited by register usage. For reference, the following guidelines dictate where an "uptick" in occupancy occurs. +// If kernel threads=256, register crossovers are at 128, 80, 64, 48, 40 +// If kernel threads=128, register crossovers are at 128, 96, 80, 72, 64, 56, 48, 40 +// If kernel threads=64, register crossovers are at 128, 112, 96, 88, 80, 72, 64, 56, 48, 40 string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { #if CUDA_BACKEND int regs = 0; @@ -596,7 +600,7 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { use_override = "REGCF64"; break; case FFT3161: - regs = nW == 8 ? 96 : 64; + regs = nW == 8 ? 96 : 64; // Tested on 4090, nW=8, CUDA 13.0 (88 regs is possible without spilling but is slower) use_override = "REGCF3161"; break; case FFT3261: @@ -650,15 +654,15 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { break; case MIDIN31: // Register usage depends on MIDDLE if (fft.shape.middle == 16) regs = 56; - else if (fft.shape.middle == 8) regs = 44; - else if (fft.shape.middle == 4) regs = 32; + else if (fft.shape.middle == 8) regs = 48; // Tested on 4090, CUDA 13.0 (40 regs is possible without spilling but is not measurably faster) + else if (fft.shape.middle == 4) regs = 32; // Tested on 5070Ti, CUDA 13.2. else regs = -1; use_override = "REGMI31"; break; case MIDIN61: // Register usage depends on MIDDLE if (fft.shape.middle == 16) regs = 96; - else if (fft.shape.middle == 8) regs = 72; - else if (fft.shape.middle == 4) regs = 64; + else if (fft.shape.middle == 8) regs = 64; // Tested on 4090, CUDA 13.0 + else if (fft.shape.middle == 4) regs = -1; // Tested on 5070Ti, CUDA 13.2 (48 regs is possible without spilling but is slower), best is -1. else regs = -1; use_override = "REGMI61"; break; @@ -672,11 +676,12 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { } break; case TAIL31: // Register usage depends on NH (assumes double-wide kernel) - regs = nH == 8 ? 64 : 48; + regs = nH == 8 ? -1 : 48; // Tested on 4090, NH=8, CUDA 13.0. Occupancy is limited by LDS memory use, register usage of 48 is possible, best is 64. + // Tested on 5070Ti, NH=8, CUDA 13.2. Occupancy is limited by LDS memory use, register usage of 56 is possible, best is -1. use_override = "REGTS31"; break; case TAIL61: // Register usage depends on NH (assumes double-wide kernel) - regs = nH == 8 ? 96 : 64; + regs = nH == 8 ? 96 : 64; // Tested on 4090, nH=8, CUDA 13.0 (80 regs is possible without spilling but is slower) use_override = "REGTS61"; break; case MIDOUT: // Register usage depends on MIDDLE and the FFT/NTT type @@ -700,15 +705,16 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { break; case MIDOUT31: // Register usage depends on MIDDLE if (fft.shape.middle == 16) regs = 48; - else if (fft.shape.middle == 8) regs = 40; - else if (fft.shape.middle == 4) regs = 32; + else if (fft.shape.middle == 8) regs = 40; // Tested on 4090, CUDA 13.0 + else if (fft.shape.middle == 4) regs = -1; // Tested on 5070Ti, CUDA 13.2 (32 regs is possible without spilling but is slower), best is -1. else regs = -1; use_override = "REGMO31"; break; case MIDOUT61: // Register usage depends on MIDDLE if (fft.shape.middle == 16) regs = 96; - else if (fft.shape.middle == 8) regs = 64; - else if (fft.shape.middle == 4) regs = 64; + else if (fft.shape.middle == 8) regs = 64; // Tested on 4090, CUDA 13.0, best is 64. + // Tested on 5070Ti, CUDA 13.2, best is 72. + else if (fft.shape.middle == 4) regs = 64; // Tested on 5070Ti, CUDA 13.2 (48 regs is possible without spilling but is slower), best is 64. else regs = -1; use_override = "REGMO61"; break; From 1ef77163246d82389dbef1871a3d021d8430dcc1 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 10 Aug 2026 02:46:08 +0000 Subject: [PATCH 118/214] Wrote code to set CUDA LDS carveout. Did not help on mt laptop GPU. Default is the new code is turned off. --- src/cuda/clwrap_cuda.cpp | 28 ++++++++++++++++++++++++---- src/cuda/tinycuda.h | 1 + 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 20cd3c21..77aef8cb 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -178,9 +178,10 @@ int clReleaseCommandQueue(cl_command_queue q) { // ---- Program compilation (NVRTC) ---- -cl_program clCreateProgramWithSource(cl_context /*ctx*/, unsigned count, const char** strings, +cl_program clCreateProgramWithSource(cl_context ctx, unsigned count, const char** strings, const size_t* lengths, int* err) { auto* prog = new _cl_program; + prog->context = ctx; for (unsigned i = 0; i < count; i++) { if (lengths && lengths[i]) { prog->source.append(strings[i], lengths[i]); @@ -192,11 +193,12 @@ cl_program clCreateProgramWithSource(cl_context /*ctx*/, unsigned count, const return prog; } -cl_program clCreateProgramWithBinary(cl_context /*ctx*/, unsigned /*nDevices*/, const cl_device_id*, +cl_program clCreateProgramWithBinary(cl_context ctx, unsigned /*nDevices*/, const cl_device_id*, const size_t* lengths, const unsigned char** binaries, int* binaryStatus, int* err) { // "Binary" in CUDA land = PTX string auto* prog = new _cl_program; + prog->context = ctx; if (lengths && binaries && lengths[0] > 0) { prog->ptx.assign((const char*)binaries[0], lengths[0]); prog->compiled = true; @@ -418,7 +420,7 @@ int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id return CL_SUCCESS; } -cl_program clLinkProgram(cl_context /*ctx*/, unsigned /*nDevices*/, const cl_device_id*, +cl_program clLinkProgram(cl_context ctx, unsigned /*nDevices*/, const cl_device_id*, const char* /*options*/, unsigned nProgs, const cl_program* progs, void (*)(cl_program, void*), void*, int* err) { ensureContextCurrent(); @@ -430,6 +432,7 @@ cl_program clLinkProgram(cl_context /*ctx*/, unsigned /*nDevices*/, const cl_d } auto* linked = new _cl_program; + linked->context = ctx; linked->ptx = progs[0]->ptx; linked->compiled = true; // Carry preprocessed source through for KERNEL(N) parsing in clCreateKernel @@ -566,7 +569,24 @@ cl_kernel clCreateKernel(cl_program prog, const char* name, int* err) { moduleRetain(k->parentModule); // kernel keeps the module alive past clReleaseProgram // Shared memory carveout: default adaptive carveout is optimal for mixed kernel workloads. - + // Remainder of memory will be used for L1 cache. +// This was not measurably faster on my 570Ti Laptop. We should try it on other GPUs. +// If MULTI_Q is set, we might need to set all the middleIn, tailSquare, and middleOut kernels to use the same carveout value (which +// negates the primary benefit since middleIn and middleOut are the kernels using low carveouts). For now, disable the capability by default. +if (getenv("TRY_LDS_CARVEOUT")) + { + int numRegs = 0, shmem = 0, maxThreads = 0, maxShared = 0; + cuFuncGetAttribute(&numRegs, CU_FUNC_ATTRIBUTE_NUM_REGS, k->func); + cuFuncGetAttribute(&shmem, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, k->func); + cuFuncGetAttribute(&maxThreads, CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, k->func); + cuDeviceGetAttribute(&maxShared, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR, prog->context->dev); + + int max_occupancy = 65536 / (numRegs * maxThreads); // Maximum occupancy due to register pressure + int carveout = 100 * (max_occupancy * shmem) / maxShared; // Percent of shared memory needed for max occupancy + if (carveout > 100) carveout = 100; + cuFuncSetAttribute(k->func, CU_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT, carveout); + } + // Log register and shared memory usage per kernel when PRPLL_DUMP_PTX is set { static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index 0ebb1bf4..c8ff49ef 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -43,6 +43,7 @@ using cl_mem = _cl_mem*; // cl_program: dual-purpose — stores either source string or compiled PTX/module struct _cl_program { + cl_context context; std::string source; // OpenCL source (before NVRTC compilation) std::string preprocessedSource; // CUDA source after preprocessOpenCL (for parsing __launch_bounds__) std::string ptx; // Compiled PTX (after NVRTC compilation) From 14e21600a614676fe06af821ea39d0793907c5f8 Mon Sep 17 00:00:00 2001 From: george Date: Fri, 14 Aug 2026 23:47:56 +0000 Subject: [PATCH 119/214] Fixed NCLOAD TRIG31 macro. Fixed read() and write() routines to use FFTLOAD and FFTSTORE. Removed an extraneous tune of FFT_STORE=4. --- src/cl/base.cl | 69 +++++++++++++++++++++++++++++++++++++++++++------- src/tune.cpp | 2 +- 2 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index d2a5297f..11a48ddf 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -316,7 +316,7 @@ typedef global const float2* BigTabFP32; // // nVidia GPUs have lots of different caching options for loads and stores. // AMD GPUs have have far fewer options for loads and stores. -// These routines let us try the different options. +// These routines and macros let us try the different options. // // Basic load and store. Presumably stored in all caches using a standard LRU algorithm. @@ -404,7 +404,7 @@ void OVERLOAD L2STORE(i32 *mem, i32 val) { #define L2STORE STORE #endif -// Routines for loading data from memory into the L1 and L2 caches, but cache line is marked evict first. +// Routines for loading data from memory into the L1 and L2 caches, but cache line is marked evict first to limit cache pollution. #if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher T2 OVERLOAD EFLOAD(CP(T2) mem) { @@ -517,7 +517,7 @@ GF31 OVERLOAD LULOAD(TrigGF31 mem) { #define LULOAD LOAD #endif -// Routines for loading a read-only and placing it in the non-coherent texture cache. +// Routines for loading a read-only value and placing it in the non-coherent texture cache. #if HAS_PTX >= 500 // Texture cache requires sm_50 support or higher T2 OVERLOAD NCLOAD(Trig mem) { @@ -557,13 +557,60 @@ i32 OVERLOAD NCLOAD(i32 *mem) { } GF31 OVERLOAD NCLOAD(TrigGF31 mem) { GF31 retval; - __asm("ld.global.lu.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + __asm("ld.global.nc.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); return retval; } #else #define NCLOAD LOAD #endif +// Routines for loading data from memory into the L1 and L2 caches. This should be same as the default LOAD macro. + +#if HAS_PTX >= 200 // Cache hints requires sm_20 support or higher +T2 OVERLOAD CALOAD(CP(T2) mem) { + T2 retval; + __asm("ld.global.ca.v2.f64 {%0, %1}, [%2];" : "=d"(retval.x), "=d"(retval.y) : "l"(mem)); + return retval; +} +T OVERLOAD CALOAD(TrigSingle mem) { + T retval; + __asm("ld.global.ca.f64 %0, [%1];" : "=d"(retval) : "l"(mem)); + return retval; +} +F2 OVERLOAD CALOAD(CP(F2) mem) { + F2 retval; + __asm("ld.global.ca.v2.f32 {%0, %1}, [%2];" : "=f"(retval.x), "=f"(retval.y) : "l"(mem)); + return retval; +} +F OVERLOAD CALOAD(TrigSingleFP32 mem) { + F retval; + __asm("ld.global.ca.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); + return retval; +} +i64 OVERLOAD CALOAD(i64 *mem) { + i64 retval; + __asm("ld.global.ca.b64 %0, [%1];" : "=l"(retval) : "l"(mem)); + return retval; +} +GF61 OVERLOAD CALOAD(TrigGF61 mem) { + GF61 retval; + __asm("ld.global.ca.v2.b64 {%0, %1}, [%2];" : "=l"(retval.x), "=l"(retval.y) : "l"(mem)); + return retval; +} +i32 OVERLOAD CALOAD(i32 *mem) { + i32 retval; + __asm("ld.global.ca.b32 %0, [%1];" : "=r"(retval) : "l"(mem)); + return retval; +} +GF31 OVERLOAD CALOAD(TrigGF31 mem) { + GF31 retval; + __asm("ld.global.ca.v2.b32 {%0, %1}, [%2];" : "=r"(retval.x), "=r"(retval.y) : "l"(mem)); + return retval; +} +#else +#define CALOAD LOAD +#endif + // // These macros map various types of data accesses to one of the load/store routines above // @@ -588,6 +635,8 @@ GF31 OVERLOAD NCLOAD(TrigGF31 mem) { #define FFTLOAD EFLOAD #elif FFTLOAD_TYPE == 4 #define FFTLOAD LULOAD +#elif FFTLOAD_TYPE == 5 +#define FFTLOAD NCLOAD #else #define FFTLOAD LOAD #endif @@ -613,6 +662,8 @@ GF31 OVERLOAD NCLOAD(TrigGF31 mem) { #define CSLOAD EFLOAD #elif CSLOAD_TYPE == 4 #define CSLOAD LULOAD +#elif CSLOAD_TYPE == 5 +#define CSLOAD NCLOAD #else #define CSLOAD LOAD #endif @@ -645,7 +696,7 @@ GF31 OVERLOAD NCLOAD(TrigGF31 mem) { #endif // Routines for loading trig data that is used once but is smaller than a cache line. The rest of the cache line will be needed soon. -// If possible, data should saved in L1(?) and L2 caches and perhaps marked evict first. +// If possible, data should be saved in L1(?) and L2 caches and perhaps marked evict first. // TS stands for "Trig Several reuses". #if TSLOAD_TYPE == 1 @@ -695,24 +746,24 @@ void PREFETCHL2(const __global void *addr) { #if FFT_FP64 void OVERLOAD read(u32 WG, u32 N, T2 *u, const global T2 *in, u32 base) { in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } + for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG]); } } void OVERLOAD write(u32 WG, u32 N, T2 *u, global T2 *out, u32 base) { out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } + for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG], u[i]); } } #endif #if FFT_FP32 void OVERLOAD read(u32 WG, u32 N, F2 *u, const global F2 *in, u32 base) { in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } + for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG]); } } void OVERLOAD write(u32 WG, u32 N, F2 *u, global F2 *out, u32 base) { out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } + for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG], u[i]); } } #endif diff --git a/src/tune.cpp b/src/tune.cpp index 3b4ee686..4558c72a 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -614,7 +614,7 @@ void Tune::tune() { u64 const exponent = primes.prevPrime(fft.maxExp()); u32 best_fft_store = 0; double best_cost = -1.0; - for (u32 const fft_store : {0, 1, 2, 3, 4}) { + for (u32 const fft_store : {0, 1, 2, 3}) { if (fft_store >= 2 && (!NVIDIAGPU || NO_ASM)) continue; args->flags["STORES"] = to_string(stores / 10 * 10 + fft_store); double const cost = Gpu::make(exponent, shared, fft, {}, false)->timePRP(quick); From 4dd32c978646e6cdb369717d0a556c7d90e4bae1 Mon Sep 17 00:00:00 2001 From: george Date: Sat, 15 Aug 2026 19:15:35 +0000 Subject: [PATCH 120/214] Created combo datatypes such as T2_GF61. This generate cleaner PTX code than sharing code via typecasting a GF61 to a T2. --- src/cl/base.cl | 72 +++++++------- src/cl/fftbase.cl | 50 ++++------ src/cl/middle.cl | 237 +++++++-------------------------------------- src/cl/tailutil.cl | 184 +++++++++++------------------------ 4 files changed, 144 insertions(+), 399 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 11a48ddf..2059f15a 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -394,6 +394,12 @@ void OVERLOAD L2STORE(P(T2) mem, T2 val) { void OVERLOAD L2STORE(P(F2) mem, F2 val) { __asm("st.global.cg.v2.f32 [%0], {%1, %2};" : : "l"(mem), "f"(val.x), "f"(val.y)); } +void OVERLOAD L2STORE(P(GF61) mem, GF61 val) { + __asm("st.global.cg.v2.b64 [%0], {%1, %2};" : : "l"(mem), "l"(val.x), "l"(val.y)); +} +void OVERLOAD L2STORE(P(GF31) mem, GF31 val) { + __asm("st.global.cg.v2.b32 [%0], {%1, %2};" : : "l"(mem), "r"(val.x), "r"(val.y)); +} void OVERLOAD L2STORE(i64 *mem, i64 val) { __asm("st.global.cg.b64 [%0], %1;" : : "l"(mem), "l"(val)); } @@ -460,6 +466,12 @@ void OVERLOAD EFSTORE(P(T2) mem, T2 val) { void OVERLOAD EFSTORE(P(F2) mem, F2 val) { __asm("st.global.cs.v2.f32 [%0], {%1, %2};" : : "l"(mem), "f"(val.x), "f"(val.y)); } +void OVERLOAD EFSTORE(P(GF61) mem, GF61 val) { + __asm("st.global.cs.v2.b64 [%0], {%1, %2};" : : "l"(mem), "l"(val.x), "l"(val.y)); +} +void OVERLOAD EFSTORE(P(GF31) mem, GF31 val) { + __asm("st.global.cs.v2.b32 [%0], {%1, %2};" : : "l"(mem), "r"(val.x), "r"(val.y)); +} void OVERLOAD EFSTORE(i64 *mem, i64 val) { __asm("st.global.cs.b64 [%0], %1;" : : "l"(mem), "l"(val)); } @@ -743,51 +755,43 @@ void PREFETCHL2(const __global void *addr) { #endif } -#if FFT_FP64 -void OVERLOAD read(u32 WG, u32 N, T2 *u, const global T2 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG]); } -} +// Some routines can be written for any 64-bit data type (T2 or GF61). Same for 32-bit data types (F2 or GF31). +// Some routines can be written to work 32-bit and 64-bit data types. +// These #defines make it easy to write those routines. This used to be done with type-casting, but +// this method generates better PTX code (not sure if that results in any better run times). -void OVERLOAD write(u32 WG, u32 N, T2 *u, global T2 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG], u[i]); } -} +#if FFT_FP64 +#define T_Z61 T +#define T2_GF61 T2 +#define T2_F2_GF31_GF61 T2 +#define as_T2_GF61 as_double2 +#endif +#if NTT_GF61 +#define T_Z61 Z61 +#define T2_GF61 GF61 +#define T2_F2_GF31_GF61 GF61 +#define as_T2_GF61 as_ulong2 #endif - #if FFT_FP32 -void OVERLOAD read(u32 WG, u32 N, F2 *u, const global F2 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG]); } -} - -void OVERLOAD write(u32 WG, u32 N, F2 *u, global F2 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG], u[i]); } -} +#define F_Z31 F +#define F2_GF31 F2 +#define T2_F2_GF31_GF61 F2 #endif - #if NTT_GF31 -void OVERLOAD read(u32 WG, u32 N, GF31 *u, const global GF31 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } -} - -void OVERLOAD write(u32 WG, u32 N, GF31 *u, global GF31 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } -} +#define F_Z31 Z31 +#define F2_GF31 GF31 +#define T2_F2_GF31_GF61 GF31 #endif -#if NTT_GF61 -void OVERLOAD read(u32 WG, u32 N, GF61 *u, const global GF61 *in, u32 base) { +#if FFT_FP64 || NTT_GF61 || FFT_FP32 || NTT_GF31 +void OVERLOAD read(u32 WG, u32 N, T2_F2_GF31_GF61 *u, const global T2_F2_GF31_GF61 *in, u32 base) { in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = in[i * WG]; } + for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG]); } } -void OVERLOAD write(u32 WG, u32 N, GF61 *u, global GF61 *out, u32 base) { +void OVERLOAD write(u32 WG, u32 N, T2_F2_GF31_GF61 *u, global T2_F2_GF31_GF61 *out, u32 base) { out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { out[i * WG] = u[i]; } + for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG], u[i]); } } #endif diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 9416c76b..19bdb196 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -20,7 +20,7 @@ #define LDS_BYTES (WG * RADIX * SHUFL_BYTES) #endif -#if FFT_FP64 | NTT_GF61 +#if FFT_FP64 || NTT_GF61 // Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. // Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- @@ -33,7 +33,7 @@ // before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and // only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively // minor optimization would be to special case the first shufl call. -void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { +void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { u32 mask = f - 1; assert((mask & (mask + 1)) == 0); @@ -48,8 +48,8 @@ void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. if (SHUFL_BYTES == 16) { - local T2* lds = ((local T2*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); + local T2_GF61* lds = lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); #if LDSPAD // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. @@ -170,8 +170,8 @@ void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. else if (SHUFL_BYTES == 8) { - local T* lds = ((local T*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T); + local T_Z61* lds = ((local T_Z61*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); #if LDSPAD // Special case first n == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. @@ -352,29 +352,29 @@ void OVERLOAD shufl64(local T2 *lds2, T2 *u, u32 f, u32 numWG, u32 lowMe) { bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } } } #endif -#if FFT_FP32 | NTT_GF31 +#if FFT_FP32 || NTT_GF31 -// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats. -void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats or Z31s. +void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { u32 mask = f - 1; assert((mask & (mask + 1)) == 0); @@ -391,8 +391,8 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. if (SHUFL_BYTES >= 8) { - local F2* lds = ((local F2*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); + local F2_GF31* lds = lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); #if LDSPAD // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. @@ -520,8 +520,8 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. else if (SHUFL_BYTES == 4) { - local F* lds = ((local F*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F); + local F_Z31* lds = ((local F_Z31*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); #if LDSPAD // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. @@ -621,10 +621,6 @@ void OVERLOAD shufl32(local F2 *lds2, F2 *u, u32 f, u32 numWG, u32 lowMe) { #if FFT_FP64 -void OVERLOAD shufl(local T2 *lds, T2 *u, u32 f, u32 numWG, u32 lowMe) { - shufl64(lds, u, f, numWG, lowMe); -} - void OVERLOAD chainMul4(T2 *u, T2 w) { u[1] = cmul(u[1], w); @@ -1195,10 +1191,6 @@ void OVERLOAD fft_common(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 l #if FFT_FP32 -void OVERLOAD shufl(local F2 *lds, F2 *u, u32 f, u32 numWG, u32 lowMe) { - shufl32(lds, u, f, numWG, lowMe); -} - void OVERLOAD fft_RADIX(F2 *u) { #if RADIX == 4 fft4(u); @@ -1658,10 +1650,6 @@ void OVERLOAD fft_common(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 low #if NTT_GF31 -void OVERLOAD shufl(local GF31 *lds, GF31 *u, u32 f, u32 numWG, u32 lowMe) { - shufl32((local F2 *) lds, (F2 *) u, f, numWG, lowMe); -} - void OVERLOAD fft_RADIX(GF31 *u) { #if RADIX == 4 fft4(u); @@ -1744,10 +1732,6 @@ void OVERLOAD fft_common(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 #if NTT_GF61 -void OVERLOAD shufl(local GF61 *lds, GF61 *u, u32 f, u32 numWG, u32 lowMe) { - shufl64((local T2 *) lds, (T2 *) u, f, numWG, lowMe); -} - void OVERLOAD fft_RADIX(GF61 *u) { #if RADIX == 4 fft4(u); diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 35a251cf..370850d1 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -54,7 +54,7 @@ // u[i] i ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...SMALL_HEIGHT-1 (multiples of one) -void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { +void OVERLOAD writeCarryFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines @@ -65,7 +65,7 @@ void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { #endif } -void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { #if PAD_SIZE > 0 // Each work group reads successive y's which increments by one pad size. // Rather than having u[i] also increment by one, we choose a larger pad increment @@ -90,7 +90,7 @@ void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { // x ranges 0...SMALL_HEIGHT-1 (multiples of one) (also known as 0...G_H-1 and 0...NH-1) // y ranges 0...MIDDLE*WIDTH-1 (multiples of SMALL_HEIGHT) -void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleInLine (P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) { //u32 SIZEY = IN_WG / IN_SIZEX; //u32 num_x_chunks = WIDTH / IN_SIZEX; // Number of x chunks @@ -125,7 +125,7 @@ void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) // Read a line for tailFused or fftHin // This reads partially transposed data as written by fftMiddleIn -void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 SIZEY = IN_WG / IN_SIZEX; #if PAD_SIZE > 0 @@ -200,7 +200,7 @@ void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { // i in u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...WIDTH-1 (multiples of BIG_HEIGHT) (processed in batches of OUT_WG/OUT_SIZEX) -void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; @@ -215,7 +215,7 @@ void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { #endif } -void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 // Each u[i] increments by one pad size. @@ -278,7 +278,7 @@ void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { // adjusted to effect a transpose. Or caller must transpose the x and y values and send us an out pointer with thread_id added in. // In other words, caller is responsible for deciding the best way to transpose x and y values. -void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleOutLine (P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) { //u32 SIZEY = OUT_WG / OUT_SIZEX; //u32 num_x_chunks = SMALL_HEIGHT / OUT_SIZEX; // Number of x chunks @@ -311,7 +311,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 chunk_y, u32 chunk_x) } // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { +void OVERLOAD readCarryFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 SIZEY = OUT_WG / OUT_SIZEX; #if PAD_SIZE > 0 @@ -375,12 +375,12 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { /**************************************************************************/ -/* Similar to above, but for an FFT based on FP32 */ +/* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ #if FFT_FP32 || NTT_GF31 -void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { +void OVERLOAD writeCarryFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { #if PAD_SIZE > 0 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; out += line * WIDTH + line * PAD_SIZE + line / SMALL_HEIGHT * BIG_PAD_SIZE + me; // One pad every line + a big pad every SMALL_HEIGHT lines @@ -391,7 +391,7 @@ void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { #endif } -void OVERLOAD readMiddleInLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { #if PAD_SIZE > 0 // Each work group reads successive y's which increments by one pad size. // Rather than having u[i] also increment by one, we choose a larger pad increment @@ -404,7 +404,7 @@ void OVERLOAD readMiddleInLine(F2 *u, CP(F2) in, u32 y, u32 x) { #endif } -void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleInLine (P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) { #if PAD_SIZE > 0 u32 SIZEY = IN_WG / IN_SIZEX; @@ -431,7 +431,7 @@ void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) // Read a line for tailFused or fftHin // This reads partially transposed data as written by fftMiddleIn -void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 SIZEY = IN_WG / IN_SIZEX; #if PAD_SIZE > 0 // Adjust in pointer based on the x value used in writeMiddleInLine @@ -480,7 +480,7 @@ void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { #endif } -void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 u32 BIG_PAD_SIZE = (PAD_SIZE/2+1)*PAD_SIZE; @@ -495,7 +495,7 @@ void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { #endif } -void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { #if PAD_SIZE > 0 #if MIDDLE == 4 || MIDDLE == 8 || MIDDLE == 16 // Each u[i] increments by one pad size. @@ -512,7 +512,7 @@ void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { #endif } -void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleOutLine (P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) { #if PAD_SIZE > 0 u32 SIZEY = OUT_WG / OUT_SIZEX; @@ -536,7 +536,7 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 chunk_y, u32 chunk_x) #endif } -void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { +void OVERLOAD readCarryFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 SIZEY = OUT_WG / OUT_SIZEX; #if PAD_SIZE > 0 // Adjust in pointer based on the x value used in writeMiddleOutLine @@ -585,90 +585,6 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { #endif -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M31^2) */ -/**************************************************************************/ - -#if NTT_GF31 - -// Since F2 and GF31 are the same size we can simply call the floats based code - -void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { - writeCarryFusedLine((F2 *) u, (P(F2)) out, line, me); -} - -void OVERLOAD readMiddleInLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleInLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF31) out, GF31 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleInLine ((P(F2)) out, (F2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readTailFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { - readTailFusedLine((CP(F2)) in, (F2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { - writeTailFusedLine((F2 *) u, (P(F2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleOutLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF31) out, GF31 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleOutLine ((P(F2)) out, (F2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { - readCarryFusedLine((CP(F2)) in, (F2 *) u, line, me); -} - -#endif - - -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M61^2) */ -/**************************************************************************/ - -#if NTT_GF61 - -// Since T2 and GF61 are the same size we can simply call the doubles based code - -void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { - writeCarryFusedLine((T2 *) u, (P(T2)) out, line, me); -} - -void OVERLOAD readMiddleInLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleInLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF61) out, GF61 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleInLine ((P(T2)) out, (T2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readTailFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { - readTailFusedLine((CP(T2)) in, (T2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { - writeTailFusedLine((T2 *) u, (P(T2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleOutLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF61) out, GF61 *u, u32 chunk_y, u32 chunk_x) { - writeMiddleOutLine ((P(T2)) out, (T2 *) u, chunk_y, chunk_x); -} - -void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { - readCarryFusedLine((CP(T2)) in, (T2 *) u, line, me); -} - -#endif @@ -776,7 +692,7 @@ void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { // line ranges 0...BIG_HEIGHT-1 (multiples of one) // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { +void OVERLOAD readCarryFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); @@ -784,7 +700,7 @@ void OVERLOAD readCarryFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { } // Write a line from carryFused. This data will be read by fftMiddleIn. -void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT +void OVERLOAD writeCarryFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW) + (middle * SIZEM) + (line % 16 * SIZEBLK) + SWIZ(line % 16, line / 16) * 16 + (me % 16); @@ -799,13 +715,13 @@ void OVERLOAD writeCarryFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { // me i // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...SMALL_HEIGHT-1 (multiples of one) -void OVERLOAD readMiddleInLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { in += (x / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, y / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM]); } } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. -void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleInLine (P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) { out += (x / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, y / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } @@ -820,14 +736,14 @@ void OVERLOAD writeMiddleInLine (P(T2) out, T2 *u, u32 y, u32 x) // line ranges 0...MIDDLE*WIDTH-1 (multiples of SMALL_HEIGHT) // Read a line for tailSquare/Mul or fftHin -void OVERLOAD readTailFusedLine(CP(T2) in, T2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT in += (width / 16 * SIZEW) + (middle * SIZEM) + (width % 16 * SIZEBLK) + (me % 16); for (i32 i = 0; i < NH; ++i) { u[i] = FFTLOAD(&in[SWIZ(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } } -void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(T2_GF61 *u, P(T2_GF61) out, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT out += (width / 16 * SIZEW) + (middle * SIZEM) + (width % 16 * SIZEBLK) + (me % 16); @@ -842,13 +758,13 @@ void OVERLOAD writeTailFusedLine(T2 *u, P(T2) out, u32 line, u32 me) { // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...WIDTH-1 (multiples of BIG_HEIGHT) -void OVERLOAD readMiddleOutLine(T2 *u, CP(T2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { in += (y / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, x / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM]); } } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. -void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleOutLine (P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) { out += (y / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, x / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } @@ -858,7 +774,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) /**************************************************************************/ -/* Similar to above, but for an FFT based on FP32 */ +/* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ #if FFT_FP32 || NTT_GF31 @@ -895,7 +811,7 @@ void OVERLOAD writeMiddleOutLine (P(T2) out, T2 *u, u32 y, u32 x) // line ranges 0...BIG_HEIGHT-1 (multiples of one) // Read a line for carryFused or FFTW. This line was written by writeMiddleOutLine above. -void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { +void OVERLOAD readCarryFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one in += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); @@ -903,7 +819,7 @@ void OVERLOAD readCarryFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { } // Write a line from carryFused. This data will be read by fftMiddleIn. -void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT +void OVERLOAD writeCarryFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { // me is multiples of BIG_HEIGHT u32 middle = line / SMALL_HEIGHT; // Multiples of SMALL_HEIGHT line = line % SMALL_HEIGHT; // Multiples of one out += (me / 16 * SIZEW32) + (middle * SIZEM32) + (line % 16 * SIZEBLK32) + SWIZ32(line % 16, line / 16) * 16 + (me % 16); @@ -918,13 +834,13 @@ void OVERLOAD writeCarryFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { // me // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...SMALL_HEIGHT-1 (multiples of one) -void OVERLOAD readMiddleInLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleInLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { in += (x / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, y / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM32]); } } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. -void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleInLine (P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) { out += (x / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, y / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } @@ -939,14 +855,14 @@ void OVERLOAD writeMiddleInLine (P(F2) out, F2 *u, u32 y, u32 x) // line ranges 0...MIDDLE*WIDTH-1 (multiples of SMALL_HEIGHT) // Read a line for tailSquare/Mul or fftHin -void OVERLOAD readTailFusedLine(CP(F2) in, F2 *u, u32 line, u32 me) { +void OVERLOAD readTailFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT in += (width / 16 * SIZEW32) + (middle * SIZEM32) + (width % 16 * SIZEBLK32) + (me % 16); for (i32 i = 0; i < NH; ++i) { u[i] = FFTLOAD(&in[SWIZ32(width % 16, (i * SMALL_HEIGHT / NH + me) / 16) * 16]); } } -void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { +void OVERLOAD writeTailFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { u32 width = line % WIDTH; // Multiples of BIG_HEIGHT u32 middle = line / WIDTH; // Multiples of SMALL_HEIGHT out += (width / 16 * SIZEW32) + (middle * SIZEM32) + (width % 16 * SIZEBLK32) + (me % 16); @@ -961,13 +877,13 @@ void OVERLOAD writeTailFusedLine(F2 *u, P(F2) out, u32 line, u32 me) { // u[i] ranges 0...MIDDLE-1 (multiples of SMALL_HEIGHT) // y ranges 0...WIDTH-1 (multiples of BIG_HEIGHT) -void OVERLOAD readMiddleOutLine(F2 *u, CP(F2) in, u32 y, u32 x) { +void OVERLOAD readMiddleOutLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { in += (y / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, x / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { u[i] = FFTLOAD(&in[i * SIZEM32]); } } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. -void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 y, u32 x) +void OVERLOAD writeMiddleOutLine (P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) { out += (y / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, x / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } @@ -976,89 +892,4 @@ void OVERLOAD writeMiddleOutLine (P(F2) out, F2 *u, u32 y, u32 x) #endif -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M31^2) */ -/**************************************************************************/ - -#if NTT_GF31 - -// Since F2 and GF31 are the same size we can simply call the floats based code - -void OVERLOAD readCarryFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { - readCarryFusedLine((CP(F2)) in, (F2 *) u, line, me); -} - -void OVERLOAD writeCarryFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { - writeCarryFusedLine((F2 *) u, (P(F2)) out, line, me); -} - -void OVERLOAD readMiddleInLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleInLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF31) out, GF31 *u, u32 y, u32 x) { - writeMiddleInLine ((P(F2)) out, (F2 *) u, y, x); -} - -void OVERLOAD readTailFusedLine(CP(GF31) in, GF31 *u, u32 line, u32 me) { - readTailFusedLine((CP(F2)) in, (F2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF31 *u, P(GF31) out, u32 line, u32 me) { - writeTailFusedLine((F2 *) u, (P(F2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF31 *u, CP(GF31) in, u32 y, u32 x) { - readMiddleOutLine((F2 *) u, (CP(F2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF31) out, GF31 *u, u32 y, u32 x) { - writeMiddleOutLine ((P(F2)) out, (F2 *) u, y, x); -} - -#endif - - -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M61^2) */ -/**************************************************************************/ - -#if NTT_GF61 - -// Since T2 and GF61 are the same size we can simply call the doubles based code - -void OVERLOAD readCarryFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { - readCarryFusedLine((CP(T2)) in, (T2 *) u, line, me); -} - -void OVERLOAD writeCarryFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { - writeCarryFusedLine((T2 *) u, (P(T2)) out, line, me); -} - -void OVERLOAD readMiddleInLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleInLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleInLine (P(GF61) out, GF61 *u, u32 y, u32 x) { - writeMiddleInLine ((P(T2)) out, (T2 *) u, y, x); -} - -void OVERLOAD readTailFusedLine(CP(GF61) in, GF61 *u, u32 line, u32 me) { - readTailFusedLine((CP(T2)) in, (T2 *) u, line, me); -} - -void OVERLOAD writeTailFusedLine(GF61 *u, P(GF61) out, u32 line, u32 me) { - writeTailFusedLine((T2 *) u, (P(T2)) out, line, me); -} - -void OVERLOAD readMiddleOutLine(GF61 *u, CP(GF61) in, u32 y, u32 x) { - readMiddleOutLine((T2 *) u, (CP(T2)) in, y, x); -} - -void OVERLOAD writeMiddleOutLine (P(GF61) out, GF61 *u, u32 y, u32 x) { - writeMiddleOutLine ((P(T2)) out, (T2 *) u, y, x); -} - -#endif - #endif diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index 1af416e7..da79b6f7 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -30,14 +30,14 @@ // 64-bit implementations of reverse routines -#if FFT_FP64 | NTT_GF61 +#if FFT_FP64 || NTT_GF61 -void OVERLOAD reverse(local T2 *lds2, T2 *u, bool bump) { +void OVERLOAD reverse(local T2_GF61 *lds2, T2_GF61 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; if (SHUFL_BYTES_H >= 8) { - local T2 *lds = lds2; + local T2_GF61 *lds = lds2; bar(WG); #if NH == 8 lds[revMe + 0 * WG] = u[3]; @@ -53,7 +53,7 @@ void OVERLOAD reverse(local T2 *lds2, T2 *u, bool bump) { } else if (SHUFL_BYTES_H == 4) { - local T *lds = (local T *) lds2; + local T_Z61 *lds = (local T_Z61 *) lds2; bar(WG); #if NH == 8 lds[revMe + 0 * WG] = u[3].x; @@ -81,13 +81,13 @@ void OVERLOAD reverse(local T2 *lds2, T2 *u, bool bump) { } } -void OVERLOAD reverseLine(local T2 *lds, T2 *u) { +void OVERLOAD reverseLine(local T2_GF61 *lds, T2_GF61 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; if (SHUFL_BYTES_H == 16) { - local T2 *ldsOut = lds + revMe; - local T2 *ldsIn = lds + me; + local T2_GF61 *ldsOut = lds + revMe; + local T2_GF61 *ldsIn = lds + me; bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i]; } bar(WG); @@ -95,8 +95,8 @@ void OVERLOAD reverseLine(local T2 *lds, T2 *u) { } else if (SHUFL_BYTES_H == 8) { - local T *ldsOut = (local T *) lds + revMe; - local T *ldsIn = (local T *) lds + me; + local T_Z61 *ldsOut = (local T_Z61 *) lds + revMe; + local T_Z61 *ldsIn = (local T_Z61 *) lds + me; bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].x; } bar(WG); @@ -113,19 +113,19 @@ void OVERLOAD reverseLine(local T2 *lds, T2 *u) { bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).x; } bar(WG); - for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.x = ldsIn[WG * i]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.x = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).y; } bar(WG); - for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.y = ldsIn[WG * i]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.y = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).z; } bar(WG); - for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.z = ldsIn[WG * i]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.z = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = as_int4(u[i]).w; } bar(WG); - for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.w = ldsIn[WG * i]; u[i] = as_double2(tmp); } + for (u32 i = 0; i < NH; ++i) { int4 tmp = as_int4(u[i]); tmp.w = ldsIn[WG * i]; u[i] = as_T2_GF61(tmp); } } } @@ -133,13 +133,13 @@ void OVERLOAD reverseLine(local T2 *lds, T2 *u) { // These versions are for the kernel(s) that use a double-wide workgroup (u in half the workgroup, v in the other half) // -void OVERLOAD reverse2(local T2 *lds2, T2 *u) { +void OVERLOAD reverse2(local T2_GF61 *lds2, T2_GF61 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; if (SHUFL_BYTES_H >= 8) { - local T2 *lds = lds2; - if (me >= WG) lds += LDS_BYTES / sizeof(T2); + local T2_GF61 *lds = lds2; + if (me >= WG) lds += LDS_BYTES / sizeof(T2_GF61); // For NH=8, u[0] to u[3] are left unchanged. Write to lds: // u[7]rev u[6]rev u[5]rev u[4]rev // v[7]rev v[6]rev v[5]rev v[4]rev @@ -155,8 +155,8 @@ void OVERLOAD reverse2(local T2 *lds2, T2 *u) { } else if (SHUFL_BYTES_H == 4) { - local T *lds = (local T *) lds2; - if (me >= WG) lds += LDS_BYTES / sizeof(T); + local T_Z61 *lds = (local T_Z61 *) lds2; + if (me >= WG) lds += LDS_BYTES / sizeof(T_Z61); bar(WG); for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i].x; } bar(WG); @@ -169,16 +169,16 @@ void OVERLOAD reverse2(local T2 *lds2, T2 *u) { } // This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(local T2* lds2, T2 *u) { +void OVERLOAD revCrossLine(local T2_GF61 *lds2, T2_GF61 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; u32 revLowMe = WG - 1 - lowMe; if (SHUFL_BYTES_H >= 8) { - local T2 *ldsOut = lds2; - local T2 *ldsIn = lds2; - if (me < WG) ldsOut += LDS_BYTES / sizeof(T2); // Crossing LDS halves - else ldsIn += LDS_BYTES / sizeof(T2); // Staying within LDS halves (just like shufl) + local T2_GF61 *ldsOut = lds2; + local T2_GF61 *ldsIn = lds2; + if (me < WG) ldsOut += LDS_BYTES / sizeof(T2_GF61); // Crossing LDS halves + else ldsIn += LDS_BYTES / sizeof(T2_GF61); // Staying within LDS halves (just like shufl) bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. @@ -186,10 +186,10 @@ void OVERLOAD revCrossLine(local T2* lds2, T2 *u) { } else if (SHUFL_BYTES_H == 4) { - local T *ldsOut = (local T *) lds2; - local T *ldsIn = (local T *) lds2; - if (me < WG) ldsOut += LDS_BYTES / sizeof(T); - else ldsIn += LDS_BYTES / sizeof(T); + local T_Z61 *ldsOut = (local T_Z61 *) lds2; + local T_Z61 *ldsIn = (local T_Z61 *) lds2; + if (me < WG) ldsOut += LDS_BYTES / sizeof(T_Z61); + else ldsIn += LDS_BYTES / sizeof(T_Z61); bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].x; } bar(); // we need a full bar because we just crossed halves @@ -211,7 +211,7 @@ void OVERLOAD revCrossLine(local T2* lds2, T2 *u) { // u[2] u[3] // Returned in u[1] // v[3]rev v[2]rev // Returned in u[2] // v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local T2 *lds, T2 *u) { +void OVERLOAD reverseLine2(local T2_GF61 *lds, T2_GF61 *u) { u32 me = get_local_id(0); // NOTE: It is important that this routine use lds memory in coordination with shufl. Failure to do so would require an @@ -228,7 +228,7 @@ void OVERLOAD reverseLine2(local T2 *lds, T2 *u) { // That means saving to lds using index: me < WG ? me % WG + i * WG : 8*WG-1 - me % WG - i * WG #if 1 - local T2 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + local T2_GF61 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } @@ -236,18 +236,18 @@ void OVERLOAD reverseLine2(local T2 *lds, T2 *u) { bar(); for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*WG]; } #else - local T *ldsOut = (local T *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + local T_Z61 *ldsOut = (local T_Z61 *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*WG] = u[i].y; } - local T *ldsIn = (local T *) lds + me; + local T_Z61 *ldsIn = (local T_Z61 *) lds + me; bar(); for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*WG]; u[i].y = ldsIn[NH*2*WG + i * 2*WG]; } #endif } // Undo a reverseLine2 -void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { +void OVERLOAD unreverseLine2(local T2_GF61 *lds, T2_GF61 *u) { u32 me = get_local_id(0); // NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl. By initially @@ -256,7 +256,7 @@ void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { // lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. #if 1 - local T2 *ldsOut = lds + me; + local T2_GF61 *ldsOut = lds + me; for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i]; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: @@ -269,7 +269,7 @@ void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { bar(); for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } #else - local T *ldsOut = (local T *) lds + me; + local T_Z61 *ldsOut = (local T_Z61 *) lds + me; for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i].x; ldsOut[NH*2*WG + i * 2*WG] = u[i].y; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: @@ -277,7 +277,7 @@ void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { // 0..GH-1 +1*WG GH-1..0 +6*WG // 0..GH-1 +2*WG GH-1..0 +5*WG // 0..GH-1 +3*WG GH-1..0 +4*WG - local T *ldsIn = (local T *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); + local T_Z61 *ldsIn = (local T_Z61 *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); i32 ldsInc = (me < WG) ? WG : -WG; bar(); for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*WG]; } @@ -290,12 +290,12 @@ void OVERLOAD unreverseLine2(local T2 *lds, T2 *u) { /**************************************************************************/ -/* Similar to above, but for an FFT based on FP32 */ +/* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ -#if FFT_FP32 | NTT_GF31 +#if FFT_FP32 || NTT_GF31 -void OVERLOAD reverse(local F2 *lds, F2 *u, bool bump) { +void OVERLOAD reverse(local F2_GF31 *lds, F2_GF31 *u, bool bump) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me + bump; @@ -315,13 +315,13 @@ void OVERLOAD reverse(local F2 *lds, F2 *u, bool bump) { } } -void OVERLOAD reverseLine(local F2 *lds, F2 *u) { +void OVERLOAD reverseLine(local F2_GF31 *lds, F2_GF31 *u) { u32 me = get_local_id(0); u32 revMe = WG - 1 - me; if (SHUFL_BYTES_H >= 8) { - local F2 *ldsOut = lds + revMe; - local F2 *ldsIn = lds + me; + local F2_GF31 *ldsOut = lds + revMe; + local F2_GF31 *ldsIn = lds + me; bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i]; } bar(WG); @@ -329,8 +329,8 @@ void OVERLOAD reverseLine(local F2 *lds, F2 *u) { } else if (SHUFL_BYTES_H == 4) { - local F *ldsOut = (local F *) lds + revMe; - local F *ldsIn = (local F *) lds + me; + local F_Z31 *ldsOut = (local F_Z31 *) lds + revMe; + local F_Z31 *ldsIn = (local F_Z31 *) lds + me; bar(WG); for (u32 i = 0; i < NH; ++i) { ldsOut[WG * (NH - 1 - i)] = u[i].x; } bar(WG); @@ -346,7 +346,7 @@ void OVERLOAD reverseLine(local F2 *lds, F2 *u) { // These versions are for the kernel(s) that use a double-wide workgroup (u in half the workgroup, v in the other half) // -void OVERLOAD reverse2(local F2 *lds, F2 *u) { +void OVERLOAD reverse2(local F2_GF31 *lds, F2_GF31 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; @@ -368,14 +368,14 @@ void OVERLOAD reverse2(local F2 *lds, F2 *u) { } // This is used to reverse the second part of a line, and cross the reversed parts between the halves. -void OVERLOAD revCrossLine(local F2* lds2, F2 *u) { +void OVERLOAD revCrossLine(local F2_GF31 *lds2, F2_GF31 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; u32 revLowMe = WG - 1 - lowMe; if (SHUFL_BYTES_H >= 4) { - local F2 *ldsOut = lds2; - local F2 *ldsIn = lds2; + local F2_GF31 *ldsOut = lds2; + local F2_GF31 *ldsIn = lds2; if (me < WG) ldsOut += LDS_BYTES / sizeof(F2); else ldsIn += LDS_BYTES / sizeof(F2); bar(); // we need a full bar because we're crossing halves @@ -395,7 +395,7 @@ void OVERLOAD revCrossLine(local F2* lds2, F2 *u) { // u[2] u[3] // Returned in u[1] // v[3]rev v[2]rev // Returned in u[2] // v[1]rev v[0]rev // Returned in u[3] -void OVERLOAD reverseLine2(local F2 *lds, F2 *u) { +void OVERLOAD reverseLine2(local F2_GF31 *lds, F2_GF31 *u) { u32 me = get_local_id(0); // NOTE: It is important that this routine use lds memory in coordination with shufl. Failure to do so would require an @@ -412,7 +412,7 @@ void OVERLOAD reverseLine2(local F2 *lds, F2 *u) { // That means saving to lds using index: me < WG ? me % WG + i * WG : 8*WG-1 - me % WG - i * WG #if 1 - local F2 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + local F2_GF31 *ldsOut = lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { *ldsOut = u[i]; } @@ -420,18 +420,18 @@ void OVERLOAD reverseLine2(local F2 *lds, F2 *u) { bar(); for (u32 i = 0; i < NH; ++i) { u[i] = lds[i * 2*WG]; } #else - local F *ldsOut = (local F *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); + local F_Z31 *ldsOut = (local F_Z31 *) lds + (me < WG ? me % WG : (NH*2)*WG-1 - me % WG); i32 ldsOutInc = (me < WG) ? WG : -WG; for (u32 i = 0; i < NH; ++i, ldsOut += ldsOutInc) { ldsOut[0] = u[i].x; ldsOut[NH*2*WG] = u[i].y; } - local F *ldsIn = (local F *) lds + me; + local F_Z31 *ldsIn = (local F_Z31 *) lds + me; bar(); for (u32 i = 0; i < NH; ++i) { u[i].x = ldsIn[i * 2*WG]; u[i].y = ldsIn[NH*2*WG + i * 2*WG]; } #endif } // Undo a reverseLine2 -void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { +void OVERLOAD unreverseLine2(local F2_GF31 *lds, F2_GF31 *u) { u32 me = get_local_id(0); // NOTE: It is important that this routine use lds memory in coordination with reverseLine2 and shufl. By initially @@ -440,7 +440,7 @@ void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { // lds memory) we can issue a qualified bar() call before calling FFT_HEIGHT2. #if 1 - local F2 *ldsOut = lds + me; + local F2_GF31 *ldsOut = lds + me; for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i]; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: @@ -453,7 +453,7 @@ void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { bar(); for (u32 i = 0; i < NH; ++i, lds += ldsInc) { u[i] = *lds; } #else - local F *ldsOut = (local F *) lds + me; + local F_Z31 *ldsOut = (local F_Z31 *) lds + me; for (u32 i = 0; i < NH; ++i) { ldsOut[i * 2*WG] = u[i].x; ldsOut[NH*2*WG + i * 2*WG] = u[i].y; } // For NH=4, the lds indices (where to read each outgoing u[i] which has v[i] in the upper threads) looks like this: @@ -461,7 +461,7 @@ void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { // 0..GH-1 +1*WG GH-1..0 +6*WG // 0..GH-1 +2*WG GH-1..0 +5*WG // 0..GH-1 +3*WG GH-1..0 +4*WG - local F *ldsIn = (local F *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); + local F_Z31 *ldsIn = (local F_Z31 *) lds + ((me < WG) ? me % WG : (NH*2)*WG-1 - me % WG); i32 ldsInc = (me < WG) ? WG : -WG; bar(); for (u32 i = 0; i < NH; ++i, ldsIn += ldsInc) { u[i].x = ldsIn[0]; u[i].y = ldsIn[NH*2*WG]; } @@ -471,77 +471,3 @@ void OVERLOAD unreverseLine2(local F2 *lds, F2 *u) { #endif #endif - - -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M31^2) */ -/**************************************************************************/ - -#if NTT_GF31 - -void OVERLOAD reverse(local GF31 *lds, GF31 *u, bool bump) { - reverse((local F2 *) lds, (F2 *) u, bump); -} - -void OVERLOAD reverseLine(local GF31 *lds, GF31 *u) { - reverseLine((local F2 *) lds, (F2 *) u); -} - -void OVERLOAD reverse2(local GF31 *lds, GF31 *u) { - reverse2((local F2 *) lds, (F2 *) u); -} - -void OVERLOAD revCrossLine(local GF31* lds, GF31 *u) { - revCrossLine((local F2 *) lds, (F2 *) u); -} - -#if 0 // Unused - -void OVERLOAD reverseLine2(local GF31 *lds, GF31 *u) { - reverseLine2((local F2 *) lds, (F2 *) u); -} - -void OVERLOAD unreverseLine2(local GF31 *lds, GF31 *u) { - unreverseLine2((local F2 *) lds, (F2 *) u); -} - -#endif - -#endif - - -/**************************************************************************/ -/* Similar to above, but for an NTT based on GF(M61^2) */ -/**************************************************************************/ - -#if NTT_GF61 - -void OVERLOAD reverse(local GF61 *lds, GF61 *u, bool bump) { - reverse((local T2 *) lds, (T2 *) u, bump); -} - -void OVERLOAD reverseLine(local GF61 *lds, GF61 *u) { - reverseLine((local T2 *) lds, (T2 *) u); -} - -void OVERLOAD reverse2(local GF61 *lds, GF61 *u) { - reverse2((local T2 *) lds, (T2 *) u); -} - -void OVERLOAD revCrossLine(local GF61* lds, GF61 *u) { - revCrossLine((local T2 *) lds, (T2 *) u); -} - -#if 0 // Unused - -void OVERLOAD reverseLine2(local GF61 *lds, GF61 *u) { - reverseLine2((local T2 *) lds, (T2 *) u); -} - -void OVERLOAD unreverseLine2(local GF61 *lds, GF61 *u) { - unreverseLine2((local T2 *) lds, (T2 *) u); -} - -#endif - -#endif From 0194d2ef0e7a3bc2691e8aa7fb47534e7000fe40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E7=A6=8F=E3=81=A1=E3=82=83=E3=82=93?= Date: Tue, 18 Aug 2026 00:36:14 +0900 Subject: [PATCH 121/214] OpenCL: rename the Z61 fma() overload so it stops hiding the builtin Declaring a user overload named fma() removes the OpenCL builtin fma(float,float,float) from the overload set. In any build with both FFT_FP32 and NTT_GF61 -- FFT3261, FFT3231, FFT323161 -- the float call sites then resolve to fma(Z61,Z61,Z61), convert their float arguments to ulong, and return ulong2 where float2 is expected: fftbase.cl:1212: error: assigning to '__private F2' (aka '__private float2') from incompatible type 'ulong2' (vector of 2 'ulong' values) 1212 | base = U2(fma(a, -w.y, w.x), fma(a, w.x, -w.y)); That is a hard error under ROCm, so every FP32-bearing transform type fails to build. It is also the type PRPLL picks by default on a machine with no tune.txt, so a fresh build does not start at all. The overload has no call sites, so renaming it costs nothing. Reproduced on Radeon Pro V620 (gfx1030), ROCm 7.14, OpenCL 3581.0 (HSA1.1,LC): prpll -prp 104000021 -iters 10000 # picks 4M 4:1K:8:256 Same root cause as #2, which was opened in March and closed by its author without review. --- src/cl/math.cl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cl/math.cl b/src/cl/math.cl index 0cd73719..a5f44b26 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -962,7 +962,12 @@ Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u128 c, const u32 a_m61_count, const u32 b Z61 OVERLOAD mul(Z61 a, Z61 b) { return modM61(weakMul(a, b, 2, 2)); } -Z61 OVERLOAD fma(Z61 a, Z61 b, Z61 c) { return modM61(weakMulAdd(a, b, c, 2, 2)); } +// Not named fma(): declaring a user overload of fma() hides the OpenCL builtin +// fma(float,float,float), so in a build with both FFT_FP32 and NTT_GF61 the float call sites +// (chainMul4() in fftbase.cl, for one) resolve to this overload instead, convert their float +// arguments to ulong, and return ulong2 where float2 is expected. This overload has no call +// sites, so the name is free to change. +Z61 OVERLOAD fmaZ61(Z61 a, Z61 b, Z61 c) { return modM61(weakMulAdd(a, b, c, 2, 2)); } // Multiply by 2 Z61 OVERLOAD mul2(Z61 a) { return add(a, a); } From 46f037575ee2d05c8d98078f9d4d445e707a1e58 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 21 Aug 2026 01:06:56 -0400 Subject: [PATCH 122/214] Fix wrong residue table after reducePower() --- src/Proof.cpp | 13 +++++++++++-- src/Proof.h | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Proof.cpp b/src/Proof.cpp index 6ad2987b..a869d5cc 100644 --- a/src/Proof.cpp +++ b/src/Proof.cpp @@ -142,8 +142,11 @@ ProofSet::ProofSet(u64 E, u32 power) fs::create_directories(proofPath(E)); - vector spans; - for (u64 span = (E + 1) / 2; spans.size() < power; span = (span + 1) / 2) { spans.push_back(span); } + rebuildPoints(); +} + +void ProofSet::rebuildPoints() { + points.clear(); points.push_back(0); u32 p; @@ -171,6 +174,12 @@ ProofSet::ProofSet(u64 E, u32 power) } } +void ProofSet::reducePower() { + assert(power > 0); + --power; + rebuildPoints(); +} + bool ProofSet::isInPoints(u64 E, u32 power, u64 k) { if (k == E) { return true; } // special-case E u64 start = 0; diff --git a/src/Proof.h b/src/Proof.h index 68fc1f4d..383eaef9 100644 --- a/src/Proof.h +++ b/src/Proof.h @@ -59,6 +59,8 @@ class ProofSet { private: vector points; + + void rebuildPoints(); bool isValidTo(u64 limitK) const; @@ -86,6 +88,6 @@ class ProofSet { void save(u64 k, const Words& words) const { return save(E, power, k, words); } Words load(u64 k) const { return load(E, power, k); } - void reducePower() { power--; } + void reducePower(); std::pair> computeProof(Gpu *gpu) const; }; From 96646c399ad1387ebe6901709d1b857e53528af7 Mon Sep 17 00:00:00 2001 From: george Date: Sat, 22 Aug 2026 02:02:15 +0000 Subject: [PATCH 123/214] Second attempt at using the C preprocessor to effect C++ templates. --- src/cl/base.cl | 40 --- src/cl/carryfused.cl | 4 +- src/cl/expand.cl | 53 ++++ src/cl/fftbase.cl | 600 +--------------------------------------- src/cl/fftmiddlein.cl | 4 +- src/cl/fftmiddleout.cl | 4 +- src/cl/fftp.cl | 4 +- src/cl/fftw.cl | 4 +- src/cl/middle.cl | 38 ++- src/cl/shufl.cl | 601 +++++++++++++++++++++++++++++++++++++++++ src/cl/tailmul.cl | 7 +- src/cl/tailsquare.cl | 7 +- 12 files changed, 707 insertions(+), 659 deletions(-) create mode 100644 src/cl/expand.cl create mode 100644 src/cl/shufl.cl diff --git a/src/cl/base.cl b/src/cl/base.cl index 2059f15a..4cf747e1 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -755,46 +755,6 @@ void PREFETCHL2(const __global void *addr) { #endif } -// Some routines can be written for any 64-bit data type (T2 or GF61). Same for 32-bit data types (F2 or GF31). -// Some routines can be written to work 32-bit and 64-bit data types. -// These #defines make it easy to write those routines. This used to be done with type-casting, but -// this method generates better PTX code (not sure if that results in any better run times). - -#if FFT_FP64 -#define T_Z61 T -#define T2_GF61 T2 -#define T2_F2_GF31_GF61 T2 -#define as_T2_GF61 as_double2 -#endif -#if NTT_GF61 -#define T_Z61 Z61 -#define T2_GF61 GF61 -#define T2_F2_GF31_GF61 GF61 -#define as_T2_GF61 as_ulong2 -#endif -#if FFT_FP32 -#define F_Z31 F -#define F2_GF31 F2 -#define T2_F2_GF31_GF61 F2 -#endif -#if NTT_GF31 -#define F_Z31 Z31 -#define F2_GF31 GF31 -#define T2_F2_GF31_GF61 GF31 -#endif - -#if FFT_FP64 || NTT_GF61 || FFT_FP32 || NTT_GF31 -void OVERLOAD read(u32 WG, u32 N, T2_F2_GF31_GF61 *u, const global T2_F2_GF31_GF61 *in, u32 base) { - in += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG]); } -} - -void OVERLOAD write(u32 WG, u32 N, T2_F2_GF31_GF61 *u, global T2_F2_GF31_GF61 *out, u32 base) { - out += base + (u32) get_local_id(0); - for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG], u[i]); } -} -#endif - // On "classic" AMD GCN GPUs such as Radeon VII, the wavefront size was always 64. On RDNA GPUs the wavefront can // be configured to be either 64 or 32. We use the FAST_BARRIER define as an indicator for GCN GPUs. // On Nvidia GPUs the wavefront size is 32. diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 8f70e16a..23c8fa42 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -4,7 +4,9 @@ #include "fftwidth.cl" #include "carryutil.cl" #include "weight.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" void spin() { #if defined(__has_builtin) && __has_builtin(__builtin_amdgcn_s_sleep) diff --git a/src/cl/expand.cl b/src/cl/expand.cl new file mode 100644 index 00000000..b60c3161 --- /dev/null +++ b/src/cl/expand.cl @@ -0,0 +1,53 @@ +// Copyright (C) Mihai Preda + +// Some routines can be written for any 64-bit data type (T2 or GF61). Same for 32-bit data types (F2 or GF31). +// Some routines can be written to work 32-bit and 64-bit data types. +// These #defines make it easy to write those routines. This used to be done with type-casting, but +// this method generates better PTX code (not sure if that results in any better run times). + +#if FFT_FP64 +#define T_Z61 T +#define T2_GF61 T2 +#define T2_F2_GF31_GF61 T2 +#define as_T2_GF61 as_double2 +#include INCLUDE_FILE +#undef T_Z61 +#undef T2_GF61 +#undef T2_F2_GF31_GF61 +#undef as_T2_GF61 +#endif + +#if NTT_GF61 +#define T_Z61 Z61 +#define T2_GF61 GF61 +#define T2_F2_GF31_GF61 GF61 +#define as_T2_GF61 as_ulong2 +#include INCLUDE_FILE +#undef T_Z61 +#undef T2_GF61 +#undef T2_F2_GF31_GF61 +#undef as_T2_GF61 +#endif + +#if FFT_FP32 +#define F_Z31 F +#define F2_GF31 F2 +#define T2_F2_GF31_GF61 F2 +#include INCLUDE_FILE +#undef F_Z31 +#undef F2_GF31 +#undef T2_F2_GF31_GF61 +#endif + +#if NTT_GF31 +#define F_Z31 Z31 +#define F2_GF31 GF31 +#define T2_F2_GF31_GF61 GF31 +#include INCLUDE_FILE +#undef F_Z31 +#undef F2_GF31 +#undef T2_F2_GF31_GF61 +#endif + +#undef INCLUDE_FILE + diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 19bdb196..f011d792 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -20,604 +20,8 @@ #define LDS_BYTES (WG * RADIX * SHUFL_BYTES) #endif -#if FFT_FP64 || NTT_GF61 - -// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. -// Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- -// even when operating on differernt sized data elements as can happen in an M31+M61 NTT. -// WG = workgroup size of a single fft_WIDTH or fft_HEIGHT -// n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT -// numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously -// lowMe = me % WG -// NOTE: shufl routines perform a bar(WG) at the start but not at the end. After calling shufl, a bar(WG) is required -// before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and -// only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively -// minor optimization would be to special case the first shufl call. -void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - int force_default = 0; -#if NOWG2 // For timing tests only. Option to not turn off LDS bank conflict code when numWG > 1. I've not found a GPU where this is beneficial. - if (numWG > 1) force_default = 1; -#endif -#if NOLDS2 // For timing tests only. Option to not turn off LDS bank for second shufl calls. I've not found a GPU where this is beneficial. - if (f != 1) force_default = 1; -#endif - - // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. - if (SHUFL_BYTES == 16) { - local T2_GF61* lds = lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); - -#if LDSPAD - // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 8, 72..., 16... lds[64..127] = +1 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad 1 value every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe & 7) * (WG + 1) + (lowMe / 8) * 8 + i] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 32 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } - return; - } - - // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // No padding of LDS blocks is needed to eliminate bank conflicts! Groups of 8 threads are already in separate LDS banks. - // We could however save a bar() by writing to same locations that the previous shufl wrote to. - if (0 && f == 8 && RADIX == 8) { - // for (u32 i = 0; i < RADIX; ++i) { lds[something] = u[i]; } - bar(WG); - //for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[something]; } - return; - } - - // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1, 65..., 16... lds[64..127] = +2 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad 1 value every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + (lowMe & 1) * 4 + i] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 4 + (lowMe / 32) * 8 + ((lowMe / 8) & 3) * (WG + 1) + (lowMe & 7)]; } - return; - } - - // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16... 4.. lds[64..127] = +1 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 16... 1.. output[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * (WG + 4) + i * 4 + (lowMe & 3)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - return; - } -#endif - -#if LDSSWIZ - // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } - return; - } - - // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and - // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. - // We can however save a bar() by writing to same locations that previous shufl wrote to. - if (!force_default && f == 8 && RADIX == 8) { - for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } - return; - } - - // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } - return; - } - - // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } - return; - } -#endif - - // Otherwise, execute the original shufl code - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } - } - - // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. - else if (SHUFL_BYTES == 8) { - local T_Z61* lds = ((local T_Z61*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); - -#if LDSPAD - // Special case first n == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Pad 8 values after every row to eliminate bank conflicts. - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - return; - } - - // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - return; - } -#endif - -#if LDSSWIZ - // Special case first n == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } - return; - } - - // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } - return; - } - - // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } - return; - } - - // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } - return; - } -#endif - - // Execute the original shufl code - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } - } - - // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. - // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! - else if (SHUFL_BYTES == 4) { - // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. - // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly - // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. - local int* lds = (local int*) lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); - - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - } -} - -#endif - - -#if FFT_FP32 || NTT_GF31 - -// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats or Z31s. -void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - //GW - would a 16 byte implementation be useful? Less LDS conflict work? - - int force_default = 0; -#if NOWG2 - if (numWG > 1) force_default = 1; -#endif -#if NOLDS2 - if (f != 1) force_default = 1; -#endif - - // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. - if (SHUFL_BYTES >= 8) { - local F2_GF31* lds = lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); - -#if LDSPAD - // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Pad 8 values after every 64 values to eliminate bank conflicts. - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i]; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - return; - } - - // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - return; - } -#endif - -#if LDSSWIZ - // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } - return; - } - - // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } - return; - } - - // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } - return; - } - - // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } - return; - } -#endif - - // Execute the original shufl code - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } - } - - // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. - else if (SHUFL_BYTES == 4) { - local F_Z31* lds = ((local F_Z31*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); - -#if LDSPAD - // Special case first n == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 2, 66..., 3, 67..., 32, 96... lds[64..127] = +4 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } - return; - } - - // Special case second n == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Pad 8 values after every 64 values to eliminate bank conflicts. - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - return; - } - - // Special case first n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 32 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 7.., 32... lds[64..127] = +8 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } - return; - } - - // Special case second n == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 1.... ... 8... lds[64..127] = +2 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } - return; - } -#endif - - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } - } -} - -#endif - +#define INCLUDE_FILE "shufl.cl" +#include "expand.cl" #if FFT_FP64 diff --git a/src/cl/fftmiddlein.cl b/src/cl/fftmiddlein.cl index a7c34538..5c416917 100644 --- a/src/cl/fftmiddlein.cl +++ b/src/cl/fftmiddlein.cl @@ -2,7 +2,9 @@ #include "base.cl" #include "fft-middle.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if !INPLACE // Original implementation (not in place) diff --git a/src/cl/fftmiddleout.cl b/src/cl/fftmiddleout.cl index b3c18d96..234dadd5 100644 --- a/src/cl/fftmiddleout.cl +++ b/src/cl/fftmiddleout.cl @@ -2,7 +2,9 @@ #include "base.cl" #include "fft-middle.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if !INPLACE // Original implementation (not in place) diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index f91cd99a..620b0734 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -3,7 +3,9 @@ #include "base.cl" #include "fftwidth.cl" #include "weight.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if FFT_TYPE == FFT64 diff --git a/src/cl/fftw.cl b/src/cl/fftw.cl index 72fe1ea8..14b4f5fd 100644 --- a/src/cl/fftw.cl +++ b/src/cl/fftw.cl @@ -2,7 +2,9 @@ #include "base.cl" #include "fftwidth.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" #if FFT_FP64 diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 370850d1..90951dc5 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -34,9 +34,23 @@ #define MIDDLE_OUT_LDS_TRANSPOSE 1 #endif +// These were the original read/write routines for accessing FFT data. I'm not sure if they are used anymore. + +#ifdef T2_F2_GF31_GF61 +void OVERLOAD read(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, const global T2_F2_GF31_GF61 *in, u32 base) { + in += base + (u32) get_local_id(0); + for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG_SZ]); } +} + +void OVERLOAD write(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, global T2_F2_GF31_GF61 *out, u32 base) { + out += base + (u32) get_local_id(0); + for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG_SZ], u[i]); } +} +#endif + #if !INPLACE // Original implementation (not in place) -#if FFT_FP64 || NTT_GF61 +#ifdef T2_GF61 //**************************************************************************************** // Pair of routines to write data from carryFused and read data into fftMiddleIn @@ -90,7 +104,7 @@ void OVERLOAD readMiddleInLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { // x ranges 0...SMALL_HEIGHT-1 (multiples of one) (also known as 0...G_H-1 and 0...NH-1) // y ranges 0...MIDDLE*WIDTH-1 (multiples of SMALL_HEIGHT) -void OVERLOAD writeMiddleInLine (P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleInLine(P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) { //u32 SIZEY = IN_WG / IN_SIZEX; //u32 num_x_chunks = WIDTH / IN_SIZEX; // Number of x chunks @@ -278,7 +292,7 @@ void OVERLOAD readMiddleOutLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { // adjusted to effect a transpose. Or caller must transpose the x and y values and send us an out pointer with thread_id added in. // In other words, caller is responsible for deciding the best way to transpose x and y values. -void OVERLOAD writeMiddleOutLine (P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleOutLine(P(T2_GF61) out, T2_GF61 *u, u32 chunk_y, u32 chunk_x) { //u32 SIZEY = OUT_WG / OUT_SIZEX; //u32 num_x_chunks = SMALL_HEIGHT / OUT_SIZEX; // Number of x chunks @@ -378,7 +392,7 @@ void OVERLOAD readCarryFusedLine(CP(T2_GF61) in, T2_GF61 *u, u32 line, u32 me) { /* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ -#if FFT_FP32 || NTT_GF31 +#ifdef F2_GF31 void OVERLOAD writeCarryFusedLine(F2_GF31 *u, P(F2_GF31) out, u32 line, u32 me) { #if PAD_SIZE > 0 @@ -404,7 +418,7 @@ void OVERLOAD readMiddleInLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { #endif } -void OVERLOAD writeMiddleInLine (P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleInLine(P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) { #if PAD_SIZE > 0 u32 SIZEY = IN_WG / IN_SIZEX; @@ -512,7 +526,7 @@ void OVERLOAD readMiddleOutLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { #endif } -void OVERLOAD writeMiddleOutLine (P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) +void OVERLOAD writeMiddleOutLine(P(F2_GF31) out, F2_GF31 *u, u32 chunk_y, u32 chunk_x) { #if PAD_SIZE > 0 u32 SIZEY = OUT_WG / OUT_SIZEX; @@ -659,7 +673,7 @@ void OVERLOAD readCarryFusedLine(CP(F2_GF31) in, F2_GF31 *u, u32 line, u32 me) { // This leaves the "columns" starting at +1KB unused - suggesting a pad of +1KB before the 64Ks would yield a better distribution in the L2 cache. // However, if we have say an 8-way 16MB L2 cache then each way contains 2MB. If so, we'd want to pad 1KB before the 16th 64K FFT data value. -#if FFT_FP64 || NTT_GF61 +#ifdef T2_GF61 //**************************************************************************************** // Pair of routines to read/write data to/from carryFused @@ -721,7 +735,7 @@ void OVERLOAD readMiddleInLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. -void OVERLOAD writeMiddleInLine (P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) +void OVERLOAD writeMiddleInLine(P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) { out += (x / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, y / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } @@ -764,7 +778,7 @@ void OVERLOAD readMiddleOutLine(T2_GF61 *u, CP(T2_GF61) in, u32 y, u32 x) { } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. -void OVERLOAD writeMiddleOutLine (P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) +void OVERLOAD writeMiddleOutLine(P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) { out += (y / 16 * SIZEW) + (y % 16 * SIZEBLK) + (SWIZ(y % 16, x / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM], u[i]); } @@ -777,7 +791,7 @@ void OVERLOAD writeMiddleOutLine (P(T2_GF61) out, T2_GF61 *u, u32 y, u32 x) /* Similar to above, but for an FFT based on FP32 or GF31 */ /**************************************************************************/ -#if FFT_FP32 || NTT_GF31 +#ifdef F2_GF31 //**************************************************************************************** // Pair of routines to read/write data to/from carryFused @@ -840,7 +854,7 @@ void OVERLOAD readMiddleInLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { } // NOTE: writeMiddleInLine uses the same definition of x,y as readMiddleInLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleInLine. -void OVERLOAD writeMiddleInLine (P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) +void OVERLOAD writeMiddleInLine(P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) { out += (x / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, y / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } @@ -883,7 +897,7 @@ void OVERLOAD readMiddleOutLine(F2_GF31 *u, CP(F2_GF31) in, u32 y, u32 x) { } // NOTE: writeMiddleOutLine uses the same definition of x,y as readMiddleOutLine. Caller transposes 16x16 blocks of FFT data before calling writeMiddleOutLine. -void OVERLOAD writeMiddleOutLine (P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) +void OVERLOAD writeMiddleOutLine(P(F2_GF31) out, F2_GF31 *u, u32 y, u32 x) { out += (y / 16 * SIZEW32) + (y % 16 * SIZEBLK32) + (SWIZ32(y % 16, x / 16) * 16) + (x % 16); for (i32 i = 0; i < MIDDLE; ++i) { FFTSTORE(&out[i * SIZEM32], u[i]); } diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl new file mode 100644 index 00000000..9ed387a9 --- /dev/null +++ b/src/cl/shufl.cl @@ -0,0 +1,601 @@ +// Copyright (C) Mihai Preda + +#ifdef T2_GF61 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. +// Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- +// even when operating on differernt sized data elements as can happen in an M31+M61 NTT. +// WG = workgroup size of a single fft_WIDTH or fft_HEIGHT +// n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT +// numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously +// lowMe = me % WG +// NOTE: shufl routines perform a bar(WG) at the start but not at the end. After calling shufl, a bar(WG) is required +// before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and +// only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively +// minor optimization would be to special case the first shufl call. +void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + int force_default = 0; +#if NOWG2 // For timing tests only. Option to not turn off LDS bank conflict code when numWG > 1. I've not found a GPU where this is beneficial. + if (numWG > 1) force_default = 1; +#endif +#if NOLDS2 // For timing tests only. Option to not turn off LDS bank for second shufl calls. I've not found a GPU where this is beneficial. + if (f != 1) force_default = 1; +#endif + + // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. + if (SHUFL_BYTES == 16) { + local T2_GF61* lds = lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 8, 72..., 16... lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad 1 value every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe & 7) * (WG + 1) + (lowMe / 8) * 8 + i] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 32 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // No padding of LDS blocks is needed to eliminate bank conflicts! Groups of 8 threads are already in separate LDS banks. + // We could however save a bar() by writing to same locations that the previous shufl wrote to. + if (0 && f == 8 && RADIX == 8) { + // for (u32 i = 0; i < RADIX; ++i) { lds[something] = u[i]; } + bar(WG); + //for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[something]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1, 65..., 16... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad 1 value every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + (lowMe & 1) * 4 + i] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 4 + (lowMe / 32) * 8 + ((lowMe / 8) & 3) * (WG + 1) + (lowMe & 7)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16... 4.. lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 16... 1.. output[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * (WG + 4) + i * 4 + (lowMe & 3)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and + // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. + // We can however save a bar() by writing to same locations that previous shufl wrote to. + if (!force_default && f == 8 && RADIX == 8) { + for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } + return; + } +#endif + + // Otherwise, execute the original shufl code + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + } + + // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. + else if (SHUFL_BYTES == 8) { + local T_Z61* lds = ((local T_Z61*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); + +#if LDSPAD + // Special case first RADIX == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every row to eliminate bank conflicts. + if (!force_default && f == 8 && RADIX == 8) { + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). + if (!force_default && f == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + return; + } +#endif + + // Execute the original shufl code + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + } + + // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. + // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + else if (SHUFL_BYTES == 4) { + // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. + // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly + // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. + local int* lds = (local int*) lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); + + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + } +} + +#endif + + +#ifdef F2_GF31 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats or Z31s. +void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + //GW - would a 16 byte implementation be useful? Less LDS conflict work? + + int force_default = 0; +#if NOWG2 + if (numWG > 1) force_default = 1; +#endif +#if NOLDS2 + if (f != 1) force_default = 1; +#endif + + // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. + if (SHUFL_BYTES >= 8) { + local F2_GF31* lds = lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (!force_default && f == 8 && RADIX == 8) { + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i]; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). + if (!force_default && f == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + return; + } +#endif + + // Execute the original shufl code + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + } + + // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. + else if (SHUFL_BYTES == 4) { + local F_Z31* lds = ((local F_Z31*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 2, 66..., 3, 67..., 32, 96... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (!force_default && f == 8 && RADIX == 8) { + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 32 must have unique LDS banks. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 7.., 32... lds[64..127] = +8 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (!force_default && f == 1 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 1.... ... 8... lds[64..127] = +2 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (!force_default && f == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + return; + } +#endif + + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + } +} + +#endif + + diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 957aace9..15d9076e 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -2,8 +2,11 @@ #include "base.cl" #include "fftheight.cl" -#include "tailutil.cl" -#include "middle.cl" + +#define INCLUDE_FILE "tailutil.cl" +#include "expand.cl" +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" // If not doing L2 stripes, process the lines in any order. // If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 16 * MIDDLE tailSquare lines. diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 8d2a9a7e..53d7d13e 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -2,8 +2,11 @@ #include "base.cl" #include "fftheight.cl" -#include "tailutil.cl" -#include "middle.cl" + +#define INCLUDE_FILE "tailutil.cl" +#include "expand.cl" +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" // If not doing L2 stripes, process the lines in any order. // If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 16 * MIDDLE tailSquare lines. From 92f92e25d536ce3a4359745fabcd19f0890cd968 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 22 Aug 2026 06:57:33 -0400 Subject: [PATCH 124/214] Try reduced power proof on CRCError --- src/Gpu.cpp | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 70ea4d14..204b88ea 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -2259,18 +2259,22 @@ fs::path Gpu::saveProof(const Args& args, ProofSet& proofSet) { bool problem_proof = false; for ( ; ; ) { for (int retry = 0; retry == 0 || (retry == 1 && !problem_proof); ++retry) { - auto [proof, hashes] = proofSet.computeProof(this); - fs::path const tmpFile = proof.file(args.proofToVerifyDir); - proof.save(tmpFile); - - fs::path proofFile = proof.file(args.proofResultDir); - - bool const ok = Proof::load(tmpFile).verify(this, hashes); - log("Proof '%s' verification %s\n", tmpFile.string().c_str(), ok ? "OK" : "FAILED"); - if (ok) { - fancyRename(tmpFile, proofFile); - log("Proof '%s' generated\n", proofFile.string().c_str()); - return proofFile; + try { + auto [proof, hashes] = proofSet.computeProof(this); + fs::path const tmpFile = proof.file(args.proofToVerifyDir); + proof.save(tmpFile); + + fs::path proofFile = proof.file(args.proofResultDir); + + bool const ok = Proof::load(tmpFile).verify(this, hashes); + log("Proof '%s' verification %s\n", tmpFile.string().c_str(), ok ? "OK" : "FAILED"); + if (ok) { + fancyRename(tmpFile, proofFile); + log("Proof '%s' generated\n", proofFile.string().c_str()); + return proofFile; + } + } catch (const CRCError&) { + break; } } problem_proof = true; From d58ef5f62797d1efcde336f318cc5dd5835f91da Mon Sep 17 00:00:00 2001 From: george Date: Sat, 22 Aug 2026 15:00:24 +0000 Subject: [PATCH 125/214] One more include file needed fixing for new combo types (templating) --- src/cl/ffthin.cl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index 52c39322..e95eb88d 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -2,7 +2,9 @@ #include "base.cl" #include "fftheight.cl" -#include "middle.cl" + +#define INCLUDE_FILE "middle.cl" +#include "expand.cl" // If not doing L2 stripes, process the lines in any order. // If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 2 * stripe_group_size * 16 * MIDDLE tailSquare lines. From 6cf0dcdd64ff6e1af3825652bfb10346359f4a81 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 1 Sep 2026 21:18:41 +0000 Subject: [PATCH 126/214] Faster WIDTH and HEIGHT = 1K (radix 8). Improved GF31 cmul. Allow HEIGHT > WIDTH in -tune. Faster radix-16 code. --- src/FFTConfig.cpp | 13 +- src/FFTConfig.h | 4 +- src/Gpu.cpp | 2 +- src/TrigBufCache.cpp | 100 ++- src/cl/fft16.cl | 75 ++- src/cl/fft4.cl | 23 +- src/cl/fft8.cl | 245 ++++++- src/cl/fftbase.cl | 664 ++++++++++++++++++- src/cl/math.cl | 408 ++++++++++-- src/cl/shufl.cl | 1488 +++++++++++++++++++++++++----------------- 10 files changed, 2256 insertions(+), 766 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index 120bc287..ee13528a 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -86,9 +86,8 @@ vector FFTShape::allShapes(u32 sizeFrom, u32 sizeTo) { for (enum FFT_TYPES const type : {FFT64, FFT6431, FFT3161, FFT3261, FFT61, FFT323161}) { for (u32 const width : {256, 512, 1024, 4096}) { for (u32 const height : {256, 512, 1024}) { - if (width == 256 && height == 1024) { continue; } // Skip because we prefer width >= height for (u32 const middle : {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) { - if (type != FFT64 && (middle & (middle - 1))) continue; // Reject non-power-of-two NTTs + if (type != FFT64 && type != FFT32 && (middle & (middle - 1))) continue; // Reject non-power-of-two NTTs u32 const sz = width * height * middle * 2; if (sizeFrom <= sz && sz <= sizeTo) { configs.emplace_back(type, width, middle, height); @@ -192,17 +191,15 @@ bool FFTShape::needsLargeCarry(u64 E) const { // Return TRUE for "favored" shapes. That is, those that are most likely to be useful. To save time in generating bpw data, only these favored // shapes have their bpw data pre-computed. Bpw for non-favored shapes is guessed from the bpw data we do have. Also. -tune will normally only // time favored shapes. These are the rules for deciding favored shapes: -// WIDTH >= HEIGHT // WIDTH=4K: HEIGHT>=512, MIDDLE>=9 (2*8 combos) // WIDTH=1K: MIDDLE>=5 (3*12 combos) // WIDTH=512: MIDDLE>=4 (2*13 combos) // WIDTH=256: MIDDLE>=1 (16 combos) bool FFTShape::isFavoredShape() const { - return width >= height && - ((width == 4096 && height >= 512 && middle >= 9) || - (width == 1024 && middle >= 5) || - (width == 512 && middle >= 4) || - (width == 256 && middle >= 1)); + return ((width == 4096 && height >= 512 && middle >= 9) || + (width == 1024 && middle >= 5) || + (width == 512 && middle >= 4) || + (width == 256 && middle >= 1)); } FFTConfig::FFTConfig(const string& spec) { diff --git a/src/FFTConfig.h b/src/FFTConfig.h index 3863ec19..4c5d1769 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -42,8 +42,8 @@ class FFTShape { explicit FFTShape(const string& spec); [[nodiscard]] u32 size() const { return width * height * middle * 2; } - [[nodiscard]] u32 nW() const { return (width == 1024 || width == 256 /*|| width == 4096*/) ? 4 : 8; } - [[nodiscard]] u32 nH() const { return (height == 1024 || height == 256 /*|| height == 4096*/) ? 4 : 8; } + [[nodiscard]] u32 nW() const { return (/*width == 1024 ||*/ width == 256) ? 4 : 8; } + [[nodiscard]] u32 nH() const { return (/*height == 1024 ||*/ height == 256) ? 4 : 8; } [[nodiscard]] float minBpw() const { return fft_type != FFT32 ? 3.0f : 1.0f; } [[nodiscard]] float maxBpw() const { return *std::ranges::max_element(bpw); } diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 70ea4d14..b2da45dd 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -620,7 +620,7 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { use_override = "REGCF3231"; break; case FFT6431: - regs = -1; + regs = nW == 8 ? -1 : -1; // Tested on TitanV, NW=8, CUDA 13.0. NW=4 not tested. use_override = "REGCF6431"; break; case FFT31: diff --git a/src/TrigBufCache.cpp b/src/TrigBufCache.cpp index d46d6082..f9e532c5 100644 --- a/src/TrigBufCache.cpp +++ b/src/TrigBufCache.cpp @@ -143,10 +143,26 @@ static vector genSmallTrigFP64(u32 size, u32 radix) { u32 const WG = size / radix; vector tab; -// old fft_WIDTH and fft_HEIGHT - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(0 && radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(0 && radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + } + } + } + + // original fft_WIDTH and fft_HEIGHT + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(radix / line >= 8 ? root1Fancy(size, col * line) : root1(size, col * line)); + } } } tab.resize(size); @@ -397,10 +413,26 @@ static vector genSmallTrigFP32(u32 size, u32 radix) { u32 const WG = size / radix; vector tab; -// old fft_WIDTH and fft_HEIGHT - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(0 && radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(0 && radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + } + } + } + + // original fft_WIDTH and fft_HEIGHT + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(radix / line >= 8 ? root1FancyFP32(size, col * line) : root1FP32(size, col * line)); + } } } tab.resize(size); @@ -627,13 +659,31 @@ uint2 root1GF31(u32 N, u32 k) { static vector genSmallTrigGF31(u32 size, u32 radix) { u32 const WG = size / radix; vector tab; - GF31 const root1size = GF31::root_one(size); - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(root1GF31(root1size, col * line)); + + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(root1GF31(root1size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(root1GF31(root1size, col * line)); + } + } + } + + // Standard roots + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(root1GF31(root1size, col * line)); + } } } + tab.resize(size); return tab; } @@ -789,13 +839,31 @@ ulong2 root1GF61(u32 N, u32 k) { static vector genSmallTrigGF61(u32 size, u32 radix) { u32 const WG = size / radix; vector tab; - GF61 const root1size = GF61::root_one(size); - for (u32 line = 1; line < radix; ++line) { - for (u32 col = 0; col < WG; ++col) { - tab.push_back(root1GF61(root1size, col * line)); + + // New SIZE=256, RADIX=8 which is really mixed radix-4 and radix-8. This is for a 4 * 8 * 8 implementation. + if (size == 256 && radix == 8) { + for (u32 line = 1; line < radix/2; ++line) { + for (u32 col = 0; col < WG*2; ++col) { + tab.push_back(root1GF61(root1size, col * line)); + } + } + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; col += 4) { + tab.push_back(root1GF61(root1size, col * line)); + } } } + + // Standard roots + else { + for (u32 line = 1; line < radix; ++line) { + for (u32 col = 0; col < WG; ++col) { + tab.push_back(root1GF61(root1size, col * line)); + } + } + } + tab.resize(size); return tab; } diff --git a/src/cl/fft16.cl b/src/cl/fft16.cl index 7cbbb24b..edb494be 100644 --- a/src/cl/fft16.cl +++ b/src/cl/fft16.cl @@ -2,33 +2,53 @@ #if FFT_FP64 -#if 0 +#if 1 #if 1 #include "fft4.cl" -// 24 FMA (of which 16 MUL) + 136 ADD +// 56 FMA + 96 ADD void OVERLOAD fft16(T2 *u) { double C1 = 0.92387953251128674, // cos(tau/16) - S1 = 0.38268343236508978; // sin(tau/16) + S1 = 0.38268343236508978, // sin(tau/16) + S1_over_C1 = 0.4142135623730950488017, + C1_over_S1 = 2.4142135623730950488017; for (int i = 0; i < 4; ++i) { fft4by(u, i, 4, 16); } - u[5] = cmul(u[ 5], U2(C1, S1)); - u[7] = cmul(u[ 7], U2(S1, C1)); - u[13] = cmul(u[13], U2(S1, C1)); - u[15] = cmul(u[15], -U2(C1, S1)); - - u[6] = mul_t8(u[6]); - u[9] = mul_t8(u[9]); - u[11] = mul_3t8(u[11]); - u[14] = mul_3t8(u[14]); - - u[10] = mul_t4(u[10]); - - for (int i = 0; i < 4; ++i) { fft4by(u, 4 * i, 1, 16); } + X2(u[0], u[2]); + X2_mul_t4(u[1], u[3]); + X2(u[0], u[1]); + X2(u[2], u[3]); + SWAP(u[1], u[2]); + + u[9] = mul_t8_delayed(u[9]); // delays a mul by M_SQRT1_2 + u[11] = mul_t8_delayed(u[11]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + X2t4(u[8], u[10]); + X2t4_mul_t4(u[9], u[11]); + X2ad(u[8], u[9], M_SQRT1_2); + X2ad(u[10], u[11], M_SQRT1_2); + SWAP(u[9], u[10]); + + u[5] = partial_cmul(u[5], S1_over_C1); // delays a mul by C1 + u[6] = mul_t8_delayed(u[6]); // delays a mul by M_SQRT1_2 + u[7] = partial_cmul(u[7], C1_over_S1); // delays a mul by S1 + X2ad(u[4], u[6], M_SQRT1_2); + X2ad_mul_t4(u[5], u[7], S1_over_C1); // mul by S1/C1, now both are delaying a mul by C1 + X2ad(u[4], u[5], C1); // apply delayed mul by C1 + X2ad(u[6], u[7], C1); // apply delayed mul by C1 + SWAP(u[5], u[6]); + + u[13] = partial_cmul(u[13], C1_over_S1); // delays a mul by S1 + u[14] = mul_t8_delayed(u[14]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + u[15] = partial_cmul(u[15], S1_over_C1); // delays a mul by -C1 + X2t4ad(u[12], u[14], M_SQRT1_2); + X2ad_mul_t4(u[13], u[15], -C1_over_S1); // mul by -C1/S1, now both are delaying a mul by S1 + X2ad(u[12], u[13], S1); // apply delayed mul by S1 + X2ad(u[14], u[15], S1); // apply delayed mul by S1 + SWAP(u[13], u[14]); SWAP(u[1], u[4]); SWAP(u[2], u[8]); @@ -36,8 +56,6 @@ void OVERLOAD fft16(T2 *u) { SWAP(u[6], u[9]); SWAP(u[7], u[13]); SWAP(u[11], u[14]); - - // for (int i = 0; i < 4; ++i) { fft4by(u, i, 4, 16); } } #else @@ -207,6 +225,8 @@ void OVERLOAD fft16(F2 *u) { void OVERLOAD fft16(GF31 *u) { const Z31 C1 = 1556715293; const Z31 S1 = 978592373; + const Z31 negC1 = M31 - C1; + const Z31 negS1 = M31 - S1; X2(u[0], u[8]); X2(u[1], u[9]); @@ -217,10 +237,10 @@ void OVERLOAD fft16(GF31 *u) { X2_mul_3t8(u[6], u[14]); X2(u[7], u[15]); - u[ 9] = cmul(u[ 9], U2( C1, S1)); // 1t16 - u[11] = cmul(u[11], U2( S1, C1)); // 3t16 - u[13] = cmul(u[13], U2(neg(S1), C1)); // 5t16 //GWBUG - check if optimizer is eliminating the neg (or better yet perhaps tweak follow up code to expect a negative) - u[15] = cmul(u[15], U2(neg(C1), S1)); // 7t16 + u[ 9] = cmul_const(u[ 9], U2( C1, S1)); // 1t16 + u[11] = cmul_const(u[11], U2( S1, C1)); // 3t16 + u[13] = cmul_const(u[13], U2(negS1, C1)); // 5t16 + u[15] = cmul_const(u[15], U2(negC1, S1)); // 7t16 fft8Core(u); fft8Core(u + 8); @@ -247,9 +267,6 @@ void OVERLOAD fft16(GF31 *u) { #include "fft8.cl" void OVERLOAD fft16(GF61 *u) { - const Z61 C1 = 22027337052962166ULL; - const Z61 S1 = 1693317751237720973ULL; - X2(u[0], u[8]); X2(u[1], u[9]); X2_mul_t8(u[2], u[10]); @@ -259,10 +276,10 @@ void OVERLOAD fft16(GF61 *u) { X2_mul_3t8(u[6], u[14]); X2(u[7], u[15]); - u[ 9] = cmul(u[ 9], U2( C1, S1)); // 1t16 - u[11] = cmul(u[11], U2( S1, C1)); // 3t16 - u[13] = cmul(u[13], U2(neg(S1), C1)); // 5t16 //GWBUG - check if optimizer is eliminating the neg (or better yet perhaps tweak follow up code to expect a negative) - u[15] = cmul(u[15], U2(neg(C1), S1)); // 7t16 + u[9] = mul_t16(u[9]); + u[11] = mul_3t16(u[11]); + u[13] = mul_5t16(u[13]); + u[15] = mul_7t16(u[15]); fft8Core(u); fft8Core(u + 8); diff --git a/src/cl/fft4.cl b/src/cl/fft4.cl index 1aafbb69..80b2c3af 100644 --- a/src/cl/fft4.cl +++ b/src/cl/fft4.cl @@ -6,7 +6,7 @@ void OVERLOAD fft4Core(T2 *u) { X2(u[0], u[2]); - X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2_mul_t4(u[1], u[3]); X2(u[0], u[1]); X2(u[2], u[3]); @@ -26,7 +26,7 @@ void OVERLOAD fft4by(T2 *u, u32 base, u32 step, u32 M) { double x1 = A(1).x + A(3).x; double y3 = A(1).x - A(3).x; double y1 = A(1).y + A(3).y; - double x3 = -(A(1).y - A(3).y); + double x3 = A(3).y - A(1).y; double a0 = x0 + x1; double a1 = x0 - x1; @@ -48,10 +48,9 @@ void OVERLOAD fft4by(T2 *u, u32 base, u32 step, u32 M) { #else X2(A(0), A(2)); - X2(A(1), A(3)); + X2_mul_t4(A(1), A(3)); X2(A(0), A(1)); - A(3) = mul_t4(A(3)); X2(A(2), A(3)); SWAP(A(1), A(2)); @@ -74,7 +73,7 @@ void OVERLOAD fft4(T2 *u) { fft4by(u, 0, 1, 4); } void OVERLOAD fft4Core(F2 *u) { X2(u[0], u[2]); - X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2_mul_t4(u[1], u[3]); X2(u[0], u[1]); X2(u[2], u[3]); @@ -94,7 +93,7 @@ void OVERLOAD fft4by(F2 *u, u32 base, u32 step, u32 M) { float x1 = A(1).x + A(3).x; float y3 = A(1).x - A(3).x; float y1 = A(1).y + A(3).y; - float x3 = -(A(1).y - A(3).y); + float x3 = A(3).y - A(1).y; float a0 = x0 + x1; float a1 = x0 - x1; @@ -116,10 +115,9 @@ void OVERLOAD fft4by(F2 *u, u32 base, u32 step, u32 M) { #else X2(A(0), A(2)); - X2(A(1), A(3)); + X2_mul_t4(A(1), A(3)); X2(A(0), A(1)); - A(3) = mul_t4(A(3)); X2(A(2), A(3)); SWAP(A(1), A(2)); @@ -194,6 +192,15 @@ void OVERLOAD fft4(GF31 *u) { fft4by(u, 0, 1, 4); } #if NTT_GF61 +void OVERLOAD fft4Core(GF61 *u) { + X2q(&u[0], &u[2]); // u[0] = 0..2+, u[2] = -1-..1+ + X2q_mul_t4(&u[1], &u[3]); // u[1] = 0..2+, u[3] = -1-..1+ + X2q(&u[0], &u[1]); // u[0] = 0..4+, u[1] = -2-..2+ + X2q(&u[2], &u[3]); // u[2] = -2..2+, u[0] = -2-..2+ + u[0] = modM61q(u[0], 0); + for (u32 i = 1; i <= 3; ++i) u[i] = modM61q(u[i], 3); +} + // 16 ADD void OVERLOAD fft4by(GF61 *u, u32 base, u32 step, u32 M) { diff --git a/src/cl/fft8.cl b/src/cl/fft8.cl index 99ffa23c..6a9bb104 100644 --- a/src/cl/fft8.cl +++ b/src/cl/fft8.cl @@ -6,23 +6,18 @@ #if FFT_FP64 -T2 mul_t8_delayed(T2 a) { return U2(a.x - a.y, a.x + a.y); } -T2 mul_3t8_delayed(T2 a) { return U2(-(a.x + a.y), a.x - a.y); } -//#define X2_apply_delay(a, b) { T2 t = a; a = t + M_SQRT1_2 * b; b = t - M_SQRT1_2 * b; } -#define X2_apply_delay(a, b) { T2 t = a; a.x = fma(b.x, M_SQRT1_2, a.x); a.y = fma(b.y, M_SQRT1_2, a.y); b.x = fma(-M_SQRT1_2, b.x, t.x); b.y = fma(-M_SQRT1_2, b.y, t.y); } - void OVERLOAD fft4CoreSpecial(T2 *u) { X2(u[0], u[2]); - X2_mul_t4(u[1], u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); - X2_apply_delay(u[0], u[1]); - X2_apply_delay(u[2], u[3]); + X2t4_mul_t4(u[1], u[3]); // u[3] = mul_t4(u[3]); X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2ad(u[0], u[1], M_SQRT1_2); + X2ad(u[2], u[3], M_SQRT1_2); } void OVERLOAD fft8Core(T2 *u) { X2(u[0], u[4]); - X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); - X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); - X2(u[3], u[7]); u[7] = mul_3t8_delayed(u[7]); + X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); // Delays a mul by M_SQRT1_2 + X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); + X2(u[3], u[7]); u[7] = mul_t8_delayed(u[7]); // Delays a mul by i*M_SQRT1_2 (cheaper than calling mul_3t8_delayed) fft4Core(u); fft4CoreSpecial(u + 4); } @@ -44,23 +39,18 @@ void OVERLOAD fft8(T2 *u) { #if FFT_FP32 -F2 mul_t8_delayed(F2 a) { return U2(a.x - a.y, a.x + a.y); } -F2 mul_3t8_delayed(F2 a) { return U2(-(a.x + a.y), a.x - a.y); } -//#define X2_apply_delay(a, b) { F2 t = a; a = t + M_SQRT1_2 * b; b = t - M_SQRT1_2 * b; } -#define X2_apply_delay(a, b) { F2 t = a; a.x = fma(b.x, (float) M_SQRT1_2, a.x); a.y = fma(b.y, (float) M_SQRT1_2, a.y); b.x = fma((float) -M_SQRT1_2, b.x, t.x); b.y = fma((float) -M_SQRT1_2, b.y, t.y); } - void OVERLOAD fft4CoreSpecial(F2 *u) { X2(u[0], u[2]); - X2_mul_t4(u[1], u[3]); // X2(u[1], u[3]); u[3] = mul_t4(u[3]); - X2_apply_delay(u[0], u[1]); - X2_apply_delay(u[2], u[3]); + X2t4_mul_t4(u[1], u[3]); // u[3] = mul_t4(u[3]); X2(u[1], u[3]); u[3] = mul_t4(u[3]); + X2ad(u[0], u[1], M_SQRT1_2); + X2ad(u[2], u[3], M_SQRT1_2); } void OVERLOAD fft8Core(F2 *u) { X2(u[0], u[4]); - X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); - X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); - X2(u[3], u[7]); u[7] = mul_3t8_delayed(u[7]); + X2(u[1], u[5]); u[5] = mul_t8_delayed(u[5]); // Delays a mul by M_SQRT1_2 + X2_mul_t4(u[2], u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); + X2(u[3], u[7]); u[7] = mul_t8_delayed(u[7]); // Delays a mul by i*M_SQRT1_2 (cheaper than calling mul_3t8_delayed) fft4Core(u); fft4CoreSpecial(u + 4); } @@ -151,3 +141,214 @@ void OVERLOAD fft8(GF61 *u) { } #endif + + + + +//*********************************************************************************************************************** +// In a primarily radix 8 FFT, support other some other options such as radix-4 and radix-16 +//*********************************************************************************************************************** + +#if FFT_FP64 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(T2 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); +} + +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(T2 *u) { + fft8(u); +} +void OVERLOAD fft8_16b(T2 *u) { + const double C1 = 0.92387953251128674, // cos(tau/16) + S1 = 0.38268343236508978, // sin(tau/16) + S1_over_C1 = 0.4142135623730950488017, + C1_over_S1 = 2.4142135623730950488017; + + X2t4(u[0], u[4]); + X2t4(u[1], u[5]); + X2t4(u[2], u[6]); + X2t4(u[3], u[7]); + + u[1] = partial_cmul(u[1], S1_over_C1); // delays a mul by C1 + u[2] = mul_t8_delayed(u[2]); // delays a mul by M_SQRT1_2 + u[3] = partial_cmul(u[3], C1_over_S1); // delays a mul by S1 + X2ad(u[0], u[2], M_SQRT1_2); + X2ad_mul_t4(u[1], u[3], S1_over_C1); // mul by S1/C1, now both are delaying a mul by C1 + X2ad(u[0], u[1], C1); // apply delayed mul by C1 + X2ad(u[2], u[3], C1); // apply delayed mul by C1 + + u[5] = partial_cmul(u[5], C1_over_S1); // delays a mul by S1 + u[6] = mul_t8_delayed(u[6]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + u[7] = partial_cmul(u[7], S1_over_C1); // delays a mul by -C1 + X2t4ad(u[4], u[6], M_SQRT1_2); + X2ad_mul_t4(u[5], u[7], -C1_over_S1); // mul by -C1/S1, now both are delaying a mul by S1 + X2ad(u[4], u[5], S1); // apply delayed mul by S1 + X2ad(u[6], u[7], S1); // apply delayed mul by S1 + + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + +#endif + +#if FFT_FP32 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(F2 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); +} + +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(F2 *u) { + fft8(u); +} +void OVERLOAD fft8_16b(F2 *u) { + const float C1 = 0.92387953251128674, // cos(tau/16) + S1 = 0.38268343236508978, // sin(tau/16) + S1_over_C1 = 0.4142135623730950488017, + C1_over_S1 = 2.4142135623730950488017; + + X2t4(u[0], u[4]); + X2t4(u[1], u[5]); + X2t4(u[2], u[6]); + X2t4(u[3], u[7]); + + u[1] = partial_cmul(u[1], S1_over_C1); // delays a mul by C1 + u[2] = mul_t8_delayed(u[2]); // delays a mul by M_SQRT1_2 + u[3] = partial_cmul(u[3], C1_over_S1); // delays a mul by S1 + X2ad(u[0], u[2], M_SQRT1_2); + X2ad_mul_t4(u[1], u[3], S1_over_C1); // mul by S1/C1, now both are delaying a mul by C1 + X2ad(u[0], u[1], C1); // apply delayed mul by C1 + X2ad(u[2], u[3], C1); // apply delayed mul by C1 + + u[5] = partial_cmul(u[5], C1_over_S1); // delays a mul by S1 + u[6] = mul_t8_delayed(u[6]); // delays a mul by i*M_SQRT1_2 (a negation cheaper than mul_3t8_delayed) + u[7] = partial_cmul(u[7], S1_over_C1); // delays a mul by -C1 + X2t4ad(u[4], u[6], M_SQRT1_2); + X2ad_mul_t4(u[5], u[7], -C1_over_S1); // mul by -C1/S1, now both are delaying a mul by S1 + X2ad(u[4], u[5], S1); // apply delayed mul by S1 + X2ad(u[6], u[7], S1); // apply delayed mul by S1 + + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + +#endif + +#if NTT_GF31 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(GF31 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); +} + +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(GF31 *u) { + fft8(u); +} +void OVERLOAD fft8_16b(GF31 *u) { + const Z31 C1 = 1556715293; + const Z31 S1 = 978592373; + const Z31 negC1 = M31 - C1; + const Z31 negS1 = M31 - S1; + + X2t4(u[0], u[4]); + X2t4(u[1], u[5]); + X2t4(u[2], u[6]); + X2t4(u[3], u[7]); + + u[1] = cmul_const(u[1], U2(C1, S1)); + u[2] = mul_t8(u[2]); + u[3] = cmul_const(u[3], U2(S1, C1)); + X2(u[0], u[2]); + X2_mul_t4(u[1], u[3]); + X2(u[0], u[1]); + X2(u[2], u[3]); + + u[5] = cmul_const(u[5], U2(S1, C1)); + u[6] = mul_3t8(u[6]); + u[7] = cmul_const(u[7], U2(negC1, negS1)); + X2(u[4], u[6]); + X2_mul_t4(u[5], u[7]); + X2(u[4], u[5]); + X2(u[6], u[7]); + + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + +#endif + +#if NTT_GF61 + +// Do two fft4s on eight FFT values +void OVERLOAD fft8_4(GF61 *u) { + fft4by(u, 0, 2, 8); + fft4by(u, 1, 2, 8); +} + +// Perform the last three levels of a radix-16 butterfly. The initial radix-2 has already been performed and shufl'ed. +// This is used by SIZE=1K, RADIX=8 fft. There are two versions, one for the first eight radix-16 values and one for the second eight radix-16 values. +void OVERLOAD fft8_16a(GF61 *u) { + // shufl_and_fft2 performed "quick" adds, u[0-7] are in range 0..2+ + X2q(&u[0], &u[4]); // X2(u[0], u[4]); No reductions mod M61. u[0,4] range is 0..4+, -2-..2+ + X2q(&u[1], &u[5]); // X2(u[1], u[5]); Delay mul_t8 on u[5]. u[1,5] range is 0..4+, -2-..2+ + X2q_mul_t4(&u[2], &u[6]); // X2(u[2], u[6]); u[6] = mul_t4(u[6]); u[2,6] range is 0..4+, -2-..2+ + X2q_mul_t4(&u[3], &u[7]); // X2(u[3], u[7]); u[7] = mul_t4(u[7]); u[3,7] range is 0..4+, -2-..2+ Delay mul_t8 on u[7]. + + // Must normalize values. The delayed mul_t8s can help with that (half of the complex number needs normalizing before mul_t8). + for (u32 i = 0; i <= 3; ++i) u[i] = modM61q(u[i], 0); + u[4] = modM61q(u[4], 3); + u[5].x = optional_add((i64)u[5].x, 2*M61); // u[5] now 0-..2+, -2..2+ + u[5] = mul_t8q(u[5], 5); // Perform delayed mul_t8 (count of 5 based on u[5].y - u[5].x range of -4-..2+ + u[6] = modM61q(u[6], 3); + u[7].x = optional_add((i64)u[7].x, 2*M61); // u[7] now 0-..2+, -2..2+ + u[7] = mul_t8q(u[7], 5); // Perform delayed mul_t8. + + fft4Core(u); + fft4Core(u + 4); + + // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} +void OVERLOAD fft8_16b(GF61 *u) { + // shufl_and_fft2 performed "quick" subtracts, u[0-7] are in range -1-..1+ + X2qt4(&u[0], &u[4]); // -2..2+ + X2qt4(&u[1], &u[5]); + X2qt4(&u[2], &u[6]); + X2qt4(&u[3], &u[7]); + + // Must normalize values. Some mul_t8s can help with that (only half of the complex number needs normalizing before mul_t8). + u[0] = modM61q(u[0], 3); + u[1] = modM61q(u[1], 3); + u[2].x = optional_add((i64)u[2].x, 2*M61); // u[2] now 0-..2+, -2..2+ + u[3] = modM61q(u[3], 3); + u[4] = modM61q(u[4], 3); + u[5] = modM61q(u[5], 3); + u[6].y = optional_add((i64)u[6].y, 2*M61); // u[6] now -2-..2+, 0-..2+ + u[7] = modM61q(u[7], 3); + + u[1] = mul_t16(u[1]); + u[2] = mul_t8q(u[2], 5); // Perform mul_t8 (count of 5 based on u[2].y - u[2].x range of -4-..2+, -(u[2].x + u[2].y) range of -4-..2+ + u[3] = mul_3t16(u[3]); + fft4Core(u); + + u[5] = mul_3t16(u[5]); + u[6] = mul_3t8q(u[6], 3); // Perform mul_3t8 (count of 5 based on u[6].y - u[6].x range of -2-..4+, u[6].y + u[6].x range of -2-..4+ + u[7] = mul_9t16(u[7]); + fft4Core(u + 4); + + SWAP(u[1], u[4]); + SWAP(u[3], u[6]); +} + +#endif diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index f011d792..f0b6d37c 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -183,18 +183,128 @@ void OVERLOAD tabMul(Trig trig, T2 *u, u32 f, u32 me) { } } +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(Trig trig, T2 *u, u32 f, u32 me) { + + if (f == 1) { // fft8_4 is performed first + u32 p = me; + +// This code uses chained complex multiplies which could be faster on GPUs with great DP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN) { + T2 w = TFLOAD(&trig[p]); + T2 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + T2 base = csqTrig(w); + T2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } + + else { // fft8_4 is performed after an initial fft8 + +// This code uses chained complex multiplies which could be faster on GPUs with great DP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN) { + T2 w = TFLOAD(&trig[p]); + T2 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + T2 base = csqTrig(w); + T2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } + } +} + +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(Trig trig, T2 *u, u32 f, u32 me) { + +// This code uses chained complex multiplies which could be faster on GPUs with great DP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Apparently, chained Fancy muls at n=8 lengths are very accurate. + + if (TABMUL_CHAIN) { + u32 p = me & ~(f - 1); + T2 w = TFLOAD(&trig[p]); + +// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data +// T2 w2 = csqTrigFancy(w); +// u[2] = cmulFancy(u[2], w2); +// T2 w3 = ccubeTrigFancy(w2, w); +// u[3] = cmulFancy(u[3], w3); +// w3.x += 1; +// T2 base = cmulFancy(w3, w); +// for (int i = 4; i < 8; ++i) { +// u[i] = cmul(u[i], base); +// base = cmulFancy(base, w); +// } + + u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data + T2 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + T2 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + T2 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a + +//GW: Can any of these be Fancy? Yes, u[1] and u[2] + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } +} + //************************************************************************************ // New fft WIDTH and HEIGHT macros to support radix-4 FFTs with more FMA instructions //************************************************************************************ -// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. -// We're trying to calculate u * U2(cosine,sine). -// real = (u.x - u.y*sine_over_cosine) * cosine -// imag = (u.x*sine_over_cosine + u.y) * cosine -T2 partial_cmul(T2 u, T sine_over_cosine) { - return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); -} - // Copy of macro from fft4 and fft8 with FMAs added #define X2_via_FMA(a, c, b) { T2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } @@ -411,8 +521,8 @@ void finish_tabMul8_fft8(Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me // Do last level of fft8 X2(u[0], u[1]); X2(u[2], u[3]); - X2_apply_delay(u[4], u[5]); - X2_apply_delay(u[6], u[7]); + X2ad(u[4], u[5], M_SQRT1_2); + X2ad(u[6], u[7], M_SQRT1_2); } // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo @@ -569,6 +679,128 @@ void OVERLOAD fft_common(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 l // Finish third tabMul and perform final fft8. finish_tabMul8_fft8(trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 + +// Custom code for SIZE=256, RADIX=8, threads=32. Performed as 4 * 8 * 8. Radix-8 allows fewer +// shufls and tabmuls than radix-4. Fewer instructions, but more registers. +// Uses only 32 threads which is fine on nVidia, lousy on radeon VII (use WMUL=2, TAIL_KERNELS=2). +// +// Details for memory layout, trig data, and shufls: +// Mem: 0 1... 31 +// 32 +// ... +// 196 +// 224 ... 255 +// Only do a radix-4 fft. Non-standard TABMUL: (64 3/4 cmuls in blocks of 1 duplicated trig values) +// trig powers are: 0*0 0*1 .. 0*31 +// 0*32 .. 0*63 +// 1*0 1*1 .. 1*31 +// 1*32 .. 1*63 +// 2*0 2*1 .. 2*31 +// 2*32 .. 2*63 +// 3*0 3*1 .. 3*31 +// 3*32 .. 3*63 total trig data (6*32*16=3KB) +// non-standard shufl out: +// 0 64 .. 192 1... 7... +// 8 +// 16 +// ... +// 48 +// 56 +// standard TABMUL: (8 7/8 cmuls in blocks of 4 duplicated trig values) +// trig powers are: 0000 0000 .. 0000*7 +// 0000 4444 .. 4444*7 +// 0000 8888 .. 8888*7 +// 0000 12 ... +// 0000 16 ... +// 0000 20 ... +// 0000 24 ... +// 0000 28 ... total trig data (7*8*16=896B) +// standard shufl out: +// 0 64 .. 192 8... 56... +// 1 +// 2 +// ... +// 6 +// 7 +// +// FP64 and FP32 could benefit by starting the next fft8 after the radix-4 tabmul (easy FMA opportunities). + +// Code for SIZE=256, RADIX=8 +#elif WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Custom code for SIZE=1024, RADIX=8, threads=128. Performed as 8 * 8 * 2 * 8. Radix-8 allows fewer +// shufls and tabmuls than radix-4. Fewer instructions, but more registers. If we process 4 (or 8) +// independent width lines then we should be able to avoid the mul by w^0 in the next to last radix-8 step. +// +// Details for memory layout, trig data, and shufls: +// Mem: 0 1 ... 127 +// 128 +// 256 +// 384 +// 512 +// 640 +// 768 +// 896 ... 1023 +// standard TABMUL: (128 7/8 cmuls in blocks of 1 duplicated trig values) +// trig powers are: 0 0 0 .. 0*127 +// 0 1 2 .. 1*127 +// 0 2 4 .. 2*127 +// 0 3 6 .. 3*127 +// 0 4 8 .. 4*127 +// 0 5 10 .. 5*127 +// 0 6 12 .. 6*127 +// 0 7 14 .. 7*127 total trig data (7*128*16=14KB) +// standard shufl out: +// 0 128 .. 896 1... 15... +// 16 +// 32 +// 48 +// 64 +// 80 +// 96 +// 112 +// standard TABMUL: (16 7/8 cmuls in blocks of 8 duplicated trig values) +// trig powers are: 00000000 00000000 .. 00000000*15 +// 00000000 11111111 .. 11111111*15 +// 00000000 22222222 .. 22222222*15 +// 00000000 33333333 .. 33333333*15 +// 00000000 44444444 .. 44444444*15 +// 00000000 55555555 .. 55555555*15 +// 00000000 66666666 .. 66666666*15 +// 00000000 77777777 .. 77777777*15 total trig data (7*16*16=1.75KB) +// shufl out with an fft2: +// 0 128 .. 896 16 .. 112... 8... +// 1 +// 2 +// 3 +// 4 +// 5 +// 6 +// 7 + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + #else // Old / original version @@ -665,6 +897,121 @@ void OVERLOAD tabMul(TrigFP32 trig, F2 *u, u32 f, u32 me) { } } +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(TrigFP32 trig, F2 *u, u32 f, u32 me) { + + if (f == 1) { // fft8_4 is performed first + u32 p = me; + +// This code uses chained complex multiplies which could be faster on GPUs with great SP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN32) { + F2 w = TFLOAD(&trig[p]); + F2 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + F2 base = csqTrig(w); + F2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN32) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } + + else { // fft8_4 is performed after an initial fft8 + +// This code uses chained complex multiplies which could be faster on GPUs with great SP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN32) { + F2 w = TFLOAD(&trig[p]); + F2 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + F2 base = csqTrig(w); + F2 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN32) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } + } +} + +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(TrigFP32 trig, F2 *u, u32 f, u32 me) { + +// This code uses chained complex multiplies which could be faster on GPUs with great SP throughput or poor memory bandwidth or caching. +// This ought to be the least accurate version of Tabmul. In practice, this is just as accurate as reading precomputed values from memory. +// Apparently, chained Fancy muls at n=8 lengths are very accurate. + + if (TABMUL_CHAIN32) { + u32 p = me & ~(f - 1); + F2 w = TFLOAD(&trig[p]); + +// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data +// T2 w2 = csqTrigFancy(w); +// u[2] = cmulFancy(u[2], w2); +// T2 w3 = ccubeTrigFancy(w2, w); +// u[3] = cmulFancy(u[3], w3); +// w3.x += 1; +// T2 base = cmulFancy(w3, w); +// for (int i = 4; i < 8; ++i) { +// u[i] = cmul(u[i], base); +// base = cmulFancy(base, w); +// } + + u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data + F2 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + F2 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + F2 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN32) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a + +//GW: Can any of these be Fancy? Yes, u[1] and u[2] + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } +} + //************************************************************************************ // New fft WIDTH and HEIGHT macros to support radix-4 FFTs with more FMA instructions //************************************************************************************ @@ -674,14 +1021,6 @@ void OVERLOAD tabMul(TrigFP32 trig, F2 *u, u32 f, u32 me) { #if ENABLE_FP32_VARIANT_2 -// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. -// We're trying to calculate u * U2(cosine,sine). -// real = (u.x - u.y*sine_over_cosine) * cosine -// imag = (u.x*sine_over_cosine + u.y) * cosine -F2 partial_cmul(F2 u, F sine_over_cosine) { - return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); -} - // Copy of macro from fft4 and fft8 with FMAs added #define X2_via_FMA(a, c, b) { F2 t = a; a = fma(c, b, t); b = fma(-c, b, t); } @@ -898,8 +1237,8 @@ void finish_tabMul8_fft8(TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u3 // Do last level of fft8 X2(u[0], u[1]); X2(u[2], u[3]); - X2_apply_delay(u[4], u[5]); - X2_apply_delay(u[6], u[7]); + X2ad(u[4], u[5], M_SQRT1_2); + X2ad(u[6], u[7], M_SQRT1_2); } // revbin [0, 4, 2, 6, 1, 5, 3, 7] undo @@ -1028,6 +1367,32 @@ void OVERLOAD fft_common(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 low // Finish third tabMul and perform final fft8. finish_tabMul8_fft8(trig, preloads, u, 64, numWG, lowMe, 0); // We'd rather set save_one_more_mul to 1 +// Code for SIZE=256, RADIX=8 +#elif WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + #else // Old / original version @@ -1114,8 +1479,134 @@ void OVERLOAD tabMul(TrigGF31 trig, GF31 *u, u32 f, u32 me) { } } +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(TrigGF31 trig, GF31 *u, u32 f, u32 me) { + + if (f == 1) { // fft8_4 is performed first + u32 p = me; + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN31) { + GF31 w = TFLOAD(&trig[p]); + GF31 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF31 base = csqTrig(w); + GF31 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN31) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } + + else { // fft8_4 is performed after an initial fft8 + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN31) { + GF31 w = TFLOAD(&trig[p]); + GF31 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF31 base = csqTrig(w); + GF31 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN31) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } + } +} + +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(TrigGF31 trig, GF31 *u, u32 f, u32 me) { + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. + + if (TABMUL_CHAIN31) { + u32 p = me & ~(f - 1); + GF31 w = TFLOAD(&trig[p]); + + u[1] = cmul(u[1], w); + GF31 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + GF31 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + GF31 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN31) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a + + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } +} + void OVERLOAD fft_common(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 lowMe) { +// Code for SIZE=256, RADIX=8 +#if WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + +#else + #if !UNROLL __attribute__((opencl_unroll_hint(1))) #endif @@ -1125,6 +1616,9 @@ void OVERLOAD fft_common(local GF31 *lds, GF31 *u, TrigGF31 trig, u32 numWG, u32 shufl(lds, u, s, numWG, lowMe); } fft_RADIX(u); + +#endif + } #endif @@ -1196,8 +1690,135 @@ void OVERLOAD tabMul(TrigGF61 trig, GF61 *u, u32 f, u32 me) { } } +// Tabmul after doing an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4a(TrigGF61 trig, GF61 *u, u32 f, u32 me) { + + if (f == 1) { // fft8_4 is performed first + u32 p = me; + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + if (TABMUL_CHAIN61) { + GF61 w = TFLOAD(&trig[p]); + GF61 w2 = TFLOAD(&trig[WG + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF61 base = csqTrig(w); + GF61 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN61) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG + p])); + } + } + } + + else { // fft8_4 is performed after an initial fft8 + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. +// Perform two length=4 chain muls. + + u32 p = me / 8; // Generate index into condensed trig table that does not have duplicated trig values + trig += 7 * WG; // Skip over the trig values used in the first tabmul + if (TABMUL_CHAIN61) { + GF61 w = TFLOAD(&trig[p]); + GF61 w2 = TFLOAD(&trig[WG/8 + p]); + u[2] = cmul(u[2], w); + u[3] = cmul(u[3], w2); + GF61 base = csqTrig(w); + GF61 base2 = csqTrig(w2); + u[4] = cmul(u[4], base); + u[5] = cmul(u[5], base2); + base = ccubeTrig(base, w); + base2 = ccubeTrig(base2, w2); + u[6] = cmul(u[6], base); + u[7] = cmul(u[7], base2); + } + +// Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. + + if (!TABMUL_CHAIN61) { + for (u32 i = 2; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-2)*WG/8 + p])); + } + } + } +} + +// Later tabmuls after starting with an fft4 when RADIX=8. See the SIZE=256 code for example memory and trig layout. +void OVERLOAD tabMul8_4b(TrigGF61 trig, GF61 *u, u32 f, u32 me) { + +// This code uses chained complex multiplies which could be faster on GPUs with great throughput or poor memory bandwidth or caching. + + if (TABMUL_CHAIN61) { + u32 p = me & ~(f - 1); + GF61 w = TFLOAD(&trig[p]); + + u[1] = cmul(u[1], w); + GF61 w2 = csqTrig(w); + u[2] = cmul(u[2], w2); + GF61 w3 = ccubeTrig(w2, w); + u[3] = cmul(u[3], w3); + GF61 base = cmul(w3, w); + for (int i = 4; i < 8; ++i) { + u[i] = cmul(u[i], base); + base = cmul(base, w); + } + } + +// Theoretically, maximum accuracy. Use memory accesses (probably cached) to reduce complex muls. Beneficial when memory bandwidth is not the bottleneck. +// Radeon VII loves this case, it is faster than the chainmul case. nVidia Titan V hates this case. + + if (!TABMUL_CHAIN61) { + u32 p = (me/4) & ~(f/4 - 1); // Generate index into condensed trig table that does not have duplicated trig values + trig += 6 * WG; // Skip over the trig values used in tabmul8_4a + + for (u32 i = 1; i < RADIX; ++i) { + u[i] = cmul(u[i], TFLOAD(&trig[(i-1)*(WG/4) + p])); + } + } +} + void OVERLOAD fft_common(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 lowMe) { +// Code for SIZE=256, RADIX=8 +#if WG == 32 && NW == 8 + + fft8_4(u); + tabMul8_4a(trig, u, 1, lowMe); + shufl(lds, u, 1, 4, numWG, lowMe); + + fft8(u); + tabMul8_4b(trig, u, 4, lowMe); + shufl(lds, u, 4, numWG, lowMe); + + fft8(u); + +// Code for SIZE=1024, RADIX=8 +#elif WG == 128 && RADIX == 8 + + fft8(u); + tabMul(trig, u, 1, lowMe); + shufl(lds, u, 1, numWG, lowMe); + + fft8(u); + tabMul(trig, u, 8, lowMe); + shufl_and_fft2(lds, u, 8, numWG, lowMe); + + if (lowMe < WG / 2) fft8_16a(u); else fft8_16b(u); + +#else + #if !UNROLL __attribute__((opencl_unroll_hint(1))) #endif @@ -1207,6 +1828,9 @@ void OVERLOAD fft_common(local GF61 *lds, GF61 *u, TrigGF61 trig, u32 numWG, u32 shufl(lds, u, s, numWG, lowMe); } fft_RADIX(u); + +#endif + } #endif diff --git a/src/cl/math.cl b/src/cl/math.cl index a5f44b26..07182c42 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -112,6 +112,7 @@ i128 OVERLOAD sub(i128 a, i64 b) { i128 val; val.x = a.x - (__int128)b; return v u128 OVERLOAD make_u128(u64 hi, u64 lo) { u128 val; val.x = ((unsigned __int128)hi << 64) | lo; return val; } u64 u128_lo64(u128 val) { return val.x; } u64 u128_hi64(u128 val) { return val.x >> 64; } +u64 u128_shrlo64(u128 val, u32 bits) { return val.x >> bits; } u128 OVERLOAD add(u128 a, u128 b) { u128 val; val.x = a.x + b.x; return val; } #else // UNTESTED! typedef struct { i64 hi64; u64 lo64; } i128; @@ -129,6 +130,7 @@ i128 OVERLOAD sub(i128 a, i64 b) { i128 val; val.lo64 = a.lo64 - (u64)b; val.hi6 u128 OVERLOAD make_u128(u64 hi, u64 lo) { u128 val; val.hi64 = hi; val.lo64 = lo; return val; } u64 u128_lo64(u128 val) { return val.lo64; } u64 u128_hi64(u128 val) { return val.hi64; } +u64 u128_shrlo64(u128 val, u32 bits) { return (val.hi64 << (64 - bits)) | (val.lo64 >> bits); } u128 OVERLOAD add(u128 a, u128 b) { u128 val; val.lo64 = a.lo64 + b.lo64; val.hi64 = a.hi64 + b.hi64 + (val.lo64 < a.lo64); return val; } #endif @@ -182,6 +184,8 @@ i32 OVERLOAD optional_mod(i32 a, const i32 b) { return a; } +#define M61 ((((Z61) 1) << 61) - 1) + // Optionally add a constant value if first arg is negative. i64 OVERLOAD optional_addM61(i64 a) { #if HAS_PTX >= 100 // setp/add instruction requires sm_10 support or higher @@ -190,7 +194,7 @@ i64 OVERLOAD optional_addM61(i64 a) { " @%%p add.s64 %0, %0, 2305843009213693951;}" // if (a < 0) a = a + M61 : "+l"(a)); #else - if (a < 0) a = a + 2305843009213693951; + if (a < 0) a = a + M61; #endif return a; } @@ -258,7 +262,7 @@ i64 OVERLOAD optional_sub(i64 a, const i32 b, const i64 c) { // Multiply and add primitives -u64 OVERLOAD mul3264(u32 a, u64 b) { // 3 32-bit multiplies (instead of 4 for a u64 * u64 multiply) +u64 OVERLOAD mul3264(u32 a, u64 b) { // 3 32-bit multiplies (instead of 4 for a u32 * u64 multiply) u32 blo = lo32(b); u32 bhi = hi32(b); return make_u64(mul_hi(a, blo) + a * bhi, a * blo); @@ -300,12 +304,12 @@ u128 OVERLOAD mul64(u64 a, u64 b) { uint2 b2 = as_uint2(b); uint2 rlo2, rhi2; __asm("mul.lo.u32 %0, %4, %6;\n\t" - "mul.hi.u32 %1, %4, %6;\n\t" + "mul.hi.u32 %1, %4, %6;\n\t" "mul.lo.u32 %2, %5, %7;\n\t" - "mad.lo.cc.u32 %1, %5, %6, %1;\n\t" + "mad.lo.cc.u32 %1, %5, %6, %1;\n\t" "madc.hi.cc.u32 %2, %5, %6, %2;\n\t" "madc.hi.u32 %3, %5, %7, 0;\n\t" - "mad.lo.cc.u32 %1, %4, %7, %1;\n\t" + "mad.lo.cc.u32 %1, %4, %7, %1;\n\t" "madc.hi.cc.u32 %2, %4, %7, %2;\n\t" "addc.u32 %3, %3, 0;" : "=r"(rlo2.x), "=r"(rlo2.y), "=r"(rhi2.x), "=r"(rhi2.y) @@ -375,10 +379,11 @@ u128 OVERLOAD mad64(u64 a, u64 b, u128 c) { #endif } - // The X2 family of macros and SWAP are #defines because OpenCL does not allow pass by reference. // With NTT support added, we need to turn these macros into overloaded routines. #define X2(a, b) X2_internal(&(a), &(b)) // a = a + b, b = a - b +#define X2t4(a, b) X2t4_internal(&(a), &(b)) // X2(a, mul_t4(b)) +#define X2t4_mul_t4(a, b) X2t4_mul_t4_internal(&(a), &(b)) // X2(a, mul_t4(b)), b = mul_t4(b) #define X2conjb(a, b) X2conjb_internal(&(a), &(b)) // X2(a, conjugate(b)) #define X2_mul_t4(a, b) X2_mul_t4_internal(&(a), &(b)) // X2(a, b), b = mul_t4(b) #define X2_mul_t8(a, b) X2_mul_t8_internal(&(a), &(b)) // X2(a, b), b = mul_t8(b) @@ -386,9 +391,18 @@ u128 OVERLOAD mad64(u64 a, u64 b, u128 c) { #define X2_conjb(a, b) X2_conjb_internal(&(a), &(b)) // X2(a, b), b = conjugate(b) #define SWAP(a, b) SWAP_internal(&(a), &(b)) // a = b, b = a #define SWAP_XY(a) U2((a).y, (a).x) // Swap real and imaginary components of a +// Macros for FP64 and FP32 only. They allow some optimizations using FMA. NOTE: "ad" stands for "apply delayed" mul (see partial_cmul). +#define X2ad(a, b, d) X2ad_internal(&(a), &(b), d) // b = d * b, X2(a, b) +#define X2t4ad(a, b, d) X2t4ad_internal(&(a), &(b), d) // b = mul_t4(b), b = d * b, X2(a, b) +#define X2ad_mul_t4(a, b, d) X2ad_mul_t4_internal(&(a), &(b), d) // b = d * b, X2(a, b), b = mul_t4(b) #if FFT_FP64 +T OVERLOAD add(T a, T b) { return a + b; } +T2 OVERLOAD add(T2 a, T2 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } +T OVERLOAD sub(T a, T b) { return a - b; } +T2 OVERLOAD sub(T2 a, T2 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } + T2 OVERLOAD conjugate(T2 a) { return U2(a.x, -a.y); } // Multiply by 2 without using floating point instructions. This is a little sloppy as an input of zero returns 2^-1022. @@ -469,12 +483,18 @@ T2 OVERLOAD mul_3t8(T2 a) { // mul(a, U2(-1, 1)) * (T)(M_SQRT1_2); } // Return a+b and a-b void OVERLOAD X2_internal(T2 *a, T2 *b) { T2 t = *a; *a = t + *b; *b = t - *b; } -// Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(T2 *a, T2 *b) { T2 t = *a; *a = *a + *b; t.x = t.x - b->x; b->x = b->y - t.y; b->y = t.x; } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(T2 *a, T2 *b) { T by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + +// Same as X2(a, mul_t4(b)), b = mul_t4(b) +void OVERLOAD X2t4_mul_t4_internal(T2 *a, T2 *b) { T2 t = *a; a->x = sub(a->x, b->y); a->y = add(a->y, b->x); b->x = sub(b->x, t.y); b->y = add(t.x, b->y); } // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(T2 *a, T2 *b) { T2 t = *a; a->x = a->x + b->x; a->y = a->y - b->y; b->x = t.x - b->x; b->y = t.y + b->y; } +// Same as X2(a, b), b = mul_t4(b) +void OVERLOAD X2_mul_t4_internal(T2 *a, T2 *b) { T by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } + // Same as X2(a, b), b = conjugate(b) void OVERLOAD X2_conjb_internal(T2 *a, T2 *b) { T2 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } @@ -494,6 +514,29 @@ T2 OVERLOAD foo2(T2 a, T2 b) { a = addsub(a); b = addsub(b); return addsub(U2(RE // computes 2*[x^2+y^2 + i*(2*x*y)]. i.e. 2 * cyclical autoconvolution of (x, y) T2 OVERLOAD foo(T2 a) { return foo2(a, a); } +// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. +// We're trying to calculate u * U2(cosine,sine). Instead calculate u * U2(1,sine/cosine). +// real = (u.x - u.y*sine_over_cosine) * cosine +// imag = (u.x*sine_over_cosine + u.y) * cosine +T2 partial_cmul(T2 u, T sine_over_cosine) { + return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); +} + +T2 mul_t8_delayed(T2 a) { return U2(a.x - a.y, a.x + a.y); } // Apply mul by M_SQRT1_2 later +T2 mul_3t8_delayed(T2 a) { return U2(-(a.x + a.y), a.x - a.y); } // Apply mul by M_SQRT1_2 later. Alternatively, use mul_t8_delayed and mul by i*M_SQRT1_2 later. + +// Compute a + d * b and a - d * b +void X2ad_internal(T2 *a, T2 *b, T d) { T2 t = *a; a->x = fma(b->x, d, a->x); a->y = fma(b->y, d, a->y); b->x = fma(-d, b->x, t.x); b->y = fma(-d, b->y, t.y); } +void X2t4ad_internal(T2 *a, T2 *b, T d) { T bx = b->x; b->x = fma(d, b->y, a->x); a->x = fma(b->y, -d, a->x); b->y = fma(-d, bx, a->y); a->y = fma(bx, d, a->y); } +void X2ad_mul_t4_internal(T2 *a, T2 *b, T d) { T by = b->y; b->y = fma(-d, b->x, a->x); a->x = fma(b->x, d, a->x); b->x = -fma(-d, by, a->y); a->y = fma(by, d, a->y); } + +// Create "quick" routines for compatibility with shufl_and_fft2 and the GF61 data type. + +T OVERLOAD addq(T a, T b) { return add(a, b); } +T OVERLOAD subq(T a, T b) { return sub(a, b); } +T2 OVERLOAD addq(T2 a, T2 b) { return add(a, b); } +T2 OVERLOAD subq(T2 a, T2 b) { return sub(a, b); } + #endif @@ -503,6 +546,11 @@ T2 OVERLOAD foo(T2 a) { return foo2(a, a); } #if FFT_FP32 +F OVERLOAD add(F a, F b) { return a + b; } +F2 OVERLOAD add(F2 a, F2 b) { return U2(add(a.x, b.x), add(a.y, b.y)); } +F OVERLOAD sub(F a, F b) { return a - b; } +F2 OVERLOAD sub(F2 a, F2 b) { return U2(sub(a.x, b.x), sub(a.y, b.y)); } + F2 OVERLOAD conjugate(F2 a) { return U2(a.x, -a.y); } // Multiply by 2 without using floating point instructions. This is a little sloppy as an input of zero returns 2^-126. @@ -583,12 +631,18 @@ F2 OVERLOAD mul_3t8(F2 a) { // mul(a, U2(-1, 1)) * (F)(M_SQRT1_2); } // Return a+b and a-b void OVERLOAD X2_internal(F2 *a, F2 *b) { F2 t = *a; *a = t + *b; *b = t - *b; } -// Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(F2 *a, F2 *b) { F2 t = *a; *a = *a + *b; t.x = t.x - b->x; b->x = b->y - t.y; b->y = t.x; } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(F2 *a, F2 *b) { F by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + +// Same as X2(a, mul_t4(b)), b = mul_t4(b) +void OVERLOAD X2t4_mul_t4_internal(F2 *a, F2 *b) { F2 t = *a; a->x = sub(a->x, b->y); a->y = add(a->y, b->x); b->x = sub(b->x, t.y); b->y = add(t.x, b->y); } // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(F2 *a, F2 *b) { F2 t = *a; a->x = a->x + b->x; a->y = a->y - b->y; b->x = t.x - b->x; b->y = t.y + b->y; } +// Same as X2(a, b), b = mul_t4(b) +void OVERLOAD X2_mul_t4_internal(F2 *a, F2 *b) { F by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } + // Same as X2(a, b), b = conjugate(b) void OVERLOAD X2_conjb_internal(F2 *a, F2 *b) { F2 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } @@ -608,6 +662,29 @@ F2 OVERLOAD foo2(F2 a, F2 b) { a = addsub(a); b = addsub(b); return addsub(U2(RE // computes 2*[x^2+y^2 + i*(2*x*y)]. i.e. 2 * cyclical autoconvolution of (x, y) F2 OVERLOAD foo(F2 a) { return foo2(a, a); } +// Partial complex-multiply that delays the mul-by-cosine so it can be part of an FMA. +// We're trying to calculate u * U2(cosine,sine). Instead calculate u * U2(1,sine/cosine). +// real = (u.x - u.y*sine_over_cosine) * cosine +// imag = (u.x*sine_over_cosine + u.y) * cosine +F2 partial_cmul(F2 u, F sine_over_cosine) { + return U2(fma(-u.y, sine_over_cosine, u.x), fma(u.x, sine_over_cosine, u.y)); +} + +F2 mul_t8_delayed(F2 a) { return U2(a.x - a.y, a.x + a.y); } // Apply mul by M_SQRT1_2 later +F2 mul_3t8_delayed(F2 a) { return U2(-(a.x + a.y), a.x - a.y); } // Apply mul by M_SQRT1_2 later. Alternatively, use mul_t8_delayed and mul by i*M_SQRT1_2 later. + +// Compute a + d * b and a - d * b +void X2ad_internal(F2 *a, F2 *b, F d) { F2 t = *a; a->x = fma(b->x, d, a->x); a->y = fma(b->y, d, a->y); b->x = fma(-d, b->x, t.x); b->y = fma(-d, b->y, t.y); } +void X2t4ad_internal(F2 *a, F2 *b, F d) { F bx = b->x; b->x = fma(d, b->y, a->x); a->x = fma(b->y, -d, a->x); b->y = fma(-d, bx, a->y); a->y = fma(bx, d, a->y); } +void X2ad_mul_t4_internal(F2 *a, F2 *b, F d) { F by = b->y; b->y = fma(-d, b->x, a->x); a->x = fma(b->x, d, a->x); b->x = -fma(-d, by, a->y); a->y = fma(by, d, a->y); } + +// Create "quick" routines for compatibility with shufl_and_fft2 and the GF61 data type. + +F OVERLOAD addq(F a, F b) { return add(a, b); } +F OVERLOAD subq(F a, F b) { return sub(a, b); } +F2 OVERLOAD addq(F2 a, F2 b) { return add(a, b); } +F2 OVERLOAD subq(F2 a, F2 b) { return sub(a, b); } + #endif @@ -682,11 +759,14 @@ GF31 OVERLOAD mul_3t8(GF31 a) { return U2(shl(neg(add(a.x, a.y)), 15), shl(sub(a // Return a+b and a-b void OVERLOAD X2_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(t, *b); *b = sub(t, *b); } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(GF31 *a, GF31 *b) { GF31 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } // Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } +void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } // Same as X2(a, b), b = mul_t8(b) void OVERLOAD X2_mul_t8_internal(GF31 *a, GF31 *b) { X2(*a, *b); *b = mul_t8(*b); } @@ -727,6 +807,19 @@ Z31 OVERLOAD modM31(u64 a) { // a must u32 ahi = a >> 62; return modM31(ahi + amid + alo); // 32-bit overflow does not occur due to restrictions on input } +Z31 OVERLOAD modM31(u64 a, u32 maxbits) { + if (maxbits <= 62) { + u32 alo = lo32(a) & M31; // 31 bits + u32 ahi = hi32(a + a); // 31 bits + return modM31(ahi + alo); + } + else if (maxbits == 63) { + u32 alo = lo32(a) & M31; // 31 bits + u32 ahi = hi32(a + a); // 32 bits + return modM31(modM31(ahi) + alo); + } + return modM31(a); +} Z31 OVERLOAD modM31(i64 a) { // abs(a) must be less than 0x7FFFFFFF80000000 u32 alo = a & M31; u32 amid = ((u64) a >> 31) & M31; // Unsigned shift might be faster than signed shift @@ -761,7 +854,7 @@ GF31 OVERLOAD shr(GF31 a, u32 k) { return U2(shr(a.x, k), shr(a.y, k)); } Z31 OVERLOAD shl(Z31 a, u32 k) { return shr(a, 31 - k); } GF31 OVERLOAD shl(GF31 a, u32 k) { return U2(shl(a.x, k), shl(a.y, k)); } -Z31 OVERLOAD mul(Z31 a, Z31 b) { u64 t = a * (u64) b; return modM31(add((Z31)(t & M31), (Z31)(t >> 31))); } +Z31 OVERLOAD mul(Z31 a, Z31 b) { u64 t = a * (u64) b; return modM31(t, 62); } // Multiply by 2 Z31 OVERLOAD mul2(Z31 a) { return add(a, a); } @@ -771,39 +864,79 @@ GF31 OVERLOAD mul2(GF31 a) { return U2(mul2(a.x), mul2(a.y)); } GF31 OVERLOAD conjugate(GF31 a) { return U2(a.x, neg(a.y)); } // Complex square. input, output 31 bits. Uses (a + i*b)^2 == ((a+b)*(a-b) + i*2*a*b). +#if 0 GF31 OVERLOAD csq(GF31 a) { - u64 r = (a.x + a.y) * (u64) (a.x + neg(a.y)); // 64-bit value, max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = (a.x + a.x) * (u64) a.y; // 63-bit value, max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = (a.x + a.y) * (u64) (a.x + neg(a.y)); // 64-bit value, max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = (a.x + a.x) * (u64) a.y; // 63-bit value, max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); } +#else +GF31 OVERLOAD csq(GF31 a) { + u64 r = mad32(a.x, a.x, neg(a.y) * (u64) a.y); // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + u64 i = (a.x + a.x) * (u64) a.y; // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + return U2(modM31(r, 63), modM31(i, 63)); +} +#endif // a^2 + c +#if 0 +GF31 OVERLOAD csq_add(GF31 a, GF31 c) { + u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.x); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, c.y); // 63-bit value, mul max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); +} +#else GF31 OVERLOAD csq_add(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.x); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, c.y); // 63-bit value, mul max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, c.x)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, c.y); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); } +#endif // a^2 - c +#if 0 GF31 OVERLOAD csq_sub(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.x)); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, neg(c.y)); // 63-bit value, mul max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.x)); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, neg(c.y)); // 63-bit value, mul max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); } +#else +GF31 OVERLOAD csq_sub(GF31 a, GF31 c) { + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, neg(c.x))); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, neg(c.y)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); +} +#endif // a^2 + i*c +#if 0 +GF31 OVERLOAD csq_addi(GF31 a, GF31 c) { + u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.y)); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, c.x); // 63-bit value, mul max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); +} +#else GF31 OVERLOAD csq_addi(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), neg(c.y)); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, c.x); // 63-bit value, mul max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, neg(c.y))); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, c.x); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); } +#endif // a^2 - i*c +#if 0 +GF31 OVERLOAD csq_subi(GF31 a, GF31 c) { + u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.y); // 64-bit value, mul max = FFFF FFFE 8000 0003 (actually cannot exceed 9000 0000 0000 0000) + u64 i = mad32(a.x + a.x, a.y, neg(c.x)); // 63-bit value, max = 7FFF FFFE 8000 0001 + return U2(modM31(r), modM31(i, 63)); +} +#else GF31 OVERLOAD csq_subi(GF31 a, GF31 c) { - u64 r = mad32(a.x + a.y, a.x + neg(a.y), c.y); // 64-bit value, mul max = FFFF FFFE 0000 0004 (actually cannot exceed 9000 0000 0000 0000) - u64 i = mad32(a.x + a.x, a.y, neg(c.x)); // 63-bit value, max = 7FFF FFFE 0000 0002 - return U2(modM31(r), modM31(i)); + u64 r = mad32(a.x, a.x, mad32(neg(a.y), a.y, c.y)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + u64 i = mad32(a.x + a.x, a.y, neg(c.x)); // Max value is 2*M31^2+M31 = 7FFF FFFE 8000 0001 + return U2(modM31(r, 63), modM31(i, 63)); } +#endif // Complex mul #if 0 // One less negation, requires signed shifts. Seems microscopically faster on TitanV. @@ -815,7 +948,7 @@ GF31 OVERLOAD cmul(GF31 a, GF31 b) { u64 k1k2 = k1 + k2; // unsigned 64-bit value, max = FFFF FFFC 0000 0004 return U2(modM31(k1k3), modM31(k1k2)); } -#else +#elif 0 GF31 OVERLOAD cmul(GF31 a, GF31 b) { u32 negbx = neg(b.x); // Negate and add b values as much as possible in case b is used several times (as in a chainmul) u64 k1 = b.x * (u64)(a.x + a.y); // 63-bit value, max = 7FFF FFFE 0000 0002 @@ -823,8 +956,29 @@ GF31 OVERLOAD cmul(GF31 a, GF31 b) { u64 k1k3 = mad32(a.y, neg(b.y) + negbx, k1); // unsigned 64-bit value, max = FFFF FFFC 0000 0004 return U2(modM31(k1k3), modM31(k1k2)); } +#else // Straight forward 4 multiply version +GF31 OVERLOAD cmul(GF31 a, GF31 b) { + u64 ayby = (u64) a.y * (u64) neg(b.y); + u64 aybx = (u64) a.y * (u64) b.x; + u64 r = mad32(a.x, b.x, ayby); // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + u64 i = mad32(a.x, b.y, aybx); // Max value is 2*M31^2 = 7FFF FFFE 0000 0002 + return U2(modM31(r, 63), modM31(i, 63)); +} #endif +// Complex mul where 2nd argument is a constant. Allows cheaper modM31 in some cases. +GF31 OVERLOAD cmul_const(GF31 a, GF31 b) { +#if 0 + return cmul(a, b); +#else + u64 ayby = (u64) a.y * (u64) neg(b.y); + u64 aybx = (u64) a.y * (u64) b.x; + u64 r = mad32(a.x, b.x, ayby); // Max value is (M31 - b.y + b.x)*M31 + u64 i = mad32(a.x, b.y, aybx); // Max value is (b.x + b.y)*M31 + return U2(modM31(r, (M31 - b.y + b.x <= M31) ? 62 : 63), modM31(i, (b.x + b.y <= M31) ? 62 : 63)); +#endif +} + // Square a root of unity complex number GF31 OVERLOAD csqTrig(GF31 a) { u32 two_ay = a.y + a.y; return U2(modM31(mad32(two_ay, neg(a.y), (u32)1)), modM31(a.x * (u64)two_ay)); } @@ -843,11 +997,14 @@ GF31 OVERLOAD mul_3t8(GF31 a) { return U2(shl(neg(add(a.x, a.y)), 15), shl(sub(a // Return a+b and a-b void OVERLOAD X2_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(t, *b); *b = sub(t, *b); } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(GF31 *a, GF31 *b) { GF31 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } // Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { GF31 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } +void OVERLOAD X2_mul_t4_internal(GF31 *a, GF31 *b) { Z31 by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } // Same as X2(a, b), b = mul_t8(b) void OVERLOAD X2_mul_t8_internal(GF31 *a, GF31 *b) { X2(*a, *b); *b = mul_t8(*b); } @@ -866,6 +1023,13 @@ GF31 OVERLOAD foo(GF31 a) { return foo2(a, a); } #endif +// Create "quick" routines for compatibility with shufl_and_fft2 and the GF61 data type. + +Z31 OVERLOAD addq(Z31 a, Z31 b) { return add(a, b); } +Z31 OVERLOAD subq(Z31 a, Z31 b) { return sub(a, b); } +GF31 OVERLOAD addq(GF31 a, GF31 b) { return add(a, b); } +GF31 OVERLOAD subq(GF31 a, GF31 b) { return sub(a, b); } + #endif @@ -875,8 +1039,6 @@ GF31 OVERLOAD foo(GF31 a) { return foo2(a, a); } #if NTT_GF61 -#define M61 ((((Z61) 1) << 61) - 1) - Z61 OVERLOAD make_Z61(i32 a) { return (Z61) (a < 0 ? (i64) a + M61 : (i64) a); } // Handles all values of a Z61 OVERLOAD make_Z61(i64 a) { return (Z61) optional_addM61(a); } // a must be in range of -M61 .. M61-1 Z61 OVERLOAD make_Z61(u32 a) { return (Z61) (a); } // Handles all values of a @@ -914,9 +1076,82 @@ Z61 OVERLOAD shl(Z61 a, u32 k) { return shr(a, 61 - k); } // Return rang //Z61 OVERLOAD shl(Z61 a, u32 k) { return modM61((a << k) + ((a >> (64 - k)) << 3)); } // Return range 0..M61+epsilon, input must be M61+epsilon a full 62-bit value can overflow GF61 OVERLOAD shl(GF61 a, u32 k) { return U2(shl(a.x, k), shl(a.y, k)); } -ulong2 wideMul(u64 ab, u64 cd) { - u128 r = mul64(ab, cd); - return U2(u128_lo64(r), u128_hi64(r)); +// Maybe we can make shl faster +Z61 OVERLOAD shl30(Z61 a) { +#if TRY_SHL30 && HAS_PTX >= 320 // shf instruction requires sm_32 support or higher + uint2 b = as_uint2(a); + uint2 r, tmp; + __asm("shf.r.clamp.b32 %0, %4, %5, 31;\n\t" // High 32 bits (wrap to LSW) + "shr.b32 %1, %5, 31;\n\t" // This is needed unless we can assume highest bit is zero (wrap to MSW) + "and.b32 %2, %4, 2147483647;\n\t" // Low 31 bits + "shr.b32 %3, %2, 2;\n\t" // Shift left 30 (29 bits end up in MSW) + "shl.b32 %2, %2, 30;\n\t" // Shift left 30 (2 bits end up in LSW) + "add.cc.u32 %0, %0, %2;\n\t" + "addc.u32 %1, %1, %3;" // Use if highest bit may not be zero + //"addc.u32 %1, 0, %3;" // Use if highest bit is assumed to be zero + : "=r"(r.x), "=r"(r.y), "=r"(tmp.x), "=r"(tmp.y) : "r"(b.x), "r"(b.y) : ); + return as_ulong(r); +#else + return shl(a, 30); +#endif +} +GF61 OVERLOAD shl30(GF61 a) { return U2(shl30(a.x), shl30(a.y)); } + +// Maybe we can make shl faster +Z61 OVERLOAD shl31(Z61 a) { +#if TRY_SHL31 && HAS_PTX >= 320 // shf instruction requires sm_32 support or higher + uint2 b = as_uint2(a); + uint2 r, tmp; + __asm("shf.r.clamp.b32 %0, %4, %5, 30;\n\t" // High 32 bits (wrap to LSW) + //"shr.b32 %1, %5, 30;\n\t" // Assume highest 2 bits are zero (wrap to MSW) + "and.b32 %2, %4, 1073741823;\n\t" // Low 30 bits + "shr.b32 %3, %2, 1;\n\t" // Shift left 31 (29 bits end up in MSW) + "shl.b32 %2, %2, 31;\n\t" // Shift left 31 (1 bit ends up in LSW) + "add.cc.u32 %0, %0, %2;\n\t" + //"addc.u32 %1, %1, %3;" // Use if highest 2 bits may not be zero + "addc.u32 %1, 0, %3;" + : "=r"(r.x), "=r"(r.y), "=r"(tmp.x), "=r"(tmp.y) : "r"(b.x), "r"(b.y) : ); + return as_ulong(r); +#else + return shl(a, 31); +#endif +} +GF61 OVERLOAD shl31(GF61 a) { return U2(shl31(a.x), shl31(a.y)); } + +u64 OVERLOAD weakModM61(u128 a, u32 num_bits) { +// This is faster on TitanV, CUDA 13.0. No difference on 5070Ti. +#if HAS_PTX >= 320 // shf instruction requires sm_32 support or higher + uint2 alo = as_uint2(u128_lo64(a)); + uint2 ahi = as_uint2(u128_hi64(a)); + if (num_bits <= 125) { + __asm("shf.r.clamp.b32 %3, %2, %3, 29;\n\t" + "shf.r.clamp.b32 %2, %1, %2, 29;\n\t" + "and.b32 %1, %1, 536870911;" + : "+r"(alo.x), "+r"(alo.y), "+r"(ahi.x), "+r"(ahi.y) : ); + return (u64)as_ulong(ahi) + (u64)as_ulong(alo); + } else { + uint top6; + __asm("shr.u32 %4, %3, 26;\n\t" + "shf.r.clamp.b32 %3, %2, %3, 29;\n\t" + "shf.r.clamp.b32 %2, %1, %2, 29;\n\t" + "and.b32 %1, %1, 536870911;\n\t" + "and.b32 %3, %3, 536870911;\n\t" + "add.cc.u32 %0, %0, %4;\n\t" + "addc.u32 %1, %1, 0;" + : "+r"(alo.x), "+r"(alo.y), "+r"(ahi.x), "+r"(ahi.y), "=r"(top6) : ); + return (u64)as_ulong(ahi) + (u64)as_ulong(alo); + } +#else + u64 lo = u128_lo64(a), hi = u128_hi64(a); + u64 lo61 = lo & M61; // Max value is M61 + if (num_bits <= 125) { + hi = (hi << 3) + (lo >> 61); + return lo61 + hi; // Caller must insure this does not overflow + } else { + u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 + return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + } +#endif } // Returns a * b not modded by M61. Max value of result depends on the m61_counts of the inputs. @@ -924,39 +1159,27 @@ ulong2 wideMul(u64 ab, u64 cd) { // If n <= 6 result will be at most (n+1)*M61+epsilon. // If n > 6 result will be at most 2*M61+epsilon. Z61 OVERLOAD weakMul(Z61 a, Z61 b, const u32 a_m61_count, const u32 b_m61_count) { - ulong2 ab = wideMul(a, b); - u64 lo = ab.x, hi = ab.y; - u64 lo61 = lo & M61; // Max value is M61 + u128 ab = mul64(a, b); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61^2 + epsilon if ((a_m61_count - 1) * (b_m61_count - 1) <= 6) { - hi = (hi << 3) + (lo >> 61); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61 + epsilon - return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon + return weakModM61(ab, 125); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon } else { - u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 - return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + return weakModM61(ab, 128); // Max value is 2*M61 + epsilon } } Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u64 c, const u32 a_m61_count, const u32 b_m61_count) { u128 ab = mad64(a, b, c); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61^2 + epsilon - u64 lo = u128_lo64(ab), hi = u128_hi64(ab); - u64 lo61 = lo & M61; // Max value is M61 if ((a_m61_count - 1) * (b_m61_count - 1) <= 6) { - hi = (hi << 3) + (lo >> 61); // Max value is (a_m61_count - 1) * (b_m61_count - 1) * M61 + epsilon - return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon + return weakModM61(ab, 125); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 1) * M61 + epsilon } else { - u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 - return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + return weakModM61(ab, 128); // Max value is 2*M61 + epsilon } } Z61 OVERLOAD weakMulAdd(Z61 a, Z61 b, u128 c, const u32 a_m61_count, const u32 b_m61_count) { // Max c value assumed to be 2*M61^2+epsilon u128 ab = mad64(a, b, c); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61^2 + epsilon - u64 lo = u128_lo64(ab), hi = u128_hi64(ab); - u64 lo61 = lo & M61; // Max value is M61 if ((a_m61_count - 1) * (b_m61_count - 1) + 2 <= 6) { - hi = (hi << 3) + (lo >> 61); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 2) * M61 + epsilon - return lo61 + hi; // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 3) * M61 + epsilon + return weakModM61(ab, 125); // Max value is ((a_m61_count - 1) * (b_m61_count - 1) + 3) * M61 + epsilon } else { - u64 hi61 = ((hi << 3) + (lo >> 61)) & M61; // Max value is M61 - return lo61 + hi61 + (hi >> 58); // Max value is 2*M61 + epsilon + return weakModM61(ab, 128); // Max value is 2*M61 + epsilon } } @@ -1003,12 +1226,22 @@ GF61 OVERLOAD csqa(GF61 a, GF61 c, const u32 m61_count) { return csqa(a, c, m61_ GF61 OVERLOAD csqa(GF61 a, GF61 c) { return csqa(a, c, 2); } // Complex mul +#if 1 GF61 OVERLOAD cmul(GF61 a, GF61 b) { u128 k1 = mul64(b.x, a.x + a.y); // max value is 2*M61^2+epsilon Z61 k1k2 = weakMulAdd(a.x, b.y + neg(b.x, 2), k1, 2, 4); // max value is 6*M61+epsilon Z61 k1k3 = weakMulAdd(a.y, neg(b.y + b.x, 3), k1, 2, 4); // max value is 6*M61+epsilon return U2(modM61(k1k3), modM61(k1k2)); } +#else +GF61 OVERLOAD cmul(GF61 a, GF61 b) { + u128 ayby = mul64(a.y, neg(b.y, 2)); // max value is 2*M61^2+epsilon + u128 r = mad64(a.x, b.x, ayby); // max value is 3*M61^2+epsilon + u128 aybx = mul64(a.y, b.x); // max value is 1*M61^2+epsilon + u128 i = mad64(a.x, b.y, aybx); // max value is 2*M61^2+epsilon + return U2(add(u128_lo64(r) & M61, u128_shrlo64(r, 61)), add(u128_lo64(i) & M61, u128_shrlo64(i, 61))); +} +#endif // Square a root of unity complex number (the second version may be faster if the compiler optimizes the u128 squaring). //GF61 OVERLOAD csqTrig(GF61 a) { Z61 two_ay = a.y + a.y; return U2(modM61(1 + weakMul(two_ay, neg(a.y, 2))), mul(a.x, two_ay)); } @@ -1026,24 +1259,77 @@ GF61 OVERLOAD subi(GF61 a, GF61 b) { return U2(add(a.x, b.y), sub(a.y, b.x)); } GF61 OVERLOAD mul_t4(GF61 a) { return U2(neg(a.y), a.x); } // GWBUG: Can caller use a version that does not negate real? // mul with (-2^30, -2^30). (twiddle of tau/8 aka sqrt(i)). Note: 2 * (+/-2^30)^2 == 1 (mod M61). -GF61 OVERLOAD mul_t8(GF61 a, const u32 m61_count) { return shl(U2(a.y + neg(a.x, m61_count), neg(a.x + a.y, 2 * m61_count - 1)), 30); } +GF61 OVERLOAD mul_t8(GF61 a, const u32 m61_count) { return shl30(U2(a.y + neg(a.x, m61_count), neg(a.x + a.y, 2 * m61_count - 1))); } GF61 OVERLOAD mul_t8(GF61 a) { return mul_t8(a, 2); } // mul with (2^30, -2^30). (twiddle of 3*tau/8). -GF61 OVERLOAD mul_3t8(GF61 a, const u32 m61_count) { return shl(U2(a.x + a.y, a.y + neg(a.x, m61_count)), 30); } +GF61 OVERLOAD mul_3t8(GF61 a, const u32 m61_count) { return shl30(U2(a.x + a.y, a.y + neg(a.x, m61_count))); } GF61 OVERLOAD mul_3t8(GF61 a) { return mul_3t8(a, 2); } +// mul with twiddles of (1,3,5,7,9)*tau/16 +// Define C1 = 22027337052962166, S1 = 1693317751237720973, negC1 = M61 - C1, negS1 = M61 - S1. +// Twiddle for 1t16 is U2(C1, S1), 3t16 is U2(S1, C1), 5t16 is U2(negS1, C1), 7t16 is U2(negC1, S1), 9t16 is U2(negC1, negS1). +#if TRY_SQRT2 +// NOTE: C1/S1 = SQRT2 + 1 and S1/C1 = SQRT2 - 1. This lets us compute the cmul in two steps. For example, mul_t16 can be (a * U2(1, S1/C1)) * C1. +// Mul by SQRT2 is a mul by -2^31 which is done with a shift rather than a multiply. This reduces the total number of 64-bit multiplies (but shifts aren't cheap either). +// As a bonus, C1 is a 55-bit value which means a modM61 reduction after weakMul should not be necessary. +GF61 OVERLOAD mul_t16(GF61 a) { + // a * U2(1, S1/C1) = axbx - ayby, axby + aybx + // = ax - (ay * (SQRT2 - 1)), ax * (SQRT2 - 1) + ay + // = ax - ay * SQRT2 + ay, ax * SQRT2 - ax + ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(a.x + a_negsqrt2.y + a.y, 22027337052962166ULL, 2, 2), weakMul(neg(a_negsqrt2.x + a.x, 3) + a.y, 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_3t16(GF61 a) { + // a * U2(S1/C1, 1) = axbx - ayby, axby + aybx + // = ax * (SQRT2 - 1) - ay, ax + ay * (SQRT2 - 1) + // = ax * SQRT2 - ax - ay, ax + ay * SQRT2 - ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(neg(a_negsqrt2.x + a.x + a.y, 4), 22027337052962166ULL, 2, 2), weakMul(a.x + neg(a_negsqrt2.y + a.y, 3), 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_5t16(GF61 a) { + // a * U2(-S1/C1, 1) = axbx - ayby, axby + aybx + // = ax * -(SQRT2 - 1) - ay, ax + ay * -(SQRT2 - 1) + // = -ax * SQRT2 + ax - ay, ax - ay * SQRT2 + ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(a_negsqrt2.x + a.x + neg(a.y, 2), 22027337052962166ULL, 2, 2), weakMul(a.x + a_negsqrt2.y + a.y, 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_7t16(GF61 a) { + // a * U2(-1, S1/C1) = axbx - ayby, axby + aybx + // = -ax - (ay * (SQRT2 - 1)), ax * (SQRT2 - 1) - ay + // = -ax - ay * SQRT2 + ay, ax * SQRT2 - ax - ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(neg(a.x, 2) + a_negsqrt2.y + a.y, 22027337052962166ULL, 2, 2), weakMul(neg(a_negsqrt2.x + a.x + a.y, 4), 22027337052962166ULL, 2, 2)); +} +GF61 OVERLOAD mul_9t16(GF61 a) { + // a * U2(-1, -S1/C1) = axbx - ayby, axby + aybx + // = -ax - (ay * -(SQRT2 - 1)), ax * -(SQRT2 - 1) - ay + // = -ax + ay * SQRT2 - ay, -ax * SQRT2 + ax - ay + GF61 a_negsqrt2 = shl31(a); // Mul by (2^31 = -SQRT2) + return U2(weakMul(neg(a.x + a_negsqrt2.y + a.y, 4), 22027337052962166ULL, 2, 2), weakMul(a_negsqrt2.x + a.x + neg(a.y, 2), 22027337052962166ULL, 2, 2)); +} +#else +GF61 OVERLOAD mul_t16(GF61 a) { return cmul(a, U2(22027337052962166ULL, 1693317751237720973ULL)); } +GF61 OVERLOAD mul_3t16(GF61 a) { return cmul(a, U2(1693317751237720973ULL, 22027337052962166ULL)); } +GF61 OVERLOAD mul_5t16(GF61 a) { return cmul(a, U2(M61 - 1693317751237720973ULL, 22027337052962166ULL)); } +GF61 OVERLOAD mul_7t16(GF61 a) { return cmul(a, U2(M61 - 22027337052962166ULL, 1693317751237720973ULL)); } +GF61 OVERLOAD mul_9t16(GF61 a) { return cmul(a, U2(M61 - 22027337052962166ULL, M61 - 1693317751237720973ULL)); } +#endif + // Return a+b and a-b void OVERLOAD X2_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); *b = sub(t, *b); } +// Same as X2(a, mul_t4(b)) +void OVERLOAD X2t4_internal(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = sub(a->y, b->x); a->y = add(a->y, b->x); b->x = add(a->x, by); a->x = sub(a->x, by); } + // Same as X2(a, conjugate(b)) void OVERLOAD X2conjb_internal(GF61 *a, GF61 *b) { GF61 t = *a; a->x = add(a->x, b->x); a->y = sub(a->y, b->y); b->x = sub(t.x, b->x); b->y = add(t.y, b->y); } // Same as X2(a, b), b = mul_t4(b) -void OVERLOAD X2_mul_t4_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(*a, *b); t.x = sub(t.x, b->x); b->x = sub(b->y, t.y); b->y = t.x; } +void OVERLOAD X2_mul_t4_internal(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = sub(a->x, b->x); a->x = add(a->x, b->x); b->x = sub(by, a->y); a->y = add(a->y, by); } // Same as X2(a, b), b = mul_t8(b) -void OVERLOAD X2_mul_t8_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); t = *b + neg(t, 2); *b = shl(U2(t.x + neg(t.y, 4), t.x + t.y), 30); } +void OVERLOAD X2_mul_t8_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); t = *b + neg(t, 2); *b = shl30(U2(t.x + neg(t.y, 4), t.x + t.y)); } // Same as X2(a, b), b = mul_3t8(b) void OVERLOAD X2_mul_3t8_internal(GF61 *a, GF61 *b) { GF61 t = *a; *a = add(t, *b); *b = t + neg(*b, 2); *b = mul_3t8(*b, 4); } @@ -1061,17 +1347,21 @@ GF61 OVERLOAD foo(GF61 a) { return foo2(a, a); } // This reduces the number of m61_count*M61 addition operations too. By tracking ranges of intermediate results, the caller knows how many M61s // need to be added to make result positive prior to the final modM61. In function names, "q" stands for quick (no modM61). +Z61 OVERLOAD addq(Z61 a, Z61 b) { return a + b; } +Z61 OVERLOAD subq(Z61 a, Z61 b) { return a - b; } GF61 OVERLOAD addq(GF61 a, GF61 b) { return a + b; } GF61 OVERLOAD subq(GF61 a, GF61 b) { return a - b; } GF61 OVERLOAD addiq(GF61 a, GF61 b) { return U2(a.x - b.y, a.y + b.x); } GF61 OVERLOAD subiq(GF61 a, GF61 b) { return U2(a.x + b.y, a.y - b.x); } void OVERLOAD X2q(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; *b = t - *b; } -void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; t.x = t.x - b->x; b->x = b->y - t.y; b->y = t.x; } +void OVERLOAD X2qt4(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = a->y - b->x; a->y = a->y + b->x; b->x = a->x + by; a->x = a->x - by; } void OVERLOAD X2qconjb(GF61 *a, GF61 *b) { GF61 t = *a; a->x += b->x; a->y -= b->y; b->x = t.x - b->x; b->y = t.y + b->y; } +void OVERLOAD X2q_mul_t4(GF61 *a, GF61 *b) { Z61 by = b->y; b->y = a->x - b->x; a->x = a->x + b->x; b->x = by - a->y; a->y = a->y + by; } void OVERLOAD X2q_conjb(GF61 *a, GF61 *b) { GF61 t = *a; *a = t + *b; b->x = t.x - b->x; b->y = b->y - t.y; } -GF61 OVERLOAD mul_t8q(GF61 a, const u32 m61_count) { return shl(U2(m61_count * M61 + (a.y - a.x), m61_count * M61 - (a.x + a.y)), 30); } +GF61 OVERLOAD mul_t8q(GF61 a, const u32 m61_count) { return shl30(U2(m61_count * M61 + (a.y - a.x), m61_count * M61 - (a.x + a.y))); } +GF61 OVERLOAD mul_3t8q(GF61 a, const u32 m61_count) { return shl30(U2(m61_count * M61 + a.x + a.y, m61_count * M61 + (a.y - a.x))); } Z61 OVERLOAD optsubqu(Z61 a, const u32 m61_limit, const u32 m61_count) { return optional_sub((u64)a, (u32)(m61_limit << (61 - 32)), (u64)(m61_count * M61)); } GF61 OVERLOAD optsubqu(GF61 a, const u32 m61_limit, const u32 m61_count) { return U2(optsubqu(a.x, m61_limit, m61_count), optsubqu(a.y, m61_limit, m61_count)); } diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index 9ed387a9..be977fb0 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -1,601 +1,887 @@ -// Copyright (C) Mihai Preda - -#ifdef T2_GF61 - -// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. -// Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- -// even when operating on differernt sized data elements as can happen in an M31+M61 NTT. -// WG = workgroup size of a single fft_WIDTH or fft_HEIGHT -// n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT -// numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously -// lowMe = me % WG -// NOTE: shufl routines perform a bar(WG) at the start but not at the end. After calling shufl, a bar(WG) is required -// before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and -// only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively -// minor optimization would be to special case the first shufl call. -void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - int force_default = 0; -#if NOWG2 // For timing tests only. Option to not turn off LDS bank conflict code when numWG > 1. I've not found a GPU where this is beneficial. - if (numWG > 1) force_default = 1; -#endif -#if NOLDS2 // For timing tests only. Option to not turn off LDS bank for second shufl calls. I've not found a GPU where this is beneficial. - if (f != 1) force_default = 1; -#endif - - // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. - if (SHUFL_BYTES == 16) { - local T2_GF61* lds = lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); - -#if LDSPAD - // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 8, 72..., 16... lds[64..127] = +1 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad 1 value every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe & 7) * (WG + 1) + (lowMe / 8) * 8 + i] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 32 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // No padding of LDS blocks is needed to eliminate bank conflicts! Groups of 8 threads are already in separate LDS banks. - // We could however save a bar() by writing to same locations that the previous shufl wrote to. - if (0 && f == 8 && RADIX == 8) { - // for (u32 i = 0; i < RADIX; ++i) { lds[something] = u[i]; } - bar(WG); - //for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[something]; } - return; - } - - // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1, 65..., 16... lds[64..127] = +2 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad 1 value every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + (lowMe & 1) * 4 + i] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 4 + (lowMe / 32) * 8 + ((lowMe / 8) & 3) * (WG + 1) + (lowMe & 7)]; } - return; - } - - // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16... 4.. lds[64..127] = +1 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 16... 1.. output[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * (WG + 4) + i * 4 + (lowMe & 3)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - return; - } -#endif - -#if LDSSWIZ - // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and - // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. - // We can however save a bar() by writing to same locations that previous shufl wrote to. - if (!force_default && f == 8 && RADIX == 8) { - for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } - return; - } - - // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } - return; - } - - // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } - return; - } -#endif - - // Otherwise, execute the original shufl code - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } - } - - // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. - else if (SHUFL_BYTES == 8) { - local T_Z61* lds = ((local T_Z61*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); - -#if LDSPAD - // Special case first RADIX == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Pad 8 values after every row to eliminate bank conflicts. - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - return; - } - - // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - return; - } -#endif - -#if LDSSWIZ - // Special case first RADIX == 8 code to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } - return; - } - - // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } - return; - } - - // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } - return; - } -#endif - - // Execute the original shufl code - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } - } - - // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. - // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! - else if (SHUFL_BYTES == 4) { - // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. - // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly - // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. - local int* lds = (local int*) lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); - - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - } -} - -#endif - - -#ifdef F2_GF31 - -// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats or Z31s. -void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { - - u32 mask = f - 1; - assert((mask & (mask + 1)) == 0); - - //GW - would a 16 byte implementation be useful? Less LDS conflict work? - - int force_default = 0; -#if NOWG2 - if (numWG > 1) force_default = 1; -#endif -#if NOLDS2 - if (f != 1) force_default = 1; -#endif - - // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. - if (SHUFL_BYTES >= 8) { - local F2_GF31* lds = lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); - -#if LDSPAD - // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 16, 80... lds[64..127] = +2 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 7) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 1) * 8 + i] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i / 2) * 16 + (i & 1) * (4 * (WG + 1)) + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 128) * 16 + ((lowMe / 16) & 7) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Pad 8 values after every 64 values to eliminate bank conflicts. - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i]; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - return; - } - - // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - return; - } - - // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - return; - } -#endif - -#if LDSSWIZ - // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } - return; - } - - // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } - return; - } - - // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 - // Swizzle LDS blocks to eliminate bank conflicts. - // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } - return; - } -#endif - - // Execute the original shufl code - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } - } - - // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. - else if (SHUFL_BYTES == 4) { - local F_Z31* lds = ((local F_Z31*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); - -#if LDSPAD - // Special case first RADIX == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. - // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 2, 66..., 3, 67..., 32, 96... lds[64..127] = +4 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 8) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 - // Pad 8 values after every 64 values to eliminate bank conflicts. - if (!force_default && f == 8 && RADIX == 8) { - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - return; - } - - // Special case first RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 32 must have unique LDS banks. - // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 7.., 32... lds[64..127] = +8 - // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 - // Pad one value after every row to eliminate bank conflicts. - if (!force_default && f == 1 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } - return; - } - - // Special case second RADIX == 4 to eliminate LDS bank conflicts. We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 - // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. - // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 1.... ... 8... lds[64..127] = +2 - // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 - // Pad 4 values after every row to eliminate bank conflicts. - if (!force_default && f == 4 && RADIX == 4) { - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } - bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } - return; - } -#endif - - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } - bar(WG); - for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } - } -} - -#endif - - +// Copyright (C) Mihai Preda + +#ifdef T2_GF61 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. +// Care is taken that each simultaneous workgroup does not interfere with the LDS memory of other simultaneous workgroups -- +// even when operating on differernt sized data elements as can happen in an M31+M61 NTT. +// WG = workgroup size of a single fft_WIDTH or fft_HEIGHT +// n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT +// numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously +// lowMe = me % WG +// r usually equals RADIX if a full fft_RADIX step was just performed. On occasion u[8] values may do less than an fft8 step. +// NOTE: shufl routines perform a bar(WG) at the start but not at the end. After calling shufl, a bar(WG) is required +// before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and +// only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively +// minor optimization would be to special case the first shufl call. +void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. + // We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. + if (SHUFL_BYTES == 16) { + local T2_GF61* lds = lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 8, 72..., 16... lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad 1 value every row to eliminate bank conflicts. + if (0 && f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe & 7) * (WG + 1) + (lowMe / 8) * 8 + i] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + return; + } + + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that uses a little padding. Pad one value after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + if (f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 1) + lowMe] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 8) + (lowMe / 8) + (lowMe & 7) * (WG + 1)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // No padding of LDS blocks is needed to eliminate bank conflicts! Groups of 8 threads are already in separate LDS banks. + // We could however save a bar() by writing to same locations that the previous shufl wrote to. + if (0 && f == 8 && r == 8 && RADIX == 8) { + // for (u32 i = 0; i < RADIX; ++i) { lds[something] = u[i]; } + bar(WG); + //for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[something]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1, 65..., 16... lds[64..127] = +2 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad 1 value every row to eliminate bank conflicts. + if (f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + (lowMe & 1) * 4 + i] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 4 + (lowMe / 32) * 8 + ((lowMe / 8) & 3) * (WG + 1) + (lowMe & 7)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16... 4.. lds[64..127] = +1 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 16... 1.. output[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * (WG + 4) + i * 4 + (lowMe & 3)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). + if (f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and + // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. + // We can however save a bar() by writing to same locations that previous shufl wrote to. + if (f == 8 && r == 8 && RADIX == 8) { + for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). + if (f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). + if (f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + } + + // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. + // We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + else if (SHUFL_BYTES == 8) { + local T_Z61* lds = ((local T_Z61*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); + +#if LDSPAD + // Special case first RADIX == 8 code to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that uses a little padding. Pad two values after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + if (f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: u[0] = 0, 64, ... 448, 8, 72... u[1] = +1 + if (f == 8 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 8) * (WG + 8) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 8) * (WG + 8) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + return; + } + + // Special case alternate first RADIX == 8 code to eliminate LDS bank conflicts (only a radix-4 step was performed). + // Input values are in order and written straight to LDS memory. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +32... + // Output to LDS that uses a little padding. Pad four values after every other row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, 128, 192, 1, 65... u[1] = +8 + if (f == 1 && r == 4 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / 2 * (2 * WG + 4) + (i % 2) * WG + lowMe] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (2 * WG + 4)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / 2 * (2 * WG + 4) + (i % 2) * WG + lowMe] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (2 * WG + 4)]; } + return; + } + + // Special case alternate second RADIX == 8 to eliminate LDS bank conflicts (first shufl was partial after a radix-4 step). + // Input values are the output from a previous shufl. For example, WIDTH=256: u[0] = 0, 64, 128, 192, 1, 65... u[1] = +8 + // Output to LDS with 4 pads after each row. + // Read from LDS in output order. In the example: u[0] = 0, 64, 128, 192, 8, 72... u[1] = +1 + if (f == 4 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } + bar(WG); + if (WG == 32) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 32) * 4 + (lowMe / 4) * (WG + 4) + (lowMe & 3)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 32) * 4 + (lowMe / 32) * 4 + ((lowMe / 4) & 7) * (WG + 4) + (lowMe & 3)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } + bar(WG); + if (WG == 32) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 32) * 4 + (lowMe / 4) * (WG + 4) + (lowMe & 3)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 32) * 4 + (lowMe / 32) * 4 + ((lowMe / 4) & 7) * (WG + 4) + (lowMe & 3)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (1 && f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS using a little padding. Pad four values after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + if (0 && f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (WG + 4)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (WG + 4)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: u[0] = 0...192, 16... 32.. 48.. 1... u[1] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (0 && f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS with 4 pads after each row. + // Read from LDS in output order. In the example: u[0] = 0...192, 16... 32.. 48.. 1... u[1] = +4 + if (1 && f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 16) * 4 + (lowMe / 16) * 4 + ((lowMe / 4) & 3) * (WG + 4) + (lowMe & 3)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 16) * 4 + (lowMe / 16) * 4 + ((lowMe / 4) & 3) * (WG + 4) + (lowMe & 3)]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 code to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). + if (f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). + if (f == 8 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + } + + // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. + // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + else if (SHUFL_BYTES == 4) { + // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. + // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly + // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. + local int* lds = (local int*) lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); + + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + } +} + +// Shortcut for the most common case where caller did a full RADIX step (as opposed to the oddball cases where we have a u[8] but only did a radix 2 or 4 step). +void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { + shufl(lds2, u, f, RADIX, numWG, lowMe); +} + + +// NEEDS TONS OF WORK!!! SWIZ NOT CODED, MOST PAD CASES NOT CODED, SHUFL_BYTES = 4 needs differernt algorithm. +// At present, this is only used by WIDTH or HEIGHT = 1K with RADIX=8 and f=8. + +// Shufl two or more fft_WIDTHs or fft_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. An fft2 is also performed. +void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { + assert(RADIX == 8); + assert(SHUFL_BYTES >= 8); + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + // Start by doing the writes of a standard shufl. + // Next, each thread reads a pair of values. The lower threads add the two values, the higher threads subtract the two values. + // val1 is read from i * WG/2 + // val2 is read from 4 * WG + i * WG/2 + + // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. + if (SHUFL_BYTES == 16) { + local T2_GF61* lds = lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); + + // Execute the original shufl code with an fft2 add-on. + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + T2_GF61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + T2_GF61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } + } + + // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. + else if (SHUFL_BYTES == 8) { + local T_Z61* lds = ((local T_Z61*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); + +#if LDSPAD + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + if (f == 8 && RADIX == 8) { + local T_Z61 *ldsIn; + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 + // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 + T_Z61 val1 = lds[(i / 2) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + T_Z61 val2 = lds[(i / 2 + 4) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + if (lowMe < WG / 2) u[i].x = addq(val1, val2); + else u[i].x = subq(val1, val2); + } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 + // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 + T_Z61 val1 = lds[(i / 2) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + T_Z61 val2 = lds[(i / 2 + 4) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + if (lowMe < WG / 2) u[i].y = addq(val1, val2); + else u[i].y = subq(val1, val2); + } + return; + } +#endif + + // Execute the original shufl code with an fft2 add-on. + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + T_Z61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + T_Z61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].x = addq(val1, val2); + else u[i].x = subq(val1, val2); + } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + T_Z61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + T_Z61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].y = addq(val1, val2); + else u[i].y = subq(val1, val2); + } + } + + // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. + // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! + else if (SHUFL_BYTES == 4) { + +// NEEDS WORK!!! + + // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. + // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly + // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. + local int* lds = (local int*) lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); + + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + } +} + +#endif + + +#ifdef F2_GF31 + +// Shufl two or more fft_WIDTHs or FFT_HEIGHTs using two 4-byte floats or Z31s. +void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u32 lowMe) { + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + //GW - would a 16 byte implementation be useful? Less LDS conflict work? + + // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. + // We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. + if (SHUFL_BYTES >= 8) { + local F2_GF31* lds = lds2; + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order and written straight to LDS memory. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that uses a little padding. Pad two values after every row to eliminate bank conflicts. + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + if (f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (1 && f == 8 && r == 8 && RADIX == 8) { + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i]; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + if (0 && f == 8 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 3.., 16... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 4... lds[64..127] = +1 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + return; + } +#endif + +#if LDSSWIZ + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). + if (f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). + if (f == 8 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 1, 65... lds[64..127] = +16 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). + if (f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 192, 16, 80 ... lds[64..127] = +4 + // Swizzle LDS blocks to eliminate bank conflicts. + // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). + if (f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + } + + // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. + // We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. + else if (SHUFL_BYTES == 4) { + local F_Z31* lds = ((local F_Z31*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); + +#if LDSPAD + // Special case first RADIX == 8 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=512: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...448, 1, 65..., 2, 66..., 3, 67..., 32, 96... lds[64..127] = +4 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 + // Pad one value after every row to eliminate bank conflicts. + if (f == 1 && r == 8 && RADIX == 8) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + return; + } + + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + // Pad 8 values after every 64 values to eliminate bank conflicts. + if (f == 8 && r == 8 && RADIX == 8) { + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } + else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + return; + } + + // Special case first RADIX == 4 to eliminate LDS bank conflicts. + // Input values are in order. For example, WIDTH=256: u[0] = 0, 1, 2... u[1] = +64... + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0, 64, ...192, 1.., 2.., 7.., 32... lds[64..127] = +8 + // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 + // Pad one value after every row to eliminate bank conflicts. + if (f == 1 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + return; + } + + // Special case second RADIX == 4 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=256: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 + // Output to LDS that does not use much padding and generates good code because all the lowMe calcs can be computed up front. + // In the example: lds[0..63] = 0...192, 16..., 32..., 48..., 1.... ... 8... lds[64..127] = +2 + // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 + // Pad 4 values after every row to eliminate bank conflicts. + if (f == 4 && r == 4 && RADIX == 4) { + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } + bar(WG); + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + return; + } +#endif + + // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + } +} + +// Shortcut for the most common case where caller did a full RADIX step (as opposed to the oddball cases where we have a u[8] but only did a radix 2 or 4 step). +void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { + shufl(lds2, u, f, RADIX, numWG, lowMe); +} + + +// NEEDS TONS OF WORK!!! SWIZ NOT CODED, MOST PAD CASES NOT CODED. +// At present, this is only used by WIDTH or HEIGHT = 1K with RADIX=8 and f=8. + +// Shufl two or more fft_WIDTHs or fft_HEIGHTs operating on 32-bit values using LDS_BYTES of LDS memory. An fft2 is also performed. +void OVERLOAD shufl_and_fft2(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, u32 lowMe) { + assert(RADIX == 8); + + u32 mask = f - 1; + assert((mask & (mask + 1)) == 0); + + // Start by doing the writes of a standard shufl. + // Next, each thread reads a pair of values. The lower threads add the two values, the higher threads subtract the two values. + // val1 is read from i * WG/2 + // val2 is read from 4 * WG + i * WG/2 + + // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. + if (SHUFL_BYTES >= 8) { + local F2_GF31* lds = ((local F2_GF31*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); + +#if LDSPAD + // Special case second RADIX == 8 to eliminate LDS bank conflicts. + // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 + // Output to LDS with 8 pads after each row. + // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 + if (f == 8 && RADIX == 8) { + local F2_GF31 *ldsIn; + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 + // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 + F2_GF31 val1 = lds[(i / 2) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + F2_GF31 val2 = lds[(i / 2 + 4) * (WG / 64) * 8 + (((i & 1) * (WG / 2)) / 64) * 8 + ((lowMe % (WG / 2)) / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } + return; + } +#endif + + // Execute the original shufl code with an fft2 add-on. + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + F2_GF31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + F2_GF31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } + } + + // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. + else if (SHUFL_BYTES == 4) { + local F_Z31* lds = ((local F_Z31*) lds2); + if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); + + // Execute the original shufl code with an fft2 add-on. + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + F_Z31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + F_Z31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].x = addq(val1, val2); + else u[i].x = subq(val1, val2); + } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } + bar(WG); + for (u32 i = 0; i < RADIX; ++i) { + F_Z31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; + F_Z31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; + if (lowMe < WG / 2) u[i].y = addq(val1, val2); + else u[i].y = subq(val1, val2); + } + } +} + +#endif From 53ac8f0973c8279432d359fc8f0e7091d288a93c Mon Sep 17 00:00:00 2001 From: george Date: Sun, 13 Sep 2026 01:56:13 +0000 Subject: [PATCH 127/214] I asked AI Claude to fix CUDA shim's implementation of barriers and fences. I added atomic_cmpxchg. --- src/cuda/opencl_compat.cuh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/cuda/opencl_compat.cuh b/src/cuda/opencl_compat.cuh index 6b1a2807..203607f5 100644 --- a/src/cuda/opencl_compat.cuh +++ b/src/cuda/opencl_compat.cuh @@ -40,16 +40,24 @@ #define get_global_size(d) ((unsigned int)(gridDim.x * blockDim.x)) #define get_enqueued_local_size(d) get_local_size(d) -// ---- Barriers ---- -#define CLK_LOCAL_MEM_FENCE 0 -#define CLK_GLOBAL_MEM_FENCE 0 -#define barrier(flags) __syncthreads() +// ---- Barriers and fences ---- +#define CLK_LOCAL_MEM_FENCE (1 << 0) +#define CLK_GLOBAL_MEM_FENCE (1 << 1) + +__device__ __forceinline__ void ocl_barrier(int flags) { + __syncthreads(); // execution barrier is unconditional regardless of flags + // __syncthreads() already fences both shared and global memory per CUDA's + // documented semantics, so no additional fence is needed here for either flag. +} +#define barrier(flags) ocl_barrier(flags) -// ---- Memory fences ---- -// OpenCL write_mem_fence / read_mem_fence → CUDA __threadfence() -#define write_mem_fence(flags) __threadfence() -#define read_mem_fence(flags) __threadfence() -#define mem_fence(flags) __threadfence() +__device__ __forceinline__ void ocl_mem_fence(int flags) { + if (flags & CLK_GLOBAL_MEM_FENCE) __threadfence(); + else if (flags & CLK_LOCAL_MEM_FENCE) __threadfence_block(); +} +#define mem_fence(flags) ocl_mem_fence(flags) +#define write_mem_fence(flags) ocl_mem_fence(flags) +#define read_mem_fence(flags) ocl_mem_fence(flags) // ---- Overloadable ---- // CUDA C++ supports function overloading natively @@ -268,6 +276,7 @@ __device__ __forceinline__ uint mad_hi(uint a, uint b, uint c) { // ---- Atomic operations ---- #define atomic_max(p, v) atomicMax((unsigned int*)(p), (unsigned int)(v)) #define atomic_add(p, v) atomicAdd(p, v) +#define atomic_cmpxchg(p, old, new) atomicCAS((int *)(p), old, new) // OpenCL 2.0 C11-style atomics — optimized for CUDA carry stairway pattern. // The carryFused kernel uses: producer writes data, threadfence, bar, atomic_store(flag, 1) From 97cb52863377a702a67e0a3d4657f634b2f2d33d Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 10:55:50 -0600 Subject: [PATCH 128/214] Fix updateStats() call arity in the FFT64 carry kernel STATS path carry.cl:48, in the `#elif (STATS & (1 << (2 + MUL3)))` branch of the FFT64 carry() kernel, calls updateStats() with 3 arguments: updateStats(bufROE, posROE, carryMax); but updateStats() (carryutil.cl:138) takes 6: void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *bufROE, u32 posROE, float roundMax) This is the only one of the 36 updateStats() call sites in carry.cl and carryfused.cl that is wrong; every other FFT type in carry.cl passes the 6 arguments (e.g. :107, :183). The lds array is already declared on the line above but never passed. Effect: building the unfused carry kernel for FFT64 with carry statistics enabled (-use STATS=4, or STATS=8 with MUL3) fails to compile, aborting at kernel build time. The ROE branch just above is unaffected, which is why this has gone unnoticed. Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/cl/carry.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/carry.cl b/src/cl/carry.cl index a8ba21c1..bbfcede8 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -45,7 +45,7 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big updateStats(lds, G_W, H, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(bufROE, posROE, carryMax); + updateStats(lds, G_W, H, bufROE, posROE, carryMax); #endif } From c0b80dd56c00fd3673b4730aa9925e64641de31b Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 10:56:59 -0600 Subject: [PATCH 129/214] Add missing make_i96(i64 hi, u32 lo) to the alternate i96 implementations The i96 two-argument constructors use two different conventions, told apart only by the type of the "lo" argument: make_i96(hi, u32 lo) value = hi * 2^32 + lo make_i96(hi, u64 lo) value = hi * 2^64 + lo The active all-32-bit implementation (math.cl "#if 1") provides both. The __int128 implementation and the {u64 lo64, u32 hi32} implementation provide only the u64 form. weightAndCarryOne() for FFT3161 uses the u32 form (carryutil.cl:459-461): i64 vhi = n61 >> 1; u32 vlo = ((u32)n61 << 31) | n31; i96 value = make_i96(vhi, vlo); // (n61 << 31) + n31 so selecting either alternate implementation silently widens vlo to u64 and places vhi at bit 64 instead of bit 32 -- every FFT3161 word comes out wrong by a factor of 2^32, with no compile error. The other two call sites (carryutil.cl:352 and :420) pass a u64 lo and are unaffected, so FFT6431 and FFT3261 would keep working and the breakage would look like an FFT3161 bug. No behaviour change today, since "#if 1" selects the all-32-bit version: this just makes the other two implementations usable, as the comment at math.cl:66 ("nVidia likes this version") invites. Also documents the two conventions. Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/cl/math.cl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/cl/math.cl b/src/cl/math.cl index 07182c42..9c7de28d 100644 --- a/src/cl/math.cl +++ b/src/cl/math.cl @@ -13,6 +13,11 @@ u64 OVERLOAD make_u64(u32 hi, u32 lo) { union { uint2 ui2; u64 ul; } u; u.ui2.x i64 OVERLOAD make_i64(i32 hi, u32 lo) { union { uint2 ui2; u64 ul; } u; u.ui2.x = lo; u.ui2.y = hi; return u.ul; } // A primitive partial implementation of an i96 integer type +// NOTE: the two-argument constructors use two different conventions, distinguished only by the type of "lo": +// make_i96(hi, u32 lo) places hi at bit 32 (value = hi * 2^32 + lo) +// make_i96(hi, u64 lo) places hi at bit 64 (value = hi * 2^64 + lo) +// Every implementation below must provide both, or a caller passing a u32 "lo" silently +// widens to the bit-64 form. #if 1 // An all 32-bit implementation. The add and subtract routines desperately need to use ASM with add.cc and sub.cc PTX instructions. // This version might be best on AMD and Intel if we can generate add-with-carry instructions. @@ -63,6 +68,7 @@ i96 OVERLOAD sub(i96 a, i32 b) { return sub(a, make_i96(b)); } typedef struct { __int128 x; } i96; i96 OVERLOAD make_i96(i64 v) { i96 val; val.x = v; return val; } i96 OVERLOAD make_i96(i32 v) { i96 val; val.x = v; return val; } +i96 OVERLOAD make_i96(i64 hi, u32 lo) { i96 val; val.x = ((__int128)hi << 32) + lo; return val; } i96 OVERLOAD make_i96(i64 hi, u64 lo) { i96 val; val.x = ((unsigned __int128)hi << 64) + lo; return val; } i96 OVERLOAD make_i96(i32 hi, u64 lo) { return make_i96((i64)hi, lo); } u32 i96_hi32(i96 val) { return (unsigned __int128)val.x >> 64; } @@ -81,6 +87,7 @@ i96 OVERLOAD sub(i96 a, i32 b) { return sub(a, make_i96(b)); } typedef struct { u64 lo64; u32 hi32; } i96; i96 OVERLOAD make_i96(i64 v) { i96 val; val.hi32 = v >> 63, val.lo64 = v; return val; } i96 OVERLOAD make_i96(i32 v) { return make_i96((i64)v); } +i96 OVERLOAD make_i96(i64 hi, u32 lo) { i96 val; val.hi32 = (u64)hi >> 32, val.lo64 = ((u64)hi << 32) | lo; return val; } i96 OVERLOAD make_i96(i64 hi, u64 lo) { i96 val; val.hi32 = hi, val.lo64 = lo; return val; } i96 OVERLOAD make_i96(i32 hi, u64 lo) { i96 val; val.hi32 = hi, val.lo64 = lo; return val; } u32 i96_hi32(i96 val) { return val.hi32; } From 9633f1a90914603f50cb3eb327952ace6a4f2ed7 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 11:00:30 -0600 Subject: [PATCH 130/214] Document and enforce the CARRY32 bpw limit carry32BPW() carries the note "I have no idea why this is needed. Without it, -tune fails on FFT sizes from 256K to 1M. Perhaps it has something to do with RNDVALdoubleToLong in carryutil". The guess was right; this is the mechanism, and the 19.0 cap is a hard limit of the CARRY32 code rather than an empirical one. In the CARRY32 case weightAndCarryOne() returns as_long(RNDVAL + value) without stripping RNDVAL (carryutil.cl:217-219). RNDVAL is 3 * 2^51, so in the result bit 51 is set iff value >= 0 -- it is an indicator, not the sign extension of value -- and bits 52+ hold the exponent. carryStep(i64, i32*) takes the carry as xtract32(x, nBits), i.e. bits [nBits, nBits+32). That works precisely because the window sits below bit 51: for a negative value the sign fill runs up through bit 50 and extracts as -1, and the "+ (w < 0)" corrects it. At nBits = 19 the window is bits 19..50, the last safe position. At nBits = 20 it takes in bit 51, and every big word with a non-negative value extracts a carry of 0x80000000. A big word has nBits = EXP / NWORDS + 1, so the requirement is EXP / NWORDS <= 18, i.e. bpw < 19.0. For a 256K FFT the unclamped formula yields about 20.7, which is why -tune failed there. Changes: - Replace the comment in carry32BPW() with the above. - needsLargeCarry() compares with a strict >, so E == 19 * size() would still select CARRY32 with EXP / NWORDS == 19. Test the kernel's own EXP / NWORDS expression to close the off-by-one. (Not reachable with a prime exponent, since size() is even, but the cap should not depend on that.) - Add an #error in carryutil.cl next to the 32-bit carry instantiation, so a configuration that violates the limit fails at kernel build time instead of silently computing wrong carries. No behaviour change for any exponent that runs today. Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/FFTConfig.cpp | 17 ++++++++++++++--- src/cl/carryutil.cl | 11 ++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index ee13528a..2c260f4c 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -177,14 +177,25 @@ float FFTShape::carry32BPW() const { // We model carry with a Gumbel distrib similar to the one used for ROE, and measure carry with // -use STATS=1. See -carryTune -//GW: I have no idea why this is needed. Without it, -tune fails on FFT sizes from 256K to 1M -// Perhaps it has something to do with RNDVALdoubleToLong in carryutil -if (18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())) > 19.0) return 19.0f; + // The 19.0 cap is a hard limit of the CARRY32 code, not an empirical one, which is why the formula + // above must be clamped for the smaller FFTs (without it, -tune failed on FFT sizes from 256K to 1M). + // + // In the CARRY32 case weightAndCarryOne() returns the raw bits of RNDVAL + value rather than stripping + // RNDVAL off (carryutil.cl:217-219), so bit 51 is set iff value >= 0 and bits 52+ hold the exponent. + // carryStep(i64, i32*) then takes the carry as xtract32(x, nBits), i.e. bits [nBits, nBits+32). That + // window must stay strictly below bit 51 for the sign fill to work, so nBits <= 19, and since a big word + // has nBits = EXP / NWORDS + 1 that means EXP / NWORDS <= 18, i.e. bpw < 19.0. At nBits = 20 the window + // includes bit 51 and every big word with a non-negative value yields a large negative carry. + if (18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size())) > 19.0) return 19.0f; return float(18.35 + 0.5 * (log2(13 * 1024 * 512) - log2(size()))); } bool FFTShape::needsLargeCarry(u64 E) const { + // carry32BPW() caps at 19.0 and the comparison below is a strict >, so E == 19 * size() would still + // select CARRY32 with EXP / NWORDS == 19, which the CARRY32 code cannot handle (see carry32BPW()). + // Test the kernel's own EXP / NWORDS expression to close that off-by-one. + if (E / size() >= 19) { return true; } return E / double(size()) > carry32BPW(); } diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 9f7aa0b7..b92c7ead 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -214,7 +214,9 @@ i64 weightAndCarryOne(T u, T invWeight, i64 inCarry, float* maxROE, int sloppy_r float roundoff = fabs((float) fma(u, invWeight, RNDVALCarry - d)); *maxROE = max(*maxROE, roundoff); - // Convert to long (for CARRY32 case we don't need to strip off the RNDVAL bits) + // Convert to long (for CARRY32 case we don't need to strip off the RNDVAL bits). + // Leaving RNDVAL in place is only safe while the carry extraction window stays below bit 51 -- see the + // #error below and FFTShape::carry32BPW. if (sloppy_result_is_acceptable) return as_long(d); else return RNDVALdoubleToLong(d); @@ -799,6 +801,13 @@ Word2 carryWord(Word2 a, CarryABM* carry, bool b1, bool b2) { /* Support both 32-bit and 64-bit carries */ #if WordSize <= 4 +// A 32-bit carry means weightAndCarryOne returns RNDVAL + value un-stripped, and carryStep(i64, i32*) reads +// the carry from bits [nBits, nBits+32). That window must stay strictly below the RNDVAL bit 51, so +// nBits <= 19; a big word has nBits = EXP / NWORDS + 1. The host is supposed to select CARRY64 before this +// point (FFTShape::needsLargeCarry); fail loudly rather than compute wrong carries if it ever does not. +#if !CARRY64 && FFT_TYPE == FFT64 && EXP / NWORDS >= 19 +#error "CARRY32 requires EXP / NWORDS <= 18; this exponent needs CARRY64 (-carry long)" +#endif #define iCARRY i32 #include "carryinc.cl" #undef iCARRY From ee9eb50930f9c1779b1254ff2f40fa91c050969e Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 11:10:12 -0600 Subject: [PATCH 131/214] Fix barriers reached by only part of the workgroup in carryFused carryFused runs G_W * WMUL work-items, and three of its barriers sit inside control flow that only some of them enter. A barrier under divergent control flow is undefined in OpenCL (and for __syncthreads in CUDA). 1. The producer's bar(G_W) is inside if (gr < H / WMUL && me >= (WMUL-1) * G_W) gr is uniform, but "me >= (WMUL-1) * G_W" is not: only the last sub-workgroup arrives. 2. The consumer's bar() after the spin-wait is inside "if (me < G_W)": only the first sub-workgroup arrives. 3. The gr == H/WMUL barrier on the !OLD_FENCE path is likewise inside "if (me < G_W)". On hardware where a barrier is just an arrival counter, the halves end up one barrier out of phase for the rest of the kernel. Counting arrivals for WMUL=2, OLD_FENCE=1, 0 < gr < H/WMUL: me >= G_W: R, U1..Un, S1..Sm me < G_W: U1..Un, S1..Sm, C so R pairs with U1, U1 with U2, and Sm with C, where U* are the barriers in updateStats() and S* those in shufl_carries_up(). Inside updateStats() the high half then writes lds after its barrier k+1 while the low half reads after its barrier k, and the low half reads stale lds. That is what the workaround at carryutil.cl:150-152 is compensating for: bar(); // work around a weird CUDA NVCC bug where two bar() calls are bar(); // required???! (Titan V, CUDA 13.0, WMUL=2) The second bar() re-aligns the phase for that one reduction. It is not an NVCC bug. shufl_carries_up() survives only because it already takes its barriers outside the guards -- which is the pattern used here. Fix: every barrier is now reached by the whole workgroup, with only the work guarded. The producer barrier keeps bar(G_W)'s property of being elided when a sub-workgroup is a single wavefront (#if G_W > WAVEFRONT), so no barrier is added where there was none. Note that with ENABLE_BARSYNC the old code was correct, because bar.sync with a thread count is a genuine partial barrier; this makes correctness independent of that setting. Tested on an Intel UHD iGPU (OpenCL 24.35, so OLD_FENCE=1, G_W=64 > WAVEFRONT=32), -prp 3321937 -fft 256:2:256 -iters 2000 -block 200: WMUL=1 WMUL=2 (default) WMUL=4 before OK OUT_OF_RESOURCES OUT_OF_RESOURCES (-5) in kCarryFused after OK OK OK WMUL=1 is the one case where the guards are not divergent, and it is the only one that worked before. All passing runs agree on res64 (400: f8d720194aa2d603, 2000: dd39f676aaf2d94d), so the change is residue-neutral. The duplicated bar() in updateStats() is left in place; it should be removable now, but that wants a test on the Titan V where it was added. Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/cl/carryfused.cl | 432 +++++++++++++++++++++++++++++++++---------- 1 file changed, 333 insertions(+), 99 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 23c8fa42..19bbd4a6 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -215,8 +215,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -226,6 +224,26 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -251,12 +269,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -272,6 +293,16 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -281,11 +312,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } } else { -#if !OLD_FENCE - // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -427,8 +453,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -438,6 +462,26 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -455,12 +499,15 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -476,6 +523,16 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -485,11 +542,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut } } else { -#if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -644,8 +696,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -655,6 +705,26 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -673,12 +743,15 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -694,6 +767,16 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. if (gr < H / WMUL) { @@ -702,11 +785,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry } } else { -#if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -866,8 +944,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -877,6 +953,26 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -895,12 +991,15 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -916,6 +1015,16 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -925,11 +1034,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry } } else { -#if !OLD_FENCE - // For gr==H we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -1104,8 +1208,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -1115,6 +1217,26 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -1140,12 +1262,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Shuffle carries up shufl_carries_up(lds, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -1161,6 +1286,16 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -1170,11 +1305,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } } else { -#if !OLD_FENCE - // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -1368,8 +1498,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -1379,6 +1507,26 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -1396,12 +1544,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Shuffle carries up shufl_carries_up(ldsF2, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -1417,6 +1568,16 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -1426,11 +1587,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } } else { -#if !OLD_FENCE - // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -1628,8 +1784,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -1639,6 +1793,26 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -1656,12 +1830,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -1677,6 +1854,16 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -1686,11 +1873,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } } else { -#if !OLD_FENCE - // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -1882,8 +2064,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -1893,6 +2073,26 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -1911,12 +2111,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -1932,6 +2135,16 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -1941,11 +2154,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } } else { -#if !OLD_FENCE - // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } @@ -2170,8 +2378,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); - bar(G_W); - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else write_mem_fence(CLK_GLOBAL_MEM_FENCE); if (lowMe % WAVEFRONT == 0) { @@ -2181,6 +2387,26 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif } +#if OLD_FENCE + // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the + // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow + // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. +#if WMUL == 1 + if (gr < H) { +#else + if (gr < H / WMUL) { +#endif +#if G_W > WAVEFRONT + bar(); +#endif +#if WMUL == 1 + if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#else + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } +#endif + } +#endif + // Group zero will be redone when gr == H / WMUL if (gr == 0) { return; } @@ -2198,12 +2424,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Shuffle carries up shufl_carries_up(lds61, carry, me, lowMe); - // Wait until our carries are ready + // Wait until our carries are ready. The barrier below must be reached by every work-item of the + // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. +#if OLD_FENCE + if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + bar(); +#endif if (me < G_W) { #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - bar(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; @@ -2219,6 +2448,16 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if HAS_ASM __asm("s_setprio 1"); #endif + } + +#if !OLD_FENCE + // For the last group the carry reading is shifted, so the per-wavefront ready flags are not enough and a + // barrier is needed. gr is uniform but "me < G_W" is not, so the barrier is taken outside that guard and + // the shuttle reads resume in a second "me < G_W" block. + if (gr >= H / WMUL) { bar(); } +#endif + + if (me < G_W) { // Read from the carryShuttle carries produced by the previous WIDTH group. Rotate carries from the last WIDTH line. // The new carry layout lets the AMD compiler generate global_load_dwordx4 instructions. @@ -2228,11 +2467,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut } } else { -#if !OLD_FENCE - // For gr==H/WMUL we need the barrier since the carry reading is shifted, thus the per-wavefront trick does not apply. - bar(); -#endif - for (i32 i = 0; i < NW; ++i) { carry[i] = CSLOAD(&carryShuttlePtr[(gr - 1) * WIDTH + CarryShuttleAccess((me + G_W - 1) % G_W, i) /* ((me!=0) + NW - 1 + i) % NW*/]); } From 475f894a2ccab5a1710e27418b936fb06887e27c Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 11:20:45 -0600 Subject: [PATCH 132/214] Fix missing else that made every "FFT size" -fft spec throw The if-chain in FFTConfig::FFTConfig that dispatches on the number of ':'-separated components is missing an else: if (v.size() == 1) { *this = {FFTShape::multiSpec(spec).front(), LAST_VARIANT, CARRY_AUTO}; } if (v.size() == 3) { <-- here ... } else if (v.size() == 4) { } else if (v.size() == 5) { } else { throw "FFT spec"; } For a 1-component spec the first if runs and assigns *this correctly, and then control falls into the second, independent chain: not 3, not 4, not 5, so the else fires and throws. The spec is parsed and then immediately thrown away. That breaks the two -fft forms documented at Args.cpp:141-146: - a FFT size: 6.5M - a size range: 7M-8M Before: $ prpll -prp 3321937 -fft 256K -iters 10 Exception "FFT spec" $ prpll -prp 3321937 -fft 6.5M -iters 10 Exception "FFT spec" After, all documented forms work, and the 3/4/5-component forms are unchanged: -fft 256K FFT: 256K 256:2:256:212 -fft 6.5M FFT: 6.50M 1K:13:256:212 -fft 7M-8M FFT: 7M 1K:14:256:212 -fft 256:2:256 FFT: 256K 256:2:256:212 -fft 256:2:256:212 FFT: 256K 256:2:256:212 -fft 256:2:256:212:1 FFT: 256K 256:2:256:212:1 Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/FFTConfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index ee13528a..92f93119 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -237,7 +237,7 @@ FFTConfig::FFTConfig(const string& spec) { if (v.size() == 1) { *this = {FFTShape::multiSpec(spec).front(), LAST_VARIANT, CARRY_AUTO}; - } if (v.size() == 3) { + } else if (v.size() == 3) { *this = {FFTShape{fft_type, v[0], v[1], v[2]}, LAST_VARIANT, CARRY_AUTO}; } else if (v.size() == 4) { *this = {FFTShape{fft_type, v[0], v[1], v[2]}, parseInt(v[3]), CARRY_AUTO}; From 6fba41c6089d055178ce9457bd37f0d29b465b50 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 11:26:31 -0600 Subject: [PATCH 133/214] Validate blockSize read from the savefile blockSize comes out of the savefile, not the command line: PRPState const state = loadPRP(*getSaver()); blockSize = state.blockSize; ... assert(blockSize > 0 && logStep % blockSize == 0); u32 const checkStep = checkStepForErrors(blockSize, nErrors); -block is validated in Args.cpp:391-393, but that does not cover a value arriving from a savefile, and the v12/v13 CRC covers the check data rather than the header -- the header is plain text, so one flipped digit in "block=200" is enough. Nothing then checks the value. It reaches writeState() first, and behind that baseCheckStep(), whose default arm is "assert(false); return 0;" -- and release builds are compiled with -DNDEBUG, so that assert and the assert above both vanish, leaving checkStep == 0 and "k % checkStep" as a division by zero. Observed before this change, after editing the newest savefile's "block=200" to "block=300": Exception gpu_error: OUT_OF_RESOURCES (-5) ktailMul at src/clwrap.cpp:335 run i.e. it dies in writeState() before reaching the division by zero. Either way the run stops, even though an intact earlier savefile is sitting right there. loadPRP() already handles a bad savefile by dropping it and retrying with an earlier one when the residue does not match. This adds the same treatment for an out-of-range blockSize, checked before writeState() is given the value, and makes baseCheckStep() throw instead of returning 0. The accepted set is the one baseCheckStep already knows (200/400/500/1000), so savefiles written with block=400 keep loading even though -block no longer offers it. After the change the same corrupted savefile gives: EE 2000 on-load: invalid blockSize 300 and the run continues from the previous savefile. Normal save/resume is unaffected: resuming a 1000-iteration run reaches the same res64 at 2000 as an uninterrupted run (dd39f676aaf2d94d). Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/Gpu.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 473f554d..929e2fdb 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -523,6 +523,13 @@ class IterationTimer { } }; +// The block sizes baseCheckStep() knows about. blockSize comes from the savefile, so it must be +// validated on load (see Gpu::loadPRP) -- a bad value here would otherwise produce checkStep == 0 +// in a release build, where the assert below is compiled out, and then "k % checkStep" divides by zero. +bool isValidBlockSize(u32 blockSize) { + return blockSize == 200 || blockSize == 400 || blockSize == 500 || blockSize == 1000; +} + u32 baseCheckStep(u32 blockSize) { switch (blockSize) { case 200: return 40'000; @@ -530,8 +537,8 @@ u32 baseCheckStep(u32 blockSize) { case 500: return 200'000; case 1000: return 1'000'000; default: - assert(false); - return 0; + log("Invalid blockSize %u\n", blockSize); + throw "invalid blockSize"; } } @@ -2291,6 +2298,16 @@ PRPState Gpu::loadPRP(Saver& saver) { } PRPState state = saver.load(); + + // blockSize is read straight out of the savefile, and the v12 CRC covers the check data but not + // the header. Reject an out-of-range value the same way a residue mismatch is rejected, so an + // earlier savefile gets a chance, rather than letting it reach baseCheckStep(). + if (!isValidBlockSize(state.blockSize)) { + log("EE %9" PRIu64 " on-load: invalid blockSize %u\n", state.k, state.blockSize); + if (!state.k) { break; } + continue; + } + writeState(state.k, state.check, state.blockSize); u64 const res = dataResidue(); From f5a43372c0b12a6e66d3b3b93319478d6cff04dd Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 12:02:47 -0600 Subject: [PATCH 134/214] Pass the real workgroup count to updateStats from carry() updateStats' last-group protocol (carryutil.cl:170-176) needs num_blocks to be the number of workgroups that actually call it: u32 old_value = atomic_add(bufROE + 1, 1); if (old_value == num_blocks - 1) { bufROE[0] = posROE + 1; bufROE[1] = 0; } carry() is enqueued with hN / CARRY_LEN work-items and a workgroup of G_W, so groups = hN / (CARRY_LEN * G_W) = (WIDTH * BIG_H) / (CARRY_LEN * WIDTH/NW) = BIG_HEIGHT * NW / CARRY_LEN but every call site passed num_blocks = H = BIG_HEIGHT. Those agree only when NW == CARRY_LEN. CARRY_LEN is fixed at 8 (Gpu.cpp:48-50) while nW() returns 8 for widths 512/1024/4096 but 4 for width 256 (FFTConfig.h:45), so WIDTH=256 is off by a factor of two. For -fft 256:2:256: hN=131072, CARRY_LEN=8, NW=4, G_W=64, giving 16384/64 = 256 groups while num_blocks was 512. (carryFused is correct: 32896/128 = 257 groups less the early-returning group 0 = 256, and it passes H/WMUL = 256.) Only the ROE / STATS instrumentation is affected; residues are not. But the effects are not small: - With -carry long the counter reaches num_blocks-1 only every second launch, so the device posROE advances at half the host roePos rate and readROE() reads back roePos+2 floats of which about half were never written. - In a mixed run an unfused launch leaves the shared counter at 256 rather than 0, so the next carryFused launch sails past its own reset threshold of 255 and never resets. Once out of phase posROE can stall completely. Since ROE is what decides whether an FFT size is safe for an exponent, under-reporting it matters. Measured, -prp 3321937 -iters 1000 -block 200 -roe: 256:2:256, -carry long Z=92817 -> Z=583475 512:2:256 Z=2741640157 -> Z=2741640157 (unchanged) The WIDTH=512 case is bit-identical, which is the expected signature: NW == 8 == CARRY_LEN there, so the expression is unchanged. Residues match in every case. Note: this also rewrites the FFT64 STATS call at carry.cl:48, which is the 3-argument call fixed by the "Fix updateStats() call arity" PR. Whichever lands first, the other needs a one-line rebase on that call. With this change "-use STATS=4 -carry long" compiles and runs; on master it fails with "Can't compile carry.cl". Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/cl/carry.cl | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/cl/carry.cl b/src/cl/carry.cl index a8ba21c1..3c356cae 100644 --- a/src/cl/carry.cl +++ b/src/cl/carry.cl @@ -6,6 +6,12 @@ #include "carryutil.cl" #include "weight.cl" +// Number of workgroups this kernel is launched with: it is enqueued with hN / CARRY_LEN work-items +// and a workgroup of G_W, so hN / (CARRY_LEN * G_W) = BIG_HEIGHT * NW / CARRY_LEN. updateStats needs +// this exact count -- passing BIG_HEIGHT is only correct when NW == CARRY_LEN, which fails for WIDTH=256 +// where nW() is 4. CARRY_LEN divides BIG_HEIGHT since BIG_HEIGHT = MIDDLE * SMALL_HEIGHT. +#define CARRY_GROUPS (NW * (BIG_HEIGHT / CARRY_LEN)) + #if FFT_TYPE == FFT64 // Carry propagation with optional MUL-3, over CARRY_LEN words. @@ -42,10 +48,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #if ROE local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, roundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -101,10 +107,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(F2) in, u32 posROE, P(CarryABM) carryOut, Big #if ROE local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, roundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -177,10 +183,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF31) in, u32 posROE, P(CarryABM) carryOut, P #if ROE local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) M31; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(lds, G_W, H, bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -253,10 +259,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(GF61) in, u32 posROE, P(CarryABM) carryOut, P #if ROE local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) (M61 >> 32); // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(lds, G_W, H, bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -333,10 +339,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #if ROE local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, roundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -418,10 +424,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #if ROE local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, roundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -503,10 +509,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #if ROE local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, roundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -596,10 +602,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, P(u #if ROE local u32 lds[G_W]; float fltRoundMax = (float) roundMax / (float) 0x1FFFFFFF; // For speed, roundoff was computed as 32-bit integer. Convert to float. - updateStats(lds, G_W, H, bufROE, posROE, fltRoundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, fltRoundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } @@ -697,10 +703,10 @@ KERNEL(G_W) carry(P(Word2) out, CP(T2) in, u32 posROE, P(CarryABM) carryOut, Big #if ROE local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, roundMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, roundMax); #elif (STATS & (1 << (2 + MUL3))) local u32 lds[G_W]; - updateStats(lds, G_W, H, bufROE, posROE, carryMax); + updateStats(lds, G_W, CARRY_GROUPS, bufROE, posROE, carryMax); #endif } From ad2bf1b2607e2c5d95226658ead1c5c5df055329 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 12:27:26 -0600 Subject: [PATCH 135/214] Fix three inaccurate comments/messages No functional change. tailutil.cl:26 -- the default-value comment contradicts the table three lines above it. The table says "2 = double wide, single kernel", and SINGLE_KERNEL ((TAIL_KERNELS & 1) == 0) is indeed true for 2, matching the host default at Gpu.cpp:253 ("Default tailSquare is double-wide in one kernel"). The comment said "with two kernels". This is the spot a reader checks when confirming the host and kernel defaults agree, so it is worth being right. FFTConfig.cpp:225 -- the check is "m < 2 || m > 16" but the message told the user "Middle must be between 1 and 16", i.e. it names a value the code rejects. Anyone typing -fft 256:1:256 is told 1 is in range. Primes.h:9 -- "prims" -> "primes". (The 10 billion claim itself is right: bitset<50000> covers odd numbers to 100001, and 100001^2 is just over 1e10.) Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/FFTConfig.cpp | 2 +- src/Primes.h | 2 +- src/cl/tailutil.cl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index ee13528a..8c316c69 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -222,7 +222,7 @@ FFTConfig::FFTConfig(const string& spec) { throw "Invalid FFT spec"; } if (m < 2 || m > 16) { - log("Middle must be between 1 and 16.\n"); + log("Middle must be between 2 and 16.\n"); throw "Invalid FFT spec"; } if (h != 256 && h != 512 && h != 1024) { diff --git a/src/Primes.h b/src/Primes.h index b5dbd4e4..aee511f4 100644 --- a/src/Primes.h +++ b/src/Primes.h @@ -6,7 +6,7 @@ #include "common.h" class Primes { - std::bitset<50000> sieve; // Allows for testing prims up to 10 billion + std::bitset<50000> sieve; // Allows for testing primes up to 10 billion [[nodiscard]] bool isPrimeOdd(u64 n) const; public: diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index da79b6f7..f5d9fe51 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -23,7 +23,7 @@ // 2 = double wide, single kernel // 3 = double wide, two kernels #if !defined(TAIL_KERNELS) -#define TAIL_KERNELS 2 // Default is double-wide tailSquare with two kernels +#define TAIL_KERNELS 2 // Default is double-wide tailSquare with a single kernel #endif #define SINGLE_WIDE (TAIL_KERNELS < 2) // Old single-wide tailSquare vs. new double-wide tailSquare #define SINGLE_KERNEL ((TAIL_KERNELS & 1) == 0) // TailSquare uses a single kernel vs. two kernels From 70d6e7989ed21d3639426dfa08de50f2860fa4ff Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 14:04:47 -0600 Subject: [PATCH 136/214] Do not truncate the worktodo exponent to 32 bits Both worktodo parse paths read the exponent as u64 and then narrow it when building the Task, whose "exponent" field is itself u64: u64 exp{}; from_chars(s.c_str(), end, exp, 10); ... return {{.kind=isPRP ? Task::PRP : Task::LL, .exponent=u32(exp), ...}}; // :67 return {{.kind=Task::CERT, .exponent=u32(exp), ...}}; // :87 So a worktodo.txt entry of 2^32 or more is silently tested as exp mod 2^32 -- a different exponent entirely, with no warning, and the result would be reported against the original assignment ID. With worktodo-0.txt containing PRP=N/A,1,2,4300000019,-1 before: 20260913 14:04:04 5032723 No FFTs found in tune.txt that can handle 5032723. 20260913 14:04:04 5032723 FFT: 256K 2:256:2:256:101 (19.20 bpw) (4300000019 - 2^32 = 5032723), and after: 20260913 14:04:20 Warning: Exponent 4300000019 is not prime. Using exponent 4300000013 instead. 20260913 14:04:20 4300000013 No FFTs found in tune.txt that can handle 4300000013. The command-line paths were already fine: Args::prpExp / llExp are u64 and Worktodo::getTask passes them through unnarrowed, so "-prp 4300000019" always kept the full value. This makes worktodo.txt agree with them. No exponent that large is in circulation today, so this changes nothing for current work; it removes one of the remaining obstacles noted in 981c442 ("Changed exponent from u32 to u64. Exponents over 2^32-1 don't work yet -- debugging is needed"). Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/Worktodo.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 0fd025dd..44a5b075 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -65,7 +65,7 @@ std::optional parse(const std::string& line) { u64 exp{}; auto [ptr, _] = from_chars(s.c_str(), end, exp, 10); if (ptr != end) { exp = 0; } - if (exp > 1000) { return {{.kind=isPRP ? Task::PRP : Task::LL, .exponent=u32(exp), .AID=AID, .line=line, .squarings=0}}; } + if (exp > 1000) { return {{.kind=isPRP ? Task::PRP : Task::LL, .exponent=exp, .AID=AID, .line=line, .squarings=0}}; } } if (isCERT) { vector parts = split(topParts.back(), ','); @@ -84,7 +84,7 @@ std::optional parse(const std::string& line) { u64 squarings{0}; from_chars(s.c_str(), end, squarings, 10); //printf ("Exec cert %d %d \n", (int) exp, (int) squarings); - if (exp > 1000 && squarings > 100) { return {{.kind=Task::CERT, .exponent=u32(exp), .AID=AID, .line=line, .squarings=u32(squarings) }}; } + if (exp > 1000 && squarings > 100) { return {{.kind=Task::CERT, .exponent=exp, .AID=AID, .line=line, .squarings=u32(squarings) }}; } } } } From 43da00df498279fe8ceff930cf2dca00e0130133 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 14:18:36 -0600 Subject: [PATCH 137/214] Fix LDS write index in shufl's padded "second RADIX == 4" case With LDSPAD and SHUFL_BYTES == 16 and RADIX == 4, the f == 4 / r == 4 case writes to lds[lowMe / 4 * (WG + 4) + i * 4 + (lowMe & 3)] but reads back from lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)] The read treats the buffer as RADIX == 4 rows of stride WG + 4, selecting the row with lowMe / 16. The write treats it as WG / 4 == 16 rows, selecting the row with lowMe / 4. The two disagree, and since fftbase.cl allocates (WG * RADIX + 12) * SHUFL_BYTES -- i.e. 4 rows of 64 plus 3 pads of 4, 268 T2 entries for WG == 64 -- the write runs off the end: write index range 0..1035, LDS holds 268 192 of 256 writes out of bounds (every thread with lowMe >= 16) 192 of 256 outputs therefore wrong It looks like this was adapted from the LDSSWIZ version just below, which is lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] by substituting WG + 4 for the 16. But the 16 there is an offset *within* a 64-entry row, not a row stride, so the substitution needs to keep it and mask the row index -- exactly the shape the padded "first RADIX == 4" case above already uses, ((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + ... Solving for the write that makes the existing read reproduce the permutation of the generic (non-LDSPAD, non-LDSSWIZ) path gives ((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3) which is in range 0..267, uses all 256 slots exactly once, and matches the generic permutation element for element. RADIX == 4 implies WG == 64 here (nW()/nH() return 4 only for 256), so the 16 is well defined, as it already is in the read on the next line. Verified with -prp 2500009 -iters 400 -block 200, where every configuration must agree on res64 4de1f30100dd0b52: before after -fft 256:2:256 -use SHUFL_BYTES_W=16 Error on load OK 4de1f30100dd0b52 -fft 512:2:256 -use SHUFL_BYTES_H=16 Error on load OK 4de1f30100dd0b52 -fft 512:2:256 -use SHUFL_BYTES_W=16 OK (RADIX 8) OK 4de1f30100dd0b52 -fft 256:2:256 -use SHUFL_BYTES_W=8 OK (other path) OK 4de1f30100dd0b52 Nothing selects SHUFL_BYTES 16 by default (SHUFL_BYTES_W/H default to 8 and -tune does not sweep them), so this only affects hand tuning -- but it made a tunable silently produce wrong residues. Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/cl/shufl.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index be977fb0..0bb6446a 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -86,7 +86,7 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Pad 4 values after every row to eliminate bank conflicts. if (f == 4 && r == 4 && RADIX == 4) { bar(WG); - for (u32 i = 0; i < RADIX; ++i) { lds[lowMe / 4 * (WG + 4) + i * 4 + (lowMe & 3)] = u[i]; } + for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } bar(WG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } From 33a0296f771c43f693d5bd7260527eef182fc0c1 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 13 Sep 2026 14:30:59 -0600 Subject: [PATCH 138/214] Give short runs durable checkpoints A verified savefile is only written when k % checkStep == 0; every other check writes the single rolling "-unverified.prp" instead. checkStep is 1,000,000 for the default blockSize of 1000 (200,000 for 500, 40,000 for 200), so for any exponent smaller than checkStep that condition is never met and the only durable savefile is the one from the very first check at k = 2 * blockSize. A single error then costs the entire run. Observed on -prp 800000 (adjusted to the prime 799999). On master the run reached its final check, failed it, found the unverified savefile also failed its on-load residue check, dropped it, and restarted from k = 2000 -- discarding ~23 minutes of completed work. It then did the same again later: CC 799999 / 799999, 9ac70251f686c8e2 <- test complete EE 800000 a68faf2364746c6e 1 errors <- final check failed EE 780000 on-load: ec71c146b18a3fb8 vs. 8ac7da19aff25069 Removing bad savefile '.../799999-unverified.prp' OK 2000 on-load: blockSize 1000 <- back to the start With this change checkStep becomes 80000 for that exponent, so verified savefiles exist at 80000 and 160000, and the same failure recovers to the previous checkpoint instead of the beginning: OK 80000 ... OK 160000 ... EE 240000 ... 1 errors EE 220000 on-load: ... OK 162000 on-load: blockSize 1000 <- resumed from the 160000 savefile Staying on a multiple of logStep keeps both invariants asserted here (checkStep % logStep == 0, checkStep % blockSize == 0), since logStep % blockSize == 0 is asserted just above. Production runs are unaffected: checkStep > E / 8 is false once E exceeds 8 * checkStep, i.e. 8M for the default blockSize, so normal work keeps the current cadence. Found by an AI code audit (Claude). Co-Authored-By: Claude Opus 5 --- src/Gpu.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 473f554d..508ce3a3 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -2558,8 +2558,20 @@ PRPResult Gpu::isPrimePRP([[maybe_unused]] const Task& task) { assert(blockSize > 0 && logStep % blockSize == 0); - u32 const checkStep = checkStepForErrors(blockSize, nErrors); + u32 checkStep = checkStepForErrors(blockSize, nErrors); + + // A verified savefile is only written when k % checkStep == 0; every other check writes the single + // rolling "unverified" savefile instead. When checkStep exceeds the exponent that condition is + // never met, so the only durable savefile is the one from the very first check at k = 2 * blockSize + // and one error costs the whole run. Scale the step down so a short run still gets several + // checkpoints. Staying on a multiple of logStep preserves both invariants asserted here, because + // logStep % blockSize == 0 was asserted just above. + if (checkStep > E / 8) { + checkStep = std::max(logStep, u32(std::min(checkStep, E / 8) / logStep) * logStep); + } + assert(checkStep % logStep == 0); + assert(checkStep % blockSize == 0); u32 const power = getProofPower(k); From f50dc0b9a5ecb9ead50b9b40a256f248d591cbac Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Mon, 14 Sep 2026 07:08:41 -0600 Subject: [PATCH 139/214] Fix carryB silently dropping a carry that escapes its group carryB applies the predecessor group's carry-out and ripples it through its CARRY_LEN word pairs using carryWord(), which renormalizes both words of every pair via carryStep(). carryB has no carry-out buffer and there is no second pass, so a carry that is still non-zero after the last word of the group is discarded, leaving the first word of the next group off by one ulp. The fused carry path does not have this problem because it ends its chain with carryFinal(), which adds the carry out of the low word into the high word as "u.y += tmpCarry", i.e. without renormalizing. An un-normalized word holds the same value and is normalized on the next iteration. Do the same on the last word pair of carryB's group, in place of the final carryStep, so that nothing can escape. Reproduced with -prp 800000 -fft 256:2:256 -block 1000 -use WMUL=1 (3.05 bits/word), where the Gerbicz-Li check correctly reports a mismatch. The squaring producing iteration 178534 differs from exact CPU arithmetic by exactly 2^156153 -- a single bit, in FFT word 51168, which is the first word of a carry group. The eleven words preceding it are all at the extreme of balanced range, which is exactly the condition under which a +/-1 carry survives carryStep instead of being absorbed, so the carry rode out of the group and was dropped. The reachable range is roughly 3 to 10 bits/word. Once the carry is down to +/-1, each further word keeps it alive with probability 2^-nBits, so escaping a CARRY_LEN=8 (16-word) group takes about eleven consecutive extremal words. At 3 bits/word that is ~2^-33 per group crossing, which over 16384 groups and 178534 iterations gives an expected count near 1, matching the observation. Above ~10 bits/word the same estimate drops below 2^-100. bitsPerWord < 10 is also where Gpu.cpp forces useLongCarry, i.e. precisely where the unfused carryA/carryB path is selected. CARRY_LEN is a compile-time constant, so the added test folds away in all but the last unrolled iteration. Found and fixed by Claude Opus 5 (Anthropic's Claude Code), working with Mark Rose. Co-Authored-By: Claude Opus 5 --- src/cl/carryb.cl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/cl/carryb.cl b/src/cl/carryb.cl index ae9da09c..c675267a 100644 --- a/src/cl/carryb.cl +++ b/src/cl/carryb.cl @@ -32,6 +32,18 @@ KERNEL(G_W) carryB(P(Word2) io, CP(CarryABM) carryIn) { u32 p = i * WIDTH + me; bool biglit0 = frac_bits + (2*i) * FRAC_BPW_HI <= FRAC_BPW_HI; bool biglit1 = frac_bits + (2*i) * FRAC_BPW_HI >= -FRAC_BPW_HI; // Same as frac_bits + (2*i) * FRAC_BPW_HI + FRAC_BPW_HI <= FRAC_BPW_HI; + // carryB has no carry-out: a carry leaving the last word of this group would be dropped, silently + // losing 1 ulp at the first word of the next group. On the last word pair, add the carry into the + // high word without normalizing it -- as carryFinal does at the end of the fused carry chain -- so + // that nothing can escape the group. An un-normalized word holds the same value and is normalized + // by the next iteration. + if (i == CARRY_LEN - 1) { + Word2 a = io[p]; + a.x = carryStep(a.x + carry, &carry, biglit0); + a.y += carry; + io[p] = a; + return; + } io[p] = carryWord(io[p], &carry, biglit0, biglit1); if (!carry) { return; } } From 758ed28cadaa43199004d0c4638ab1d1eb06185c Mon Sep 17 00:00:00 2001 From: george Date: Tue, 15 Sep 2026 00:39:54 +0000 Subject: [PATCH 140/214] Tidied up AI change to bar() bug in carryFused. As suggested, duplicate bar removed from carryutil. --- src/cl/carryfused.cl | 90 +++++++++----------------------------------- src/cl/carryutil.cl | 5 +-- 2 files changed, 19 insertions(+), 76 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 19bbd4a6..ae1df813 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -227,15 +227,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -465,15 +459,9 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -708,15 +696,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -956,15 +938,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -1220,15 +1196,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -1510,15 +1480,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -1796,15 +1760,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -2076,15 +2034,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else @@ -2390,15 +2342,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #if OLD_FENCE // Order the carry stores ahead of the ready flag. This barrier must be reached by every work-item of the // workgroup: gr is uniform, but "me >= (WMUL-1) * G_W" is not, and a barrier under divergent control flow - // is undefined. As in bar(G_W), no barrier is needed when a sub-workgroup is a single wavefront. -#if WMUL == 1 - if (gr < H) { -#else + // is undefined. No barrier is needed when a sub-workgroup is a single wavefront. if (gr < H / WMUL) { -#endif -#if G_W > WAVEFRONT - bar(); -#endif + bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } #else diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index b92c7ead..4a63583d 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -147,10 +147,7 @@ void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *b // Write roundMax for high half of threads to local memory. Ignore threads not participating in the reduction. if (num_threads > WAVEFRONT) bar(); if (me >= num_threads / 2 && me < num_threads) lds[me - num_threads / 2] = u32RoundMax; - if (num_threads > WAVEFRONT) { - bar(); // work around a weird CUDA NVCC bug where two bar() calls are required???! (Titan V, CUDA 13.0, WMUL=2) - bar(); - } + if (num_threads > WAVEFRONT) bar(); // Low half of threads do a max if (me < num_threads / 2) { u32 highHalfMax = lds[me]; From 436bdb4802d04cdf00b60e27f1469986e796a7c1 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 15 Sep 2026 04:20:03 +0000 Subject: [PATCH 141/214] Got OLD_FENCE=0 working on nVidia. Not faster on TitanV. Added sync() and two flavors of barsync (learned during investigation of sharing LDS access). --- src/cl/base.cl | 51 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 4cf747e1..ab81f2d9 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -766,6 +766,24 @@ void PREFETCHL2(const __global void *addr) { #endif #endif +// Default settings for USE_REGISTER_BARSYNC. OpenCL on nVidia has compiler issues when USE_REGISTER_BARSYNC=0. Annoying, as register bar.sync is slower in many cases. +#ifndef USE_REGISTER_BARSYNC +#if CUDA_BACKEND +#define USE_REGISTER_BARSYNC 0 +#else +#define USE_REGISTER_BARSYNC 1 +#endif +#endif + +// Force divergent threads in a warp to converge. Early CUDA versions did not require this. Later versions lets the compiler choose to converge or not. +// I've not seen any cases where the compiler does not converge when we'd like it to, but just in case this routine will fix the problem. +void OVERLOAD sync() { +#if ENABLE_SYNC && HAS_PTX >= 600 // bar.warp.sync requires sm_60 support or higher + __asm("bar.warp.sync 0xffffffff;" : : ); +#endif +} + +// Create a barrier across all threads. void OVERLOAD bar(void) { // barrier(CLK_LOCAL_MEM_FENCE) is correct, but it turns out that on some GPUs // (in particular on Radeon VII and Radeon PRO VII) barrier(0) works as well and is faster. @@ -777,14 +795,39 @@ void OVERLOAD bar(void) { #endif } +// Create a barrier across a subset of threads OR across all threads if that is faster. void OVERLOAD bar(const u32 WG) { - if (WG > WAVEFRONT) { + if (WG <= WAVEFRONT) return; #if ENABLE_BARSYNC && HAS_PTX >= 200 // bar.sync with thread count requires sm_20 support or higher. Slower on TitanV, need to try on later nVidia GPUs. - __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); + __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); +// The above is GROSSLY slow on an RTX 5070Ti. The code below is much faster (may need to be expanded to handle more than four named barriers). +// WARNING, WARNING, WARNING: On TitanV using CUDA 12.9 tools and driver 580, similar code in LDSbar does not work in openCL (but works in CUDA build). +// if (get_local_id(0) / WG + 1 == 1) __asm("bar.sync 1, %0;" : : "n"(WG)); +// else if (get_local_id(0) / WG + 1 == 2) __asm("bar.sync 2, %0;" : : "n"(WG)); +// else if (get_local_id(0) / WG + 1 == 2) __asm("bar.sync 3, %0;" : : "n"(WG)); +// else __asm("bar.sync 4, %0;" : : "n"(WG)); #else - bar(); -#endif + bar(); +#endif +} + +// Create a barrier across a subset of threads. Substituting a barrier on all threads is not permitted. +void OVERLOAD barsync(const u32 numWG, const u32 WG) { + if (WG <= WAVEFRONT) return; +#if HAS_PTX >= 200 // bar.sync with thread count requires sm_20 support or higher. +#if USE_REGISTER_BARSYNC // bar.sync with a register is horribly slow on an RTX 5070Ti. + __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); +#else // WARNING, WARNING, WARNING: On TitanV using CUDA 12.9 tools and driver 580, this branch does not work in openCL (but works in CUDA build). + for (u32 i = 1; i <= numWG; i++) { + if (i == get_local_id(0) / WG + 1) { + __asm("bar.sync %0, %1;" : : "n"(i), "n"(WG)); + break; + } } +#endif +#else + #error - GPU not capable of barrier on a subset of threads +#endif } // nVidia GPUs (Hopper architecture sm 9.0 and later) support Programatic Dependent Launch where the tail end execution of one kernel can overlap From 0a87aec356212fc063613733c59a93630a200a43 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 15 Sep 2026 04:22:45 +0000 Subject: [PATCH 142/214] Oops, forgot to commit this file with OLD_FENCE fix. --- src/cl/carryfused.cl | 158 +++++++++++++++++++++++++++++-------------- 1 file changed, 106 insertions(+), 52 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index ae1df813..c8097fb5 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -212,15 +212,14 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else write_mem_fence(CLK_GLOBAL_MEM_FENCE); +#if !OLD_FENCE + barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -232,8 +231,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -267,7 +268,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -275,14 +276,18 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -315,6 +320,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -444,15 +450,14 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else write_mem_fence(CLK_GLOBAL_MEM_FENCE); +#if !OLD_FENCE + barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -464,8 +469,10 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -491,7 +498,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -499,14 +506,18 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -539,6 +550,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -681,15 +693,14 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (lowMe % WAVEFRONT == 0) { +#if !OLD_FENCE + barsync(WMUL, G_W); + if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -701,8 +712,10 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -729,7 +742,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -737,14 +750,18 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -776,6 +793,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -923,15 +941,14 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (lowMe % WAVEFRONT == 0) { +#if !OLD_FENCE + barsync(WMUL, G_W); + if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -943,8 +960,10 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -971,7 +990,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -979,14 +998,18 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1019,6 +1042,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -1180,16 +1204,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #endif for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } - // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else + // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (lowMe % WAVEFRONT == 0) { +#if !OLD_FENCE + barsync(WMUL, G_W); + if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -1201,8 +1224,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -1236,7 +1261,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -1244,14 +1269,18 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1284,6 +1313,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -1465,15 +1495,14 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (lowMe % WAVEFRONT == 0) { +#if !OLD_FENCE + barsync(WMUL, G_W); + if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -1485,8 +1514,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -1512,7 +1543,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -1520,14 +1551,18 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1560,6 +1595,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -1745,15 +1781,14 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else - write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (lowMe % WAVEFRONT == 0) { +#if !OLD_FENCE + barsync(WMUL, G_W); + if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -1765,8 +1800,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -1792,7 +1829,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -1800,14 +1837,18 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1840,6 +1881,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -2019,15 +2061,14 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else write_mem_fence(CLK_GLOBAL_MEM_FENCE); +#if !OLD_FENCE + barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -2039,8 +2080,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -2067,7 +2110,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -2075,14 +2118,18 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -2115,6 +2162,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } @@ -2327,15 +2375,14 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = 0; i < NW; ++i) { CSSTORE(&carryShuttlePtr[gr * WIDTH + CarryShuttleAccess(lowMe, i)], carry[i]); } // Tell next group that its carries are ready -#if OLD_FENCE - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); - write_mem_fence(CLK_GLOBAL_MEM_FENCE); -#else write_mem_fence(CLK_GLOBAL_MEM_FENCE); - if (lowMe % WAVEFRONT == 0) { +#if !OLD_FENCE + barsync(WMUL, G_W); + if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } + sync(); #endif } @@ -2347,8 +2394,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + sync(); #endif } #endif @@ -2374,7 +2423,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - // work_group_barrier(CLK_GLOBAL_MEM_FENCE, memory_scope_device); + sync(); bar(); #endif if (me < G_W) { @@ -2382,14 +2431,18 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; + sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } + sync(); + barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; + sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -2422,6 +2475,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } + sync(); } } From 088dd8d9fe811377d4957b8036988e713f2d11b6 Mon Sep 17 00:00:00 2001 From: george Date: Tue, 15 Sep 2026 06:53:44 +0000 Subject: [PATCH 143/214] Undid OLD_FENCE using barsync fix for nVidia. The !OLD_FENCE code is for eace WAVEFRONT to have its own flag. barsync operates per-WG, not per-WAVEFRONT, so the proposed "fix" broke AMD WIDTH>=1024. --- src/cl/carryfused.cl | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index c8097fb5..f7869fff 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -214,7 +214,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -283,7 +282,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -452,7 +450,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -513,7 +510,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -695,7 +691,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -757,7 +752,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -943,7 +937,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -1005,7 +998,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -1207,7 +1199,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -1276,7 +1267,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -1497,7 +1487,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -1558,7 +1547,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -1783,7 +1771,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -1844,7 +1831,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -2063,7 +2049,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -2125,7 +2110,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; @@ -2377,7 +2361,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE - barsync(WMUL, G_W); if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); @@ -2438,7 +2421,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - barsync(WMUL, G_W); mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; From 3dd7c5ddf94457e799dd50bf36f3ffcebbd6bfbc Mon Sep 17 00:00:00 2001 From: george Date: Tue, 15 Sep 2026 08:07:03 +0000 Subject: [PATCH 144/214] Claude-inspired fix for the nVidia OLD_FENCE=0 bug. Removed spurious sync() calls. --- src/cl/base.cl | 7 ++-- src/cl/carryfused.cl | 90 +++++++++----------------------------------- 2 files changed, 22 insertions(+), 75 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index ab81f2d9..a85341af 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -775,10 +775,11 @@ void PREFETCHL2(const __global void *addr) { #endif #endif -// Force divergent threads in a warp to converge. Early CUDA versions did not require this. Later versions lets the compiler choose to converge or not. -// I've not seen any cases where the compiler does not converge when we'd like it to, but just in case this routine will fix the problem. +// Force divergent threads in a warp to converge. AMD GCN does not require this, all threads in a WAVEFRONT operate in lockstep. Early CUDA versions did also. +// The sync is needed in cases where one thread is setting a flag or state on behalf of all the threads in a WAVEFRONT. For example, carryFused has thread 0 set +// the carries-are-ready flag on behalf of all 32 threads in a warp. void OVERLOAD sync() { -#if ENABLE_SYNC && HAS_PTX >= 600 // bar.warp.sync requires sm_60 support or higher +#if HAS_PTX >= 600 // bar.warp.sync requires sm_60 support or higher __asm("bar.warp.sync 0xffffffff;" : : ); #endif } diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index f7869fff..c9bc907d 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -214,11 +214,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -230,10 +230,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -267,7 +265,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -275,17 +272,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -318,7 +313,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -450,11 +444,11 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -466,10 +460,8 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -495,7 +487,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -503,17 +494,15 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -546,7 +535,6 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -691,11 +679,11 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -707,10 +695,8 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -737,7 +723,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -745,17 +730,15 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -787,7 +770,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -937,11 +919,11 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -953,10 +935,8 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -983,7 +963,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -991,17 +970,15 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1034,7 +1011,6 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -1199,11 +1175,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -1215,10 +1191,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -1252,7 +1226,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -1260,17 +1233,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1303,7 +1274,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -1487,11 +1457,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -1503,10 +1473,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -1532,7 +1500,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -1540,17 +1507,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1583,7 +1548,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -1771,11 +1735,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -1787,10 +1751,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -1816,7 +1778,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -1824,17 +1785,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -1867,7 +1826,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -2049,11 +2007,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -2065,10 +2023,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -2095,7 +2051,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -2103,17 +2058,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -2146,7 +2099,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } @@ -2361,11 +2313,11 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Tell next group that its carries are ready write_mem_fence(CLK_GLOBAL_MEM_FENCE); #if !OLD_FENCE + sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; atomic_store((atomic_uint *) &ready[pos], 1); } - sync(); #endif } @@ -2377,10 +2329,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut bar(G_W); #if WMUL == 1 if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #else if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } - sync(); #endif } #endif @@ -2406,7 +2356,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } - sync(); bar(); #endif if (me < G_W) { @@ -2414,17 +2363,15 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me == 0) ready[gr - 1] = 0; - sync(); #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); - mem_fence(CLK_GLOBAL_MEM_FENCE); + read_mem_fence(CLK_GLOBAL_MEM_FENCE); // Clear carry ready flag for next iteration if (me % WAVEFRONT == 0) ready[(gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT] = 0; - sync(); #endif #if HAS_ASM __asm("s_setprio 1"); @@ -2457,7 +2404,6 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut for (i32 i = NW-1; i; --i) { carry[i] = carry[i-1]; } carry[0] = carry[NW]; } - sync(); } } From 00b3935540d926786db7b93dcb2d6437a2c0bf93 Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 12:33:23 +0000 Subject: [PATCH 145/214] Args: make -version print the version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `-version` was accepted and threw "version" without printing anything — the log() call it once made is commented out, so the command line prpll -version exited with `Exiting because "version"` / `Bye` and no version. A launcher that records which build wrote a result (Task.cpp reports VERSION to PrimeNet in every result line) has nothing to read but the banner of a full start. Print the version to stdout, one plain line, without the log timestamp prefix and without the leading "v" (the same shape Task.cpp reports), then exit as before. Args.cpp gains the version.h include the commented call was missing. Reproduce: build, run `./prpll -version`; before: no version line; after: `8.0-57-g6cb4c12` (or the short SHA of a build without a v/prpll/* tag). --- src/Args.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Args.cpp b/src/Args.cpp index 179bbd83..d4744d68 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -5,9 +5,11 @@ #include "clwrap.h" #include "gpuid.h" #include "Proof.h" +#include "version.h" #include #include +#include #include #include #include @@ -286,7 +288,11 @@ void Args::parse(const string& line) { printHelp(); throw "help"; } if (key == "-version") { - // log("PRPLL %s\n", VERSION); + // Plain stdout, no log prefix: the flag exists for scripts and launchers + // that record which build wrote a result (Task.cpp reports VERSION to + // PrimeNet), so the one line must be the version and nothing else. + printf("%s\n", (VERSION[0] == 'v') ? VERSION + 1 : VERSION); + fflush(stdout); throw "version"; } if (key == "-info") { if (s.empty()) { From a8cc8b531b721c2c9afd3e2e2b6fa6c16f844966 Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 12:33:23 +0000 Subject: [PATCH 146/214] CUDA shim: fill buffers with 8-byte (and other word-multiple) patterns correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit clEnqueueFillBuffer in the CUDA shim handled 1- and 4-byte patterns and ZERO-FILLED everything else, with a comment that the common case is a zero fill. It is — Buffer::zero() is the bulk of the calls — but not the only one: Word is i64 (common.h), so Buffer::set(1), which Gpu::exponentiate() uses for exp == 0, asks for an 8-byte pattern of 1 and got a buffer of zeros under CUDA. The OpenCL path fills correctly. Fill an 8-byte pattern (and any pattern of N 32-bit words over a whole number of slots) with one strided cuMemsetD2D32Async per word: pitch = the pattern size, width = one element, height = the slot count — two driver calls for a double/i64 pattern, no host-side pattern buffer, no extra copy. 2-byte patterns take cuMemsetD16Async. A pattern the contract has no memset for returns CL_INVALID_VALUE instead of writing zeros. Reproduce (CUDA build): a Buffer of 8 words, set(1), read back — before: all zero; after: 1, 0, 0, …. Or -tune / a P-1 stage that reaches exponentiate(_, 0). --- src/cuda/clwrap_cuda.cpp | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 77aef8cb..01b74e25 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -855,13 +855,32 @@ int clEnqueueFillBuffer(cl_command_queue q, cl_mem buf, const void* pattern, unsigned char val; memcpy(&val, pattern, 1); r = cuMemsetD8Async(buf->ptr + offset, val, size, q->stream); + } else if (patternSize == 2) { + unsigned short val; + memcpy(&val, pattern, 2); + r = cuMemsetD16Async(buf->ptr + offset, val, size / 2, q->stream); } else if (patternSize == 4) { unsigned int val; memcpy(&val, pattern, 4); r = cuMemsetD32Async(buf->ptr + offset, val, size / 4, q->stream); + } else if (patternSize % 4 == 0 && size % patternSize == 0) { + // A pattern of N 32-bit words (Buffer//: N == 2): one + // strided memset per word writes word i of every pattern-sized slot — + // pitch = the pattern, width = one element, height = the slot count. + // Two driver calls instead of a host-side pattern buffer and a copy. + r = CUDA_SUCCESS; + size_t const slots = size / patternSize; + for (size_t i = 0; i < patternSize / 4 && r == CUDA_SUCCESS; ++i) { + unsigned int word; + memcpy(&word, static_cast(pattern) + 4 * i, 4); + r = cuMemsetD2D32Async(buf->ptr + offset + 4 * i, patternSize, word, 1, slots, q->stream); + } } else { - // For other pattern sizes, fall back to memset 0 (common case is zero-fill) - r = cuMemsetD8Async(buf->ptr + offset, 0, size, q->stream); + // Neither a memset width nor a whole number of word-multiple slots: the + // OpenCL contract has no answer here, and a silent zero-fill (the old + // fallback) would hand the caller data it did not ask for. + if (event) *event = nullptr; + return CL_INVALID_VALUE; } if (event) *event = nullptr; return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; From 8b955255603064e077b89c0500ffeb5002e8ba44 Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 12:33:23 +0000 Subject: [PATCH 147/214] main: exit 1 on an unhandled exception, 0 on a normal end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main() returned 0 on every path — the queue running dry, a SIGINT stop, -h, and every exception it caught: a kernel the driver refused to compile ("Can't compile transpose.cl"), "No device", a bad argument. A supervisor that restarts PRPLL had no way to tell "nothing to do" from "cannot run" except by parsing the log: with an assignment queued and exit 0 it would either treat a broken install as idle and respawn it forever, or treat every clean exit as a crash. Return 0 for the ends a run reaches on purpose — the queue ran dry, "stop requested", "help", "version" — and 1 for any other exception main() catches. The log lines are unchanged. Reproduce: run with an assignment queued on a machine whose driver rejects the kernels (or `-device 99`); before: exit status 0; after: 1. `./prpll -h` and a SIGINT stop still exit 0. --- src/main.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index c19294b8..becccf9b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -16,6 +16,7 @@ #include "Gpu.h" #include "tune.h" +#include #include #include #include @@ -45,6 +46,12 @@ static void gpuWorker(GpuCommon shared, i32 instance) { extern int putenv(char *); #endif +// The exceptions that end a run on purpose: the user's stop, and the two +// flags that only print. Everything else thrown to main() is a failure. +static bool isCleanExit(const char *reason) { + return !strcmp(reason, "stop requested") || !strcmp(reason, "help") || !strcmp(reason, "version"); +} + int main(int argc, char **argv) { //!MSVC version support #ifdef _MSC_VER @@ -62,7 +69,11 @@ int main(int argc, char **argv) { setenv("ROC_SIGNAL_POOL_SIZE", "32", 0); #endif - int const exitCode = 0; + // 0 for a normal end — the queue ran dry, a stop was requested, -h or + // -version — and 1 for an exception nobody else classified (a kernel that + // would not compile, a missing device, a bad argument), so a supervisor + // can tell "out of work" from "cannot run" without parsing the log. + int exitCode = 0; try { string const mainLine = Args::mergeArgs(argc, argv); @@ -129,10 +140,12 @@ int main(int argc, char **argv) { } } catch (const char *mes) { log("Exiting because \"%s\"\n", mes); + exitCode = isCleanExit(mes) ? 0 : 1; } catch (const string& mes) { log("Exiting because \"%s\"\n", mes.c_str()); + exitCode = isCleanExit(mes.c_str()) ? 0 : 1; } log("Bye\n"); - return exitCode; // not used yet. + return exitCode; } From 2ba2368f5c5ca9d4c7d02c6186f8f52ff78497b3 Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 12:33:23 +0000 Subject: [PATCH 148/214] Signal: stop gracefully on SIGTERM and on a "stop" file in the run directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signal.cpp installs a handler for SIGINT only. SIGTERM — what systemd, supervisors and `kill` send first — ends the process at once, forfeiting the work since the last savefile (up to logStep = 20,000 iterations), when the same request as SIGINT would have finished the block, run the Gerbicz check and written a savefile a few seconds later. On Windows no console signal reaches a hidden child at all, so a launcher has only TerminateProcess, with the same loss. Handle SIGTERM like SIGINT, and let Signal::stopRequested() also honor a file named "stop" in the run directory (the current directory after -dir): seen at the next block boundary like a signal, removed once seen so the next start is not a stop, logged when it fires. The check is one exists() per stopRequested() call — once per 1000-iteration block. Reproduce: start a PRP test; `kill -TERM ` — before: the process dies mid-block with no "Stopping, please wait.." and the savefile stays at the previous 20,000-iteration mark; after: the block completes, the check runs, the savefile is written, exit 0. `touch stop` in the run directory does the same from any platform. --- src/Signal.cpp | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/Signal.cpp b/src/Signal.cpp index 49b3bed3..f18b532b 100644 --- a/src/Signal.cpp +++ b/src/Signal.cpp @@ -1,32 +1,60 @@ // Copyright (C) Mihai Preda. #include "Signal.h" +#include "log.h" #include +#include +#include using namespace std; static volatile sig_atomic_t signalled = 0; -static void (* volatile oldHandler)(int) = nullptr; +static void (* volatile oldIntHandler)(int) = nullptr; +static void (* volatile oldTermHandler)(int) = nullptr; static void signalHandler(int signal) { signalled = signal; } +// A file named "stop" in the run directory asks for the same graceful stop +// a SIGINT does: finish the block, verify it, write the savefile, exit. +// It is the stop a launcher can request where no signal reaches the +// process — a hidden Windows child has no console to deliver a Ctrl-C to, +// and TerminateProcess forfeits the work since the last savefile. The file +// is removed once seen, so the next start is not a stop. +static const char *STOP_FILE = "stop"; + +static bool stopFileSeen() { + error_code ec; + if (!filesystem::exists(STOP_FILE, ec) || ec) { return false; } + filesystem::remove(STOP_FILE, ec); + log("Stop requested by the '%s' file\n", STOP_FILE); + return true; +} + Signal::Signal() { - if (!oldHandler) { - oldHandler = signal(SIGINT, signalHandler); + if (!oldIntHandler) { + oldIntHandler = signal(SIGINT, signalHandler); + // SIGTERM is what service managers and supervisors send first; unhandled + // it ends the process at once. + oldTermHandler = signal(SIGTERM, signalHandler); isOwner = true; } } Signal::~Signal() { release(); } -unsigned Signal::stopRequested() { return signalled; } +unsigned Signal::stopRequested() { + if (!signalled && stopFileSeen()) { signalled = SIGTERM; } + return signalled; +} void Signal::release() { if (isOwner) { isOwner = false; - signal(SIGINT, oldHandler); - oldHandler = nullptr; + signal(SIGINT, oldIntHandler); + signal(SIGTERM, oldTermHandler); + oldIntHandler = nullptr; + oldTermHandler = nullptr; } } From a34b052345879868296c55b471156525a19dae27 Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 12:33:23 +0000 Subject: [PATCH 149/214] Makefile: let VERSION be passed in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/version.inc is regenerated from `git describe` on every build. A tree without .git — a source tarball, a CI job building an exported copy, a distro package — gets an empty version string: `git describe` fails, `basename` of nothing is nothing, and the binary carries "" — which Task.cpp asserts non-empty before reporting a result. The only way around it was to edit the Makefile or pre-write src/version.inc and hope the rule's diff keeps it. Make VERSION a variable with the git describe as its default (`VERSION ?=`, expanded lazily so git runs only when the value is needed) and write that into version.inc. `make VERSION=v8.0-57-g6cb4c12` builds a binary that reports exactly that; a plain `make` is unchanged. Reproduce: `git archive HEAD | tar -x -C /tmp/t && make -C /tmp/t` — before: version.inc is `""`; after: `make -C /tmp/t VERSION=` gives the intended string. --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c41aa5fb..3d15b836 100644 --- a/Makefile +++ b/Makefile @@ -118,8 +118,14 @@ $(DEPDIR)/%.d: ; src/version.cpp : src/version.inc +# The version string compiled into the binary and reported to PrimeNet in +# every result. Defaults to `git describe` of the checkout; a build from an +# exported tree (no .git) or a packager that wants the upstream string passes +# it explicitly: make VERSION=v8.0-57-g6cb4c12 +VERSION ?= $(shell basename `git describe --tags --long --dirty --always --match 'v/prpll/*'`) + src/version.inc: FORCE - echo \"`basename \`git describe --tags --long --dirty --always --match v/prpll/*\``\" > $(BIN)/version.new + echo \"$(VERSION)\" > $(BIN)/version.new diff -q -N $(BIN)/version.new $@ >/dev/null || mv $(BIN)/version.new $@ echo Version: `cat $@` From 7e0dfff83bcdc4e8e84e63161aef50937474be9b Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 12:33:23 +0000 Subject: [PATCH 150/214] CUDA shim: answer CL_DEVICE_TOPOLOGY_AMD from the device's PCI bus id, so -pci selects a CUDA device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -pci resolves a device through the AMD topology extension (gpuid.cpp getPosFromBdf → clwrap.cpp getBdfFromDevice → CL_DEVICE_TOPOLOGY_AMD), which the CUDA shim answered with CL_INVALID_VALUE for every device — so on NVIDIA the only selector was the enumeration ordinal, -device N. That ordinal is not stable: CUDA enumerates FASTEST_FIRST unless CUDA_DEVICE_ORDER says otherwise (not nvidia-smi's PCI order, which launchers and monitoring tools use), and a second card or a driver update renumbers. The bus id is what nvidia-smi, lspci and every launcher already know the card by. The shim now fills cl_device_topology_amd from cuDeviceGetPCIBusId() ("0000:6a:00.0", domain:bus:device.function; a missing domain is tolerated) with the PCIe type tag, so getBdfFromDevice formats the same "6a:00.0" it formats for an AMD device and getPosFromBdf matches it. A driver that cannot say keeps CL_INVALID_VALUE; CL_DEVICE_BOARD_NAME_AMD and CL_DEVICE_PCIE_ID_AMD still say no, as before. Reproduction: on a machine with two NVIDIA cards, `nvidia-smi --query-gpu=index,name,pci.bus_id` and `prpll -device 1 -h` (the banner names the card) disagree once the faster card sits on the higher bus id; `prpll -pci 01:00.0` selects the card at 01:00.0 with this patch and exits with "OpenCL device with BDF '01:00.0' not found" without it. Measurement: none — device selection only; no kernel or queue changes. --- src/cuda/clwrap_cuda.cpp | 30 +++++++++++++++++++++++++++--- src/cuda/tinycuda.h | 3 ++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 77aef8cb..0cd65af9 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -1017,10 +1017,34 @@ int clGetDeviceInfo(cl_device_id dev, cl_device_info info, size_t size, void* va if (value && size >= 1) memcpy(value, empty, 1); break; } - case CL_DEVICE_BOARD_NAME_AMD: - case CL_DEVICE_PCIE_ID_AMD: case CL_DEVICE_TOPOLOGY_AMD: { - // AMD-specific queries — return failure + // The device's PCIe position in the AMD extension's shape, so `-pci + // ` selects a CUDA device the way it selects an AMD + // one (gpuid.cpp getPosFromBdf → clwrap.cpp getBdfFromDevice). The + // enumeration ordinal `-device N` names is not stable — CUDA orders + // FASTEST_FIRST unless CUDA_DEVICE_ORDER says otherwise, and a second + // card or a driver update can renumber — while the bus id is what + // nvidia-smi and every launcher already know the card by. + char bdf[32] = {0}; + if (cuDeviceGetPCIBusId(bdf, sizeof(bdf), dev->dev) != CUDA_SUCCESS) return CL_INVALID_VALUE; + // "0000:6a:00.0" — domain:bus:device.function; tolerate a missing domain. + unsigned domain = 0, bus = 0, device = 0, function = 0; + if (sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &device, &function) != 4 && + sscanf(bdf, "%x:%x.%x", &bus, &device, &function) != 3) { + return CL_INVALID_VALUE; + } + cl_device_topology_amd top{}; + top.pcie.type = CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD; + top.pcie.bus = (char) bus; + top.pcie.device = (char) device; + top.pcie.function = (char) function; + if (sizeRet) *sizeRet = sizeof(top); + if (value && size >= sizeof(top)) memcpy(value, &top, sizeof(top)); + break; + } + case CL_DEVICE_BOARD_NAME_AMD: + case CL_DEVICE_PCIE_ID_AMD: { + // AMD-specific queries with no CUDA counterpart — return failure return CL_INVALID_VALUE; } case CL_DEVICE_GLOBAL_FREE_MEMORY_AMD: { diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index c8ff49ef..f628d4f7 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -212,9 +212,10 @@ using cl_queue = cl_command_queue; #define CL_DEVICE_COMPUTE_CAPABILITY_MAJOR_NV 0x4000 #define CL_DEVICE_COMPUTE_CAPABILITY_MINOR_NV 0x4001 -// AMD-specific (unused but must exist for compilation) +// AMD-specific; the shim answers CL_DEVICE_TOPOLOGY_AMD from the CUDA device's PCI bus id #define CL_DEVICE_PCIE_ID_AMD 0x4034 #define CL_DEVICE_TOPOLOGY_AMD 0x4037 +#define CL_DEVICE_TOPOLOGY_TYPE_PCIE_AMD 1 #define CL_DEVICE_BOARD_NAME_AMD 0x4038 #define CL_DEVICE_GLOBAL_FREE_MEMORY_AMD 0x4039 From 57947a131e744bbd520dfd898f6ca5686a62ff1f Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 09:20:52 +0000 Subject: [PATCH 151/214] CUDA shim: load NVRTC's CUBIN, fall back to the PTX JIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shim asked NVRTC for PTX only and handed it to cuModuleLoadDataEx, which runs the driver's JIT compiler on every start: ~36 kernels through ptxas each time (`-cache` saves the PTX, not the SASS), and a driver older than the toolkit refuses PTX newer than it knows — CUDA_ERROR_UNSUPPORTED_PTX_VERSION, "Unsupported .version 9.1; current version is '9.0'" — so the build cannot start at all on that machine even though its GPU is fully supported. The same shim compiles for a real architecture (--gpu-architecture=sm_XY), so NVRTC has already run ptxas and holds the SASS. Ask for both: nvrtcGetCUBIN (NVRTC >= 11.1; empty when the toolkit or a compute_XY target gives none) beside the PTX. Loading tries the CUBIN first — no JIT, no PTX-version check — and on a driver that rejects it logs the reason and loads the PTX as before. --maxrregcount is applied by NVRTC's own ptxas in the CUBIN; the PTX fallback keeps the .maxnreg splice the JIT path needs. PRPLL_PTX_ONLY=1 forces the old path for an A/B. The PTX text stays on the program in every path, because clCreateKernel reads each kernel's declared work-group size from it (.maxntid, what Kernel::groupSize launches with); a CUBIN carries that only as ELF metadata. So the kernel cache stores the PTX followed by a marker line and the CUBIN, the reader splits on the marker, and a cache file from before this change (plain PTX, no marker) loads exactly as it did. A bare ELF is refused as CL_INVALID_BINARY so KernelCompiler recompiles instead of launching every kernel with the 256-thread fallback. Reproduce: start on a driver older than the build's toolkit (deacix-win, 2026-09-09: CUDA 13.2 build, 13.0 driver) — before: every start dies in clLinkProgram; after: the CUBIN loads. On any machine, the start-up kernel compile time drops by the JIT's share (measure with -verbose). Co-authored-by: Cursor --- src/cuda/clwrap_cuda.cpp | 85 ++++++++++++++++++++++++++++++++++------ src/cuda/cudawrap.cpp | 24 ++++++++++-- src/cuda/cudawrap.h | 9 +++++ src/cuda/tinycuda.h | 3 +- 4 files changed, 104 insertions(+), 17 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 77aef8cb..9ce2e14e 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -193,24 +193,73 @@ cl_program clCreateProgramWithSource(cl_context ctx, unsigned count, const char* return prog; } +// The "binary" the kernel cache stores (CL_PROGRAM_BINARIES) and hands back +// (clCreateProgramWithBinary): the PTX text, then — when NVRTC produced a +// CUBIN that this driver loaded — this marker and the CUBIN. The PTX stays, +// and stays first, because clCreateKernel reads each kernel's declared +// work-group size (.maxntid) and its PDL wait from the PTX text; a CUBIN +// carries neither as text. A blob without the marker is a plain PTX (the +// format before the CUBIN), and loads as before. +static const char CUBIN_MARKER[] = "\n// PRPLL-CUBIN\n"; +static const size_t CUBIN_MARKER_LEN = sizeof(CUBIN_MARKER) - 1; + +static string cacheBlob(const _cl_program* prog) { + if (prog->cubin.empty()) { return prog->ptx; } + string blob; + blob.reserve(prog->ptx.size() + CUBIN_MARKER_LEN + prog->cubin.size()); + blob += prog->ptx; + blob += CUBIN_MARKER; + blob += prog->cubin; + return blob; +} + +// Loads prog->cubin when there is one — SASS for this device, so no JIT and +// no PTX-version check to fail on a driver older than the toolkit — and +// otherwise, or when the driver rejects it (logged), the PTX through the JIT. +// A rejected CUBIN is dropped so the cache stores what loaded. +static CUresult loadModule(_cl_program* prog, unsigned nOpts, CUjit_option* opts, void** optVals) { + if (!prog->cubin.empty()) { + CUresult const r = cuModuleLoadDataEx(&prog->module, prog->cubin.data(), nOpts, opts, optVals); + if (r == CUDA_SUCCESS) { return r; } + const char* errName = nullptr; + cuGetErrorName(r, &errName); + fprintf(stderr, "CUBIN rejected by the driver: %s (%d) — loading the PTX through the JIT instead\n", errName ? errName : "?", (int)r); + prog->cubin.clear(); + } + return cuModuleLoadDataEx(&prog->module, prog->ptx.c_str(), nOpts, opts, optVals); +} + cl_program clCreateProgramWithBinary(cl_context ctx, unsigned /*nDevices*/, const cl_device_id*, const size_t* lengths, const unsigned char** binaries, int* binaryStatus, int* err) { - // "Binary" in CUDA land = PTX string auto* prog = new _cl_program; prog->context = ctx; if (lengths && binaries && lengths[0] > 0) { - prog->ptx.assign((const char*)binaries[0], lengths[0]); + string blob((const char*)binaries[0], lengths[0]); + // A bare ELF is a CUBIN without its PTX: nothing to read the kernels' + // work-group sizes from. Refuse it; the caller recompiles and overwrites. + if (blob.compare(0, 4, "\177ELF", 4) == 0) { + fprintf(stderr, "Cached kernel binary is a bare CUBIN (no PTX): recompiling\n"); + if (binaryStatus) binaryStatus[0] = CL_INVALID_BINARY; + if (err) *err = CL_INVALID_BINARY; + return prog; + } + size_t const mark = blob.find(CUBIN_MARKER); + if (mark == string::npos) { + prog->ptx = std::move(blob); + } else { + prog->ptx = blob.substr(0, mark); + prog->cubin = blob.substr(mark + CUBIN_MARKER_LEN); + } prog->compiled = true; - // Load the module (JIT-compile PTX to SASS) ensureContextCurrent(); - CUresult const r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); + CUresult const r = loadModule(prog, 0, nullptr, nullptr); if (r == CUDA_SUCCESS) { prog->moduleLoaded = true; moduleRetain(prog->module); // program owns one reference if (binaryStatus) binaryStatus[0] = CL_SUCCESS; } else { - fprintf(stderr, "cuModuleLoadData from cache failed: %d, PTX size=%zu\n", (int)r, lengths[0]); + fprintf(stderr, "cuModuleLoadData from cache failed: %d, blob size=%zu\n", (int)r, lengths[0]); prog->compiled = false; if (binaryStatus) binaryStatus[0] = CL_INVALID_BINARY; if (err) { *err = CL_INVALID_BINARY; return prog; } @@ -379,7 +428,11 @@ int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id } try { - prog->ptx = NvrtcProgram::compile(processedSource, "prpll_kernel.cu", nvrtcOpts, nvrtcHeaders); + auto images = NvrtcProgram::compileImages(processedSource, "prpll_kernel.cu", nvrtcOpts, nvrtcHeaders); + prog->ptx = std::move(images.ptx); + // PRPLL_PTX_ONLY=1 keeps the driver's JIT path, for comparing the two. + static const bool ptxOnly = getenv("PRPLL_PTX_ONLY") != nullptr; + prog->cubin = ptxOnly ? string{} : std::move(images.cubin); prog->compiled = true; g_lastBuildLog.clear(); } catch (const exception& e) { @@ -404,7 +457,9 @@ int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id return CL_COMPILE_PROGRAM_FAILURE; } - // If --maxrregcount is set it seems nvrtc compile ignores the setting. Instead modify the PTX and load the modified PTX. + // NVRTC applies --maxrregcount when it runs ptxas itself, i.e. in the CUBIN. + // PTX carries no register cap and the driver JIT never sees the option, so + // the PTX fallback gets a .maxnreg directive spliced in ahead of every entry. if (maxregcount) { string const maxntidPattern = ".maxntid "; @@ -434,6 +489,7 @@ cl_program clLinkProgram(cl_context ctx, unsigned /*nDevices*/, const cl_device auto* linked = new _cl_program; linked->context = ctx; linked->ptx = progs[0]->ptx; + linked->cubin = progs[0]->cubin; linked->compiled = true; // Carry preprocessed source through for KERNEL(N) parsing in clCreateKernel for (unsigned i = 0; i < nProgs; ++i) { @@ -454,7 +510,7 @@ cl_program clLinkProgram(cl_context ctx, unsigned /*nDevices*/, const cl_device (void*)(size_t)sizeof(jitErrorLog), (void*)jitErrorLog, (void*)(size_t)sizeof(jitInfoLog), (void*)jitInfoLog }; - CUresult const r = cuModuleLoadDataEx(&linked->module, linked->ptx.c_str(), 4, jitOpts, jitOptVals); + CUresult const r = loadModule(linked, 4, jitOpts, jitOptVals); if (r != CUDA_SUCCESS) { const char* errName = nullptr; cuGetErrorName(r, &errName); @@ -512,7 +568,7 @@ int clBuildProgram(cl_program prog, unsigned nDevices, const cl_device_id* devic int const err = clCompileProgram(prog, nDevices, devices, options, 0, nullptr, nullptr, nullptr, nullptr); if (err != CL_SUCCESS) return err; - CUresult const r = cuModuleLoadData(&prog->module, prog->ptx.c_str()); + CUresult const r = loadModule(prog, 0, nullptr, nullptr); if (r != CUDA_SUCCESS) return CL_BUILD_PROGRAM_FAILURE; prog->moduleLoaded = true; moduleRetain(prog->module); // program owns one reference @@ -533,15 +589,20 @@ int clGetProgramBuildInfo(cl_program /*prog*/, cl_device_id, cl_program_build_i int clGetProgramInfo(cl_program prog, cl_program_info info, size_t size, void* value, size_t* sizeRet) { if (!prog) return CL_INVALID_PROGRAM; + // The cache blob — PTX, then the CUBIN behind CUBIN_MARKER when one + // loaded; see clCreateProgramWithBinary for the reading side. if (info == CL_PROGRAM_BINARY_SIZES) { - size_t ptxSize = prog->ptx.size(); + size_t blobSize = cacheBlob(prog).size(); if (sizeRet) *sizeRet = sizeof(size_t); - if (value && size >= sizeof(size_t)) memcpy(value, &ptxSize, sizeof(size_t)); + if (value && size >= sizeof(size_t)) memcpy(value, &blobSize, sizeof(size_t)); } else if (info == CL_PROGRAM_BINARIES) { if (sizeRet) *sizeRet = sizeof(unsigned char*); if (value && size >= sizeof(unsigned char*)) { auto* const* ptrs = (unsigned char**)value; - if (ptrs[0]) memcpy(ptrs[0], prog->ptx.data(), prog->ptx.size()); + if (ptrs[0]) { + string const blob = cacheBlob(prog); + memcpy(ptrs[0], blob.data(), blob.size()); + } } } return CL_SUCCESS; diff --git a/src/cuda/cudawrap.cpp b/src/cuda/cudawrap.cpp index f4b4cb6a..c8d98e2b 100644 --- a/src/cuda/cudawrap.cpp +++ b/src/cuda/cudawrap.cpp @@ -446,6 +446,12 @@ std::string NvrtcProgram::preprocessOpenCL(const std::string& source) { std::string NvrtcProgram::compile(const std::string& source, const std::string& name, const std::vector& options, const std::vector>& headers) { + return compileImages(source, name, options, headers).ptx; +} + +NvrtcProgram::Images NvrtcProgram::compileImages(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers) { // Prepare header arrays std::vector headerSources, headerNames; for (auto& [hName, hSource] : headers) { @@ -482,14 +488,24 @@ for (auto& o : options) opts.push_back(o.c_str()); throw std::runtime_error("NVRTC compilation failed for " + name); } - // Get PTX + Images images; size_t ptxSize; NVRTC_CHECK(nvrtcGetPTXSize(prog, &ptxSize)); - std::string ptx(ptxSize, '\0'); - NVRTC_CHECK(nvrtcGetPTX(prog, ptx.data())); + images.ptx.assign(ptxSize, '\0'); + NVRTC_CHECK(nvrtcGetPTX(prog, images.ptx.data())); + +#if CUDA_VERSION >= 11010 + // The CUBIN exists only when the architecture was a real sm_XY (not + // compute_XY); a zero size means NVRTC has none to give, not an error. + size_t cubinSize = 0; + if (nvrtcGetCUBINSize(prog, &cubinSize) == NVRTC_SUCCESS && cubinSize > 0) { + images.cubin.assign(cubinSize, '\0'); + if (nvrtcGetCUBIN(prog, images.cubin.data()) != NVRTC_SUCCESS) { images.cubin.clear(); } + } +#endif nvrtcDestroyProgram(&prog); - return ptx; + return images; } // ---- Kernel launcher ---- diff --git a/src/cuda/cudawrap.h b/src/cuda/cudawrap.h index 2be6f72b..ac3639e3 100644 --- a/src/cuda/cudawrap.h +++ b/src/cuda/cudawrap.h @@ -119,6 +119,15 @@ struct NvrtcProgram { static std::string compile(const std::string& source, const std::string& name, const std::vector& options, const std::vector>& headers = {}); + + // PTX plus, when NVRTC ran its ptxas for a real --gpu-architecture=sm_XY + // (NVRTC >= 11.1), the CUBIN — SASS the driver loads without a JIT, with + // every option (--maxrregcount included) already applied. `cubin` is empty + // when the toolkit or the architecture flag gave none. + struct Images { std::string ptx; std::string cubin; }; + static Images compileImages(const std::string& source, const std::string& name, + const std::vector& options, + const std::vector>& headers = {}); }; // ---- Kernel launcher ---- diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index c8ff49ef..3a4dc3b7 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -46,7 +46,8 @@ struct _cl_program { cl_context context; std::string source; // OpenCL source (before NVRTC compilation) std::string preprocessedSource; // CUDA source after preprocessOpenCL (for parsing __launch_bounds__) - std::string ptx; // Compiled PTX (after NVRTC compilation) + std::string ptx; // Compiled PTX (after NVRTC compilation); always the text — clCreateKernel reads .maxntid from it + std::string cubin; // NVRTC's CUBIN for this device's sm, when it produced one; loaded ahead of the PTX JIT CUmodule module{}; // Loaded module (after cuModuleLoadData) bool compiled{false}; bool moduleLoaded{false}; From 256678ad447d0175ecd6369a73036b986af3a1ea Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 07:34:32 +0000 Subject: [PATCH 152/214] KernelCompiler: compile the kernels in parallel under CUDA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KernelCompiler::load() has an async path that was switched off (#if 0) because the ROCm compiler serializes parallel builds — so under CUDA the ~36 kernels of a Gpu compile one after another through NVRTC (and, before the CUBIN change, the driver JIT): a minute or more on every start of a work unit without -cache, and dozens of times over a -tune. NVRTC compiles independently per thread; nothing in the shim needed the serialization except that its shared state assumed it. Under CUDA, compile on parallel threads, bounded with a counting semaphore to the core count and to eight — an NVRTC instance holds a few hundred MB while it runs, so 36 at once, or one per thread of a 32-thread machine, would take gigabytes of host memory for a speedup that eight threads over 36 kernels already give most of. The shim's shared state becomes safe for it: the module reference counts sit behind a mutex (the CUDA context is already made current per thread), the build log is per program instead of a process-wide "last log" (clGetProgramBuildInfo now reads the program it was asked about — before, it reported whichever compile finished last), and the two debug-dump counters are atomics. The OpenCL build keeps the serial path and its reasoning. The log file and its context are thread-local (one gpuowl-N.log per worker instance), so a compile thread starts with neither and its "saving binary" / "Loaded … ms" lines would reach stdout alone, without the exponent — measured on an RTX 4090: 24 kernels loaded, zero lines in gpuowl-0.log. The thread therefore adopts the starting worker's log file and context for the task (LogLink / LogLinkScope in log.h): the lines land where -verbose readers look for them, prefixed as before. Measured (RTX 4090, WSL2, CUDA 12.0 NVRTC, 32-thread host): start to the first iteration line on a cold kernel cache 5 s → 4 s; the sum of the per-kernel load times 10.2 s → 8.0 s of thread time. The compile is far cheaper on this host than the minute the review estimated, so the gain is one second here; -verbose prints each kernel's load time on any machine. Co-authored-by: Cursor --- src/KernelCompiler.cpp | 28 ++++++++++++++++++---- src/cuda/clwrap_cuda.cpp | 51 ++++++++++++++++++++++++++-------------- src/cuda/tinycuda.h | 1 + src/log.cpp | 10 +++++++- src/log.h | 21 +++++++++++++++++ 5 files changed, 88 insertions(+), 23 deletions(-) diff --git a/src/KernelCompiler.cpp b/src/KernelCompiler.cpp index e5d22520..ab3144d5 100644 --- a/src/KernelCompiler.cpp +++ b/src/KernelCompiler.cpp @@ -8,6 +8,9 @@ #include #include #include +#include +#include +#include #include using namespace std; @@ -138,11 +141,28 @@ KernelHolder KernelCompiler::loadAux(const string& fileName, const string& kerne } std::future KernelCompiler::load(const string& fileName, const string& kernelName, const string& args) const { -#if 0 - // Do the compilation in parallel on a separate thread. - // Unfortunatelly no benefit on ROCm (the compiler serializes). - return async(std::launch::async, &KernelCompiler::loadAux, this, fileName, kernelName, args); +#ifdef CUDA_BACKEND + // NVRTC compiles independently per thread, so the ~36 kernels of a Gpu + // compile in parallel — bounded to the core count and to eight: each + // NVRTC instance holds a few hundred MB while it runs, so 36 at once, or + // one per thread of a 32-thread machine, would take gigabytes of host + // memory for a speedup that eight threads over 36 kernels already give + // most of. The CUDA shim makes the context current per thread and guards + // its shared module counts. The thread logs through this worker's log + // file and context (LogLink): the log is thread-local, and a fresh thread + // would otherwise print its "Loaded" lines to stdout alone, unprefixed. + static std::counting_semaphore<8> slots{std::max(1u, std::min(8u, std::thread::hardware_concurrency()))}; + LogLink const link = logLink(); + return async(std::launch::async, [this, fileName, kernelName, args, link] { + slots.acquire(); + struct Release { std::counting_semaphore<8>& s; ~Release() { s.release(); } } release{slots}; + LogLinkScope const logScope{link}; + return loadAux(fileName, kernelName, args); + }); #else + // Serial: the ROCm compiler serializes parallel builds anyway (no benefit + // measured), and the OpenCL runtime's thread-safety for concurrent + // clCompileProgram varies by vendor. std::promise promise; promise.set_value(loadAux(fileName, kernelName, args)); return promise.get_future(); diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 9ce2e14e..cd4b6858 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -52,22 +54,35 @@ static void ensureContextCurrent() { // device memory grows unbounded and eventually cuLaunchKernel fails with // CUDA_ERROR_OUT_OF_MEMORY. // -// Loading is single-threaded (KernelCompiler's async path is disabled), so a -// plain map without locking is sufficient. +// KernelCompiler compiles kernels on parallel threads under CUDA, each +// loading its module and creating its kernel, so the map is shared: one +// mutex around every access (the counts are the only state the compile +// threads share; the CUDA context is per-thread current, see +// ensureContextCurrent). static std::map g_moduleRefCount; +static std::mutex g_moduleRefMutex; static void moduleRetain(CUmodule m) { - if (m) { ++g_moduleRefCount[m]; } + if (!m) return; + std::lock_guard lock(g_moduleRefMutex); + ++g_moduleRefCount[m]; } static void moduleRelease(CUmodule m) { if (!m) return; - auto it = g_moduleRefCount.find(m); - if (it == g_moduleRefCount.end()) return; // untracked module — leave as-is - if (--it->second <= 0) { + bool unload = false; + { + std::lock_guard lock(g_moduleRefMutex); + auto it = g_moduleRefCount.find(m); + if (it == g_moduleRefCount.end()) return; // untracked module — leave as-is + if (--it->second <= 0) { + g_moduleRefCount.erase(it); + unload = true; + } + } + if (unload) { ensureContextCurrent(); cuModuleUnload(m); - g_moduleRefCount.erase(it); } } @@ -269,8 +284,6 @@ cl_program clCreateProgramWithBinary(cl_context ctx, unsigned /*nDevices*/, con return prog; } -// Build log storage (per-program) -static string g_lastBuildLog; int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id* devices, const char* options, unsigned numHeaders, const cl_program* headers, const char* const* headerNames, @@ -419,9 +432,8 @@ int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id // Debug: dump NVRTC options when dumping PTX { static const char* dumpPrefix = getenv("PRPLL_DUMP_PTX"); - static bool dumpedOnce = false; - if (dumpPrefix && !dumpedOnce) { - dumpedOnce = true; + static std::atomic dumpedOnce{false}; + if (dumpPrefix && !dumpedOnce.exchange(true)) { fprintf(stderr, "NVRTC options (%zu):\n", nvrtcOpts.size()); for (auto& o : nvrtcOpts) fprintf(stderr, " %s\n", o.c_str()); } @@ -434,15 +446,15 @@ int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id static const bool ptxOnly = getenv("PRPLL_PTX_ONLY") != nullptr; prog->cubin = ptxOnly ? string{} : std::move(images.cubin); prog->compiled = true; - g_lastBuildLog.clear(); + prog->buildLog.clear(); } catch (const exception& e) { - g_lastBuildLog = e.what(); + prog->buildLog = e.what(); prog->compiled = false; fprintf(stderr, "NVRTC COMPILE FAILED: %s\n", e.what()); // Dump the full preprocessed source for debugging { char fname[64]; - static int failCount = 0; + static std::atomic failCount{0}; snprintf(fname, sizeof(fname), "prpll_fail_%d.cu", failCount++); FILE* f = fopen(fname, "w"); if (f) { @@ -575,13 +587,16 @@ int clBuildProgram(cl_program prog, unsigned nDevices, const cl_device_id* devic return CL_SUCCESS; } -int clGetProgramBuildInfo(cl_program /*prog*/, cl_device_id, cl_program_build_info info, +int clGetProgramBuildInfo(cl_program prog, cl_device_id, cl_program_build_info info, size_t size, void* value, size_t* sizeRet) { + if (!prog) return CL_INVALID_PROGRAM; if (info == CL_PROGRAM_BUILD_LOG) { - size_t const len = g_lastBuildLog.size() + 1; + // This program's own log — a process-wide "last log" would report the + // diagnostics of whichever compile finished last on another thread. + size_t const len = prog->buildLog.size() + 1; if (sizeRet) *sizeRet = len; if (value && size >= len) { - memcpy(value, g_lastBuildLog.c_str(), len); + memcpy(value, prog->buildLog.c_str(), len); } } return CL_SUCCESS; diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index 3a4dc3b7..0a5845dd 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -49,6 +49,7 @@ struct _cl_program { std::string ptx; // Compiled PTX (after NVRTC compilation); always the text — clCreateKernel reads .maxntid from it std::string cubin; // NVRTC's CUBIN for this device's sm, when it produced one; loaded ahead of the PTX JIT CUmodule module{}; // Loaded module (after cuModuleLoadData) + std::string buildLog; // This program's last compile/link diagnostics (clGetProgramBuildInfo) bool compiled{false}; bool moduleLoaded{false}; diff --git a/src/log.cpp b/src/log.cpp index a5639251..a5177de8 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -10,6 +10,8 @@ thread_local string context; thread_local vector contextParts; thread_local File logFile; +// A worker's log file adopted by a helper thread (LogLinkScope); borrowed, never owned. +thread_local File* linkedLogFile = nullptr; static File stdoutFile{stdout, "stdout"}; @@ -40,10 +42,16 @@ void log(const char *fmt, ...) { va_end(va); string_view const s{logBuf}; - if (logFile) { logFile.write(s); } + if (logFile) { logFile.write(s); } else if (linkedLogFile && *linkedLogFile) { linkedLogFile->write(s); } stdoutFile.write(s); } +LogLink logLink() { return {logFile ? &logFile : linkedLogFile, context}; } + +LogLinkScope::LogLinkScope(const LogLink& link) : previous{linkedLogFile}, context{link.context} { linkedLogFile = link.file; } + +LogLinkScope::~LogLinkScope() { linkedLogFile = previous; } + LogContext::LogContext(const string& s) : part{s} { contextParts.push_back(s); context += s; diff --git a/src/log.h b/src/log.h index a49f5808..f25dc5a9 100644 --- a/src/log.h +++ b/src/log.h @@ -22,3 +22,24 @@ struct LogContext { private: std::string part; }; + +// The log file and context are thread-local (one log per worker instance), +// so a helper thread starts with neither and its log() lines reach stdout +// alone, without the exponent prefix. A thread that works on a worker's +// behalf — KernelCompiler's parallel compile under CUDA — takes a LogLink +// from the thread that starts it and adopts it for the task's duration. +class File; +struct LogLink { + File* file; // the starting thread's log file, borrowed — it outlives the task + std::string context; +}; +LogLink logLink(); + +struct LogLinkScope { + explicit LogLinkScope(const LogLink& link); + ~LogLinkScope(); + +private: + File* previous; + LogContext context; +}; From aa06a97ba78c3b39d00bdf993a3fec9d3431ae46 Mon Sep 17 00:00:00 2001 From: Sergej Kunz Date: Tue, 15 Sep 2026 09:26:11 +0000 Subject: [PATCH 153/214] CUDA: programmatic dependent launch behind -use PDL=1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernels already carry the Hopper PDL pairs — dependentLaunch() / dependentLaunchWait() (griddepcontrol.launch_dependents / .wait) in carryFused, fftMiddleIn, fftMiddleOut, tailSquare, tailMul and fftW — but both sides of the feature were compile-time and unreachable: the kernel macros tested ENABLE_PDL, which no build defines, and the only launch path that set CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION was CudaKernelLauncher::launch(), which the shim's clEnqueueNDRangeKernel never calls (it uses cuLaunchKernel directly). So the tail-overlap on sm_90+ could not be tried without editing two files. Make it one runtime switch: `-use PDL=1`. The define reaches the kernels as -DPDL=1 (base.cl: PDL defaults to 0). On the launch side the shim decides per kernel, from the compiled code: clCreateKernel already reads each kernel's PTX entry for .maxntid, and now also notes whether that entry contains griddepcontrol.wait. Only such a kernel is launched through cuLaunchKernelEx with the programmatic-serialization attribute (CUDA 12+); its wait holds every read until the previous kernel has completed, so it may start early. A kernel that never waits keeps an ordinary launch even with the flag on — the attribute on its launch would let it start while a predecessor that triggered early (carryFused signals before its final writes) is still running, and nothing else orders the two; kernel order is not fixed outside the squaring loop (carry variants, transposes, checks, P-1), so a program-wide attribute is not safe. Deriving the decision from the PTX also covers a program loaded from the kernel cache, whose compile options the shim never sees. The three launch paths in clEnqueueNDRangeKernel share one helper. -h documents the switch; PDL joins the -use key list so the option is not flagged unrecognized. Default off: nothing changes without the flag. Measure on an sm_90+ card: -iters 20000 with and without -use PDL=1, every Gerbicz check passing. Co-authored-by: Cursor --- src/Args.cpp | 2 ++ src/Gpu.cpp | 3 ++- src/cl/base.cl | 13 ++++++++-- src/cuda/clwrap_cuda.cpp | 51 ++++++++++++++++++++++++++++++++++++---- src/cuda/cudawrap.cpp | 2 +- src/cuda/tinycuda.h | 1 + 6 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/Args.cpp b/src/Args.cpp index 179bbd83..486ebeaa 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -184,6 +184,8 @@ named "config.txt" in the prpll run directory. -use INPLACE=n : Perform tranforms in-place. Great if the reduced memory usage fits in the GPU's L2 cache. 0 = not in-place, 1 = nVidia friendly access pattern, 2 = AMD friendly access pattern. -use PAD= : insert pad bytes to possibly improve memory access patterns. Val is number bytes to pad. + -use PDL=1 : CUDA on Hopper (sm_90) and later: programmatic dependent launch — a kernel's tail overlaps the + next kernel's start (fftMiddleOut → carryFused → fftMiddleIn → tailSquare). Off by default; measure. -use MIDDLE_IN_LDS_TRANSPOSE=0|1 : Transpose values in local memory before writing to global memory -use MIDDLE_OUT_LDS_TRANSPOSE=0|1 : Transpose values in local memory before writing to global memory -use TABMUL_CHAIN=: Controls how trig values are obtained in WIDTH and HEIGHT when FFT-spec is 1. diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 1916e5f5..ba5e3f79 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -292,7 +292,8 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector= 900 && ENABLE_PDL +#if CUDA_BACKEND && HAS_PTX >= 900 && PDL __asm volatile("griddepcontrol.launch_dependents;"); // same as cudaTriggerProgrammaticLaunchCompletion(); #endif } void dependentLaunchWait() { -#if CUDA_BACKEND && HAS_PTX >= 900 && ENABLE_PDL +#if CUDA_BACKEND && HAS_PTX >= 900 && PDL __asm volatile("griddepcontrol.wait;"); // same as cudaGridDependencySynchronize(); #endif } diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 77aef8cb..d82c263e 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -613,8 +613,8 @@ if (getenv("TRY_LDS_CARVEOUT")) string const entryPattern = ".entry " + string(name) + "("; size_t const pos = ptx.find(entryPattern); if (pos != string::npos) { - // Found the kernel entry. Now find .maxntid before the next .entry or opening brace - size_t searchEnd = ptx.find(".entry ", pos + 1); + // Found the kernel entry. Its text runs to the next .entry or .func. + size_t searchEnd = min(ptx.find(".entry ", pos + 1), ptx.find(".func ", pos + 1)); if (searchEnd == string::npos) searchEnd = ptx.size(); string const maxntidPattern = ".maxntid "; size_t const mpos = ptx.find(maxntidPattern, pos); @@ -624,6 +624,15 @@ if (getenv("TRY_LDS_CARVEOUT")) k->reqWorkGroupSize = val; } } + // A kernel that waits for its predecessor (griddepcontrol.wait — + // compiled in by -use PDL=1 on sm_90+) is launched with programmatic + // stream serialization: it may begin while the predecessor's tail + // still runs, and its wait holds every read until the predecessor + // has completed. A kernel without the wait keeps an ordinary launch: + // it may follow one that triggered early, and nothing else would + // order the two. Read from the compiled code, this holds for a + // cached program as much as a fresh one. + k->pdl = ptx.find("griddepcontrol.wait", pos) < searchEnd; } } @@ -727,6 +736,38 @@ cl_command_queue clCreateCommandQueueWithProperties(cl_context ctx, cl_device_id // ---- Enqueue operations ---- +// One launch for the three paths below. For a kernel whose code waits on +// its predecessor (k->pdl, see clCreateKernel; CUDA 12+, where +// cuLaunchKernelEx exists), the launch carries +// CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION: this kernel may +// begin once the previous one in the stream signals +// griddepcontrol.launch_dependents (that one's tail overlapping this one's +// prologue), and this kernel's griddepcontrol.wait holds its reads until +// the previous one has completed. Every other kernel is launched as before. +static CUresult launchKernel(cl_kernel k, unsigned numBlocksX, unsigned numBlocksY, unsigned lsX, unsigned lsY, + CUstream stream, void** argPtrs) { +#if CUDA_VERSION >= 12000 + if (k->pdl) { + CUlaunchAttribute attr{}; + attr.id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION; + attr.value.programmaticStreamSerializationAllowed = 1; + CUlaunchConfig config{}; + config.gridDimX = numBlocksX; + config.gridDimY = numBlocksY; + config.gridDimZ = 1; + config.blockDimX = lsX; + config.blockDimY = lsY; + config.blockDimZ = 1; + config.sharedMemBytes = 0; + config.hStream = stream; + config.attrs = &attr; + config.numAttrs = 1; + return cuLaunchKernelEx(&config, k->func, argPtrs, nullptr); + } +#endif + return cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, stream, argPtrs, nullptr); +} + int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, const size_t* /*globalOffset*/, const size_t* globalSize, const size_t* localSize, unsigned /*nWaits*/, @@ -756,7 +797,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, ev->hasTimings = true; ev->commandType = CL_COMMAND_NDRANGE_KERNEL; cuEventRecord(ev->start, q->stream); - CUresult const r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); + CUresult const r = launchKernel(k, numBlocksX, numBlocksY, lsX, lsY, q->stream, argPtrs); cuEventRecord(ev->end, q->stream); *event = ev; return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; @@ -771,7 +812,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, if (!pStart) { cuEventCreate(&pStart, CU_EVENT_DEFAULT); cuEventCreate(&pEnd, CU_EVENT_DEFAULT); } cuEventRecord(pStart, q->stream); - CUresult const r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); + CUresult const r = launchKernel(k, numBlocksX, numBlocksY, lsX, lsY, q->stream, argPtrs); cuEventRecord(pEnd, q->stream); cuEventSynchronize(pEnd); float ms = 0; @@ -803,7 +844,7 @@ int clEnqueueNDRangeKernel(cl_command_queue q, cl_kernel k, unsigned workDim, return r == CUDA_SUCCESS ? CL_SUCCESS : CL_OUT_OF_RESOURCES; } - CUresult const r = cuLaunchKernel(k->func, numBlocksX, numBlocksY, 1, lsX, lsY, 1, 0, q->stream, argPtrs, nullptr); + CUresult const r = launchKernel(k, numBlocksX, numBlocksY, lsX, lsY, q->stream, argPtrs); if (r != CUDA_SUCCESS) { const char* errName = nullptr; cuGetErrorName(r, &errName); diff --git a/src/cuda/cudawrap.cpp b/src/cuda/cudawrap.cpp index f4b4cb6a..ad47691f 100644 --- a/src/cuda/cudawrap.cpp +++ b/src/cuda/cudawrap.cpp @@ -495,7 +495,7 @@ for (auto& o : options) opts.push_back(o.c_str()); // ---- Kernel launcher ---- void CudaKernelLauncher::launch(CUstream stream, u32 gridSize, void** args, u32 sharedMem) { -#if CUDA_VERSION >= 12000 && ENABLE_PDL +#if CUDA_VERSION >= 12000 && defined(ENABLE_PDL) && ENABLE_PDL // enable pdl in kernel launch attributes CUlaunchAttribute attrs[1]; attrs[0].id = CU_LAUNCH_ATTRIBUTE_PROGRAMMATIC_STREAM_SERIALIZATION; diff --git a/src/cuda/tinycuda.h b/src/cuda/tinycuda.h index c8ff49ef..e442efb7 100644 --- a/src/cuda/tinycuda.h +++ b/src/cuda/tinycuda.h @@ -60,6 +60,7 @@ struct _cl_kernel { CUfunction func{}; std::string name; CUmodule parentModule{}; // Keep reference so module isn't unloaded + bool pdl{false}; // Its PTX waits on the predecessor (griddepcontrol.wait): launch with programmatic stream serialization int numArgs{0}; int reqWorkGroupSize{0}; // From __launch_bounds__(N) in source, matches OpenCL reqd_work_group_size From 9860255737b960d7826a3566656e32300e125a5d Mon Sep 17 00:00:00 2001 From: george Date: Tue, 15 Sep 2026 22:32:25 +0000 Subject: [PATCH 154/214] Tune FFTs sorted by maximum exponent handled by the FFT. --- src/FFTConfig.h | 3 ++- src/tune.cpp | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/FFTConfig.h b/src/FFTConfig.h index 4c5d1769..fd8a471e 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -47,6 +47,7 @@ class FFTShape { [[nodiscard]] float minBpw() const { return fft_type != FFT32 ? 3.0f : 1.0f; } [[nodiscard]] float maxBpw() const { return *std::ranges::max_element(bpw); } + [[nodiscard]] u64 maxExp() const { return u64(maxBpw() * size()); } [[nodiscard]] std::string spec() const { return (fft_type ? to_string(fft_type) + ':' : "") + numberK(width) + ':' + numberK(middle) + ':' + numberK(height); } [[nodiscard]] float carry32BPW() const; @@ -93,7 +94,7 @@ struct FFTConfig { [[nodiscard]] std::string spec() const; [[nodiscard]] u32 size() const { return shape.size(); } - [[nodiscard]] u64 maxExp() const { return u64(maxBpw() * shape.size()); } + [[nodiscard]] u64 maxExp() const { return shape.maxExp(); } [[nodiscard]] float minBpw() const { return shape.minBpw(); } [[nodiscard]] float maxBpw() const; diff --git a/src/tune.cpp b/src/tune.cpp index 4558c72a..c23ff146 100644 --- a/src/tune.cpp +++ b/src/tune.cpp @@ -1200,6 +1200,9 @@ skip_1K_256 = false; vector results = TuneEntry::readTuneFile(*args); + // Time FFT shapes smallest-to-largest exponent handled + std::ranges::stable_sort(shapes, [](const FFTShape& a, const FFTShape& b) { return a.maxExp() < b.maxExp(); }); + // Loop through all possible FFT shapes for (const FFTShape& shape : shapes) { From cbd99e33ffe3ac7c5e5eb136e45f8a6ca6693b2a Mon Sep 17 00:00:00 2001 From: gwoltman Date: Tue, 15 Sep 2026 20:47:56 -0400 Subject: [PATCH 155/214] Remove PDL option from help until it is proven worthwhile. Removed deprecated PDL option for CUDA on Hopper. --- src/Args.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Args.cpp b/src/Args.cpp index 486ebeaa..179bbd83 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -184,8 +184,6 @@ named "config.txt" in the prpll run directory. -use INPLACE=n : Perform tranforms in-place. Great if the reduced memory usage fits in the GPU's L2 cache. 0 = not in-place, 1 = nVidia friendly access pattern, 2 = AMD friendly access pattern. -use PAD= : insert pad bytes to possibly improve memory access patterns. Val is number bytes to pad. - -use PDL=1 : CUDA on Hopper (sm_90) and later: programmatic dependent launch — a kernel's tail overlaps the - next kernel's start (fftMiddleOut → carryFused → fftMiddleIn → tailSquare). Off by default; measure. -use MIDDLE_IN_LDS_TRANSPOSE=0|1 : Transpose values in local memory before writing to global memory -use MIDDLE_OUT_LDS_TRANSPOSE=0|1 : Transpose values in local memory before writing to global memory -use TABMUL_CHAIN=: Controls how trig values are obtained in WIDTH and HEIGHT when FFT-spec is 1. From 65d5f08772c17002db388c16ca5d87e04098380c Mon Sep 17 00:00:00 2001 From: george Date: Wed, 16 Sep 2026 01:37:44 +0000 Subject: [PATCH 156/214] Added restrict at AI's suggestion - cannot hurt. --- src/cl/base.cl | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 625dedd2..e9cabb57 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -282,35 +282,34 @@ ulong2 OVERLOAD U2(unsigned long long a, unsigned long long b) { return (ulong2) #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void - // For reasons unknown, loading trig values into nVidia's constant cache has terrible performance #if AMDGPU -typedef constant const T2* Trig; -typedef constant const T* TrigSingle; -typedef constant const F2* TrigFP32; -typedef constant const F* TrigSingleFP32; -typedef constant const GF31* TrigGF31; -typedef constant const GF61* TrigGF61; +typedef constant const T2* restrict Trig; +typedef constant const T* restrict TrigSingle; +typedef constant const F2* restrict TrigFP32; +typedef constant const F* restrict TrigSingleFP32; +typedef constant const GF31* restrict TrigGF31; +typedef constant const GF61* restrict TrigGF61; #else -typedef global const T2* Trig; -typedef global const T* TrigSingle; -typedef global const F2* TrigFP32; -typedef global const F* TrigSingleFP32; -typedef global const GF31* TrigGF31; -typedef global const GF61* TrigGF61; +typedef global const T2* restrict Trig; +typedef global const T* restrict TrigSingle; +typedef global const F2* restrict TrigFP32; +typedef global const F* restrict TrigSingleFP32; +typedef global const GF31* restrict TrigGF31; +typedef global const GF61* restrict TrigGF61; #endif // However, caching weights in nVidia's constant cache improves performance. // Even better is to not pollute the constant cache with weights that are used only once. // This requires two typedefs depending on how we want to use the BigTab pointer. // For AMD we can declare BigTab as constant or global - it doesn't really matter. -typedef constant const double2* ConstBigTab; -typedef constant const float2* ConstBigTabFP32; +typedef constant const double2* restrict ConstBigTab; +typedef constant const float2* restrict ConstBigTabFP32; #if AMDGPU -typedef constant const double2* BigTab; -typedef constant const float2* BigTabFP32; +typedef constant const double2* restrict BigTab; +typedef constant const float2* restrict BigTabFP32; #else -typedef global const double2* BigTab; -typedef global const float2* BigTabFP32; +typedef global const double2* restrict BigTab; +typedef global const float2* restrict BigTabFP32; #endif // From 8893502576eb1de832922dcd7944be8b76a6c701 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 17 Sep 2026 01:13:06 +0000 Subject: [PATCH 157/214] Fixed compiler warning message about possibly uninitialized variable --- src/clwrap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clwrap.cpp b/src/clwrap.cpp index dcc08f44..8567c7b7 100644 --- a/src/clwrap.cpp +++ b/src/clwrap.cpp @@ -238,7 +238,7 @@ Program loadSource(cl_context context, const string &source) { } string getBuildLog(cl_program program, cl_device_id deviceId) { - size_t logSize; + size_t logSize = 0; const size_t maxLogSize = 64 * 1024; int err = clGetProgramBuildInfo(program, deviceId, CL_PROGRAM_BUILD_LOG, 0, nullptr, &logSize); CHECK2(err, "clGetProgramBuildInfo"); From ad9646126efd4d0d7e9b867b9f63ab98ea4298c1 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 20:22:44 -0600 Subject: [PATCH 158/214] Define barsync() only where the hardware supports it Since 436bdb4 base.cl has failed to preprocess on every GPU that is not NVIDIA: ./base.cl:830:4: error: - GPU not capable of barrier on a subset of threads barsync() was defined unconditionally with a bare #error in its non-PTX branch. HAS_PTX is 0 for any non-NVIDIA device, so the #error fired during preprocessing regardless of whether barsync() was called -- and nothing calls it. The effect was that no kernel compiled at all on AMD or Intel OpenCL, including plain FP64 configurations. Wrap the definition in #if HAS_PTX >= 200 instead. The PTX code is unchanged; on other GPUs barsync() is simply not defined, so a future call from such a build fails at the call site, which is the behaviour the existing comment asks for ("substituting a barrier on all threads is not permitted"). Verified on an Intel UHD iGPU: master 8893502 fails with the error above for -prp 5000011 -fft 256:2:256; with this change the kernels compile and the Gerbicz check passes at 2000 and 4000 iterations. Not tested on NVIDIA (no hardware available); the PTX branch is textually identical. Found and fixed by Claude (Anthropic's Claude Code), working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/cl/base.cl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index e9cabb57..e1082d0f 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -811,10 +811,12 @@ void OVERLOAD bar(const u32 WG) { #endif } -// Create a barrier across a subset of threads. Substituting a barrier on all threads is not permitted. +// Create a barrier across a subset of threads. Substituting a barrier on all threads is not permitted, so this is only +// defined where the hardware can do it (PTX bar.sync with a thread count, sm_20 or higher). On any other GPU a call to +// barsync() fails to compile at the call site instead of the whole of base.cl failing whether or not it is used. +#if HAS_PTX >= 200 void OVERLOAD barsync(const u32 numWG, const u32 WG) { if (WG <= WAVEFRONT) return; -#if HAS_PTX >= 200 // bar.sync with thread count requires sm_20 support or higher. #if USE_REGISTER_BARSYNC // bar.sync with a register is horribly slow on an RTX 5070Ti. __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); #else // WARNING, WARNING, WARNING: On TitanV using CUDA 12.9 tools and driver 580, this branch does not work in openCL (but works in CUDA build). @@ -825,10 +827,8 @@ void OVERLOAD barsync(const u32 numWG, const u32 WG) { } } #endif -#else - #error - GPU not capable of barrier on a subset of threads -#endif } +#endif // nVidia GPUs (Hopper architecture sm 9.0 and later) support Programatic Dependent Launch where the tail end execution of one kernel can overlap // with the beginning of the next kernel. This requires a special launch kernel command that is only available in CUDA 12.0 and later. From ceb996c09eb5eb8fe15fa220c13bb17276c6d6ca Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 20:47:01 -0600 Subject: [PATCH 159/214] Fix stray brace in the NCLOAD(TrigSingleFP32) PTX string base.cl's non-coherent single-float load was __asm("ld.global.nc.f32 %0}, [%1];" ...) The string goes to ptxas verbatim, so the first FP32 configuration that actually emits this overload (an XXLOAD on a TrigSingleFP32 with the LOADS digit selecting NCLOAD, on sm_50+) fails to build with a PTX parse error, on both the OpenCL-NVIDIA and the CUDA backends. It is latent today only because unused device functions are stripped before ptxas sees them. Match the f64 sibling on the line above. Found by an AI code audit (Claude, Anthropic's Claude Code); fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/cl/base.cl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index e9cabb57..bbc47c65 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -548,7 +548,7 @@ F2 OVERLOAD NCLOAD(TrigFP32 mem) { } F OVERLOAD NCLOAD(TrigSingleFP32 mem) { F retval; - __asm("ld.global.nc.f32 %0}, [%1];" : "=f"(retval) : "l"(mem)); + __asm("ld.global.nc.f32 %0, [%1];" : "=f"(retval) : "l"(mem)); return retval; } i64 OVERLOAD NCLOAD(i64 *mem) { From 86d3f40ad2b6ea343047f55ed25507bd47640f49 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 20:48:57 -0600 Subject: [PATCH 160/214] CUDA shim: guard g_allocatedBuffers with a mutex clwrap_cuda.cpp keeps a process-wide unordered_set of live cl_mem objects so that clSetKernelArg can tell an 8-byte buffer argument from an 8-byte scalar and substitute the CUdeviceptr. clCreateBuffer inserts, clReleaseMemObject erases, and clSetKernelArg calls contains() for every buffer argument of every launch. None of it was synchronized, while -workers N (1..4) runs N gpuWorker threads against the single CUDA context; the parallel-compile change did add a mutex for g_moduleRefCount two lines below, but not here. A worker allocating its ~40 buffers (or releasing them at a work-unit boundary, or TrigBufCache lazily allocating for a new FFT) while another worker is launching kernels is therefore a data race on the set: an insert that rehashes frees the bucket array under a concurrent contains() and the process segfaults, or contains() misses a live buffer and the host address of the _cl_mem is handed to the kernel as its device pointer, giving a sticky CUDA_ERROR_ILLEGAL_ADDRESS that kills every worker. Reading mem->ptr after the lookup was also a use-after-free window against a concurrent release. Take a mutex around all three accesses, and copy the device pointer while still holding it. Not compiled here: the CUDA shim needs the CUDA SDK headers, which this machine does not have. The change is four lock_guard scopes over an existing std::mutex idiom already used in this file. Found by an AI code audit (Claude, Anthropic's Claude Code); fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/cuda/clwrap_cuda.cpp | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 60ef58bb..585a49f5 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -27,7 +27,10 @@ using namespace std; // Track allocated cl_mem objects so clSetKernelArg can distinguish buffer args from scalars. // In OpenCL, buffer args are passed as &memobj where memobj is cl_mem (a pointer to _cl_mem). // We need to convert these to CUdeviceptr for CUDA kernel launch. +// The set is shared by every worker thread (-workers N runs N Gpu instances against this one +// process-wide context), so all access goes through g_allocatedBuffersMutex. static unordered_set g_allocatedBuffers; +static std::mutex g_allocatedBuffersMutex; // Global CUDA context — set once by clCreateContext, used to ensure current before CUDA calls static CUcontext g_cudaContext = nullptr; @@ -733,10 +736,22 @@ int clSetKernelArg(cl_kernel k, unsigned pos, size_t size, const void* value) { // We need to store the CUdeviceptr (GPU address) instead of the cl_mem (host pointer). if (size == sizeof(cl_mem) && value) { cl_mem mem = *(cl_mem*)value; - if (mem && g_allocatedBuffers.contains(mem)) { - CUdeviceptr devPtr = mem->ptr; - k->setArg(pos, sizeof(CUdeviceptr), &devPtr); - return CL_SUCCESS; + if (mem) { + // Look the buffer up and copy its device pointer under the lock: another worker may be + // creating or releasing buffers concurrently, and a release deletes the _cl_mem. + CUdeviceptr devPtr = 0; + bool isBuffer = false; + { + std::lock_guard lock(g_allocatedBuffersMutex); + if (g_allocatedBuffers.contains(mem)) { + isBuffer = true; + devPtr = mem->ptr; + } + } + if (isBuffer) { + k->setArg(pos, sizeof(CUdeviceptr), &devPtr); + return CL_SUCCESS; + } } // NULL cl_mem → pass a null device pointer if (!mem) { @@ -766,7 +781,10 @@ cl_mem clCreateBuffer(cl_context /*ctx*/, cl_mem_flags flags, size_t size, void if ((flags & CL_MEM_COPY_HOST_PTR) && hostPtr) { cuMemcpyHtoD(buf->ptr, hostPtr, size); } - g_allocatedBuffers.insert(buf); + { + std::lock_guard lock(g_allocatedBuffersMutex); + g_allocatedBuffers.insert(buf); + } if (err) *err = CL_SUCCESS; return buf; } @@ -774,7 +792,10 @@ cl_mem clCreateBuffer(cl_context /*ctx*/, cl_mem_flags flags, size_t size, void int clReleaseMemObject(cl_mem buf) { if (buf) { ensureContextCurrent(); - g_allocatedBuffers.erase(buf); + { + std::lock_guard lock(g_allocatedBuffersMutex); + g_allocatedBuffers.erase(buf); + } cuMemFree(buf->ptr); delete buf; } From b08eb4b22d76888c6e46c74040a60b41b3b83071 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 21:15:56 -0600 Subject: [PATCH 161/214] Make the 4-byte shufl_and_fft2 path actually perform the fft2 The SHUFL_BYTES == 4 path of shufl_and_fft2() for 64-bit element types (FP64 and GF61, and the hybrids built on them) did a plain shufl: it wrote each 32-bit piece of the T2 values to LDS and read them straight back from lds[i * WG + lowMe]. The 16- and 8-byte paths of the same function read a pair of values (val1 from i * WG/2, val2 from 4 * WG + i * WG/2) and add or subtract them, which is the radix-2 stage that the callers, fft8_16a/fft8_16b, assume has already happened. The "assert(SHUFL_BYTES >= 8)" meant to keep the path out of use is a printf under DEBUG and nothing otherwise, and the block was labelled NEEDS WORK, but it compiled and ran. Every WIDTH or HEIGHT of 1024 goes through shufl_and_fft2 (the host forces radix 8 there), so any 64-bit FFT at 1K with -use SHUFL_BYTES_W=4 or SHUFL_BYTES_H=4 computed a wrong transform. It fails immediately: prpll -prp 44564489 -fft 1K:5:256 -use SHUFL_BYTES_W=4 EE 0 on-load: 0000000000000000 vs. 0000000000000003 The fft2 has to add and subtract whole 64-bit values, so the fixed path first gathers all four 32-bit pieces of val1 and val2 for every u[i], from the same LDS locations the wider paths use, and combines them at the end. With the fix the same command passes and produces the same residues as the default SHUFL_BYTES (c14c8c46dd4da882 at 2000, 66a437358a5a6c83 at 4000) on an Intel UHD iGPU. The path is still unoptimized for bank conflicts, as before, and about 3x slower than the default on that GPU; it is opt-in. Found by an AI code audit (Claude, Anthropic's Claude Code) that simulated every shufl.cl LDS layout against the generic one; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/cl/shufl.cl | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index 0bb6446a..9d727ab7 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -427,7 +427,6 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe // Shufl two or more fft_WIDTHs or fft_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. An fft2 is also performed. void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { assert(RADIX == 8); - assert(SHUFL_BYTES >= 8); u32 mask = f - 1; assert((mask & (mask + 1)) == 0); @@ -516,31 +515,38 @@ void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! else if (SHUFL_BYTES == 4) { - -// NEEDS WORK!!! - // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. local int* lds = (local int*) lds2; if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); + // The fft2 has to add and subtract whole 64-bit values, so first gather all four 32-bit pieces of val1 and val2 + // (same LDS locations as the 16- and 8-byte paths above), then combine. u[] stays intact as the source of the + // four write passes. + int4 v1[RADIX], v2[RADIX]; bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + for (u32 i = 0; i < RADIX; ++i) { v1[i].x = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].x = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + for (u32 i = 0; i < RADIX; ++i) { v1[i].y = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].y = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + for (u32 i = 0; i < RADIX; ++i) { v1[i].z = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].z = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } bar(WG); - for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + for (u32 i = 0; i < RADIX; ++i) { v1[i].w = lds[i * (WG / 2) + lowMe % (WG / 2)]; v2[i].w = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; } + for (u32 i = 0; i < RADIX; ++i) { + T2_GF61 val1 = as_T2_GF61(v1[i]); + T2_GF61 val2 = as_T2_GF61(v2[i]); + if (lowMe < WG / 2) u[i] = addq(val1, val2); + else u[i] = subq(val1, val2); + } } } From 1ed23cb0712c59a0cb1ae4b8326cd13b7816391c Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 21:20:13 -0600 Subject: [PATCH 162/214] Fix the 4-byte F_Z31 shufl read for WG != 64, 512 The SHUFL_BYTES == 4, LDSPAD "first RADIX == 8" special case of shufl() for 32-bit element types (FP32 and GF31) writes lds[((lowMe/4) & 7) * (WG+1) + (lowMe/32) * 32 + (lowMe & 3) * 8 + i] and had two read formulas: one for WG == 64 and one for everything else. The "everything else" formula is the inverse of that write only when WG == 512. The remaining reachable case is WG == 128 -- every WIDTH or HEIGHT of 1024, which the host always runs with radix 8 -- and there it reads the wrong elements: simulating the layout against the generic shufl, 896 of the 1024 (i, lowMe) reads return the wrong value. Any 32-bit FFT at 1K with -use SHUFL_BYTES_W=4 (or SHUFL_BYTES_H=4) therefore computed a wrong transform and fails on load. Add the WG == 128 read (the inverse with the constants folded, in the style of the existing two), keep the WG == 512 one as it was, and make the final fallback the general inverse of the write so no other WG can silently take a formula that does not match. All four forms were checked by simulation for WG = 64, 128, 256, 512, 1024: every read returns the generic shufl's value. On an Intel UHD iGPU, FP32 53:1K:8:256 with -use SHUFL_BYTES_W=4 fails on load before this change and after it produces the same res64 at 2000 iterations as the default SHUFL_BYTES (18933754a884a5ea), i.e. identical arithmetic. (That FP32 configuration fails its Gerbicz check on both paths on this GPU, so the comparison shows the paths now agree, not that FP32 is sound at 5 bits/word there.) Found by an AI code audit (Claude, Anthropic's Claude Code) that simulated every shufl.cl LDS layout against the generic one; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/cl/shufl.cl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index 0bb6446a..84b22344 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -714,13 +714,20 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + // Read back in the generic shufl's output order. The write above stores (i', me') at + // ((me'/4)&7)*(WG+1) + (me'/32)*32 + (me'&3)*8 + i', and output (i, lowMe) needs i' = lowMe & 7, + // me' = i*WG/8 + lowMe/8; the per-WG forms below are that inverse with the constants folded. + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 128) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (4 * (i & 1) + lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 512) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u32 mep = i * (WG / 8) + lowMe / 8; u[i].x = lds[((mep / 4) & 7) * (WG + 1) + (mep / 32) * 32 + (mep & 3) * 8 + (lowMe & 7)]; } bar(WG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } bar(WG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 128) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (4 * (i & 1) + lowMe / 32) * (WG + 1) + (lowMe & 31)]; } + else if (WG == 512) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + else for (u32 i = 0; i < RADIX; ++i) { u32 mep = i * (WG / 8) + lowMe / 8; u[i].y = lds[((mep / 4) & 7) * (WG + 1) + (mep / 32) * 32 + (mep & 3) * 8 + (lowMe & 7)]; } return; } From d2b3a74e5286f990d5f760229edee9f3ce5c30ef Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 21:25:40 -0600 Subject: [PATCH 163/214] Require L2_STRIPING to divide the stripe count The striped launches split the WIDTH/16 fftMiddleIn stripes into groups of L2_STRIPING and process group i together with its Hermitian partner group at WIDTH - L2_STRIPING*16 - base_lo, so the number of groups has to be even -- and a multiple of four with MULTI_Q, which further splits the groups across two queues. The startup check only clamped L2_STRIPING to a maximum (WIDTH/64, or WIDTH/128 with MULTI_Q). A value that does not divide, e.g. WIDTH=256 with L2_STRIPING=3, gives num_stripe_groups = 5: the replay loop runs two blocks, columns 96..159 are never transformed in that iteration, the "last block" +1 adjustment lands on a block whose base_hi is not WIDTH/2 so one column is squared twice, and with MULTI_Q two blocks get the same base_lo on different queues. The Gerbicz check fails on every block, so the run is unusable rather than silently wrong. After the existing clamps, round L2_STRIPING down to the largest value for which (WIDTH/16) is a multiple of 2*L2_STRIPING (4*L2_STRIPING with MULTI_Q), and say so, matching how the maximum is already handled. WIDTH/16 is a power of two, so this means the nearest power of two below. Checked on an Intel UHD iGPU: -fft 256:2:256 -use INPLACE=1,L2_STRIPING=3 now logs "L2_STRIPING must divide WIDTH/32. Changing to L2_STRIPING=2" and passes its Gerbicz check with the same residue as an unstriped run. Found by an AI code audit (Claude, Anthropic's Claude Code) of the striping scheduler; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Gpu.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ba5e3f79..b4dbe4b4 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -350,6 +350,22 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector Date: Thu, 17 Sep 2026 03:26:26 +0000 Subject: [PATCH 164/214] Allows sharing LDS memory among partitioned workgroups. Is slower on GPUs I tried. Perhaps if WMUL=4, enough LDS memory could be freed up to allow storing trig values in LDS memory -- perhaps that could be faster. There are other possible ideas I might try at a later date. --- src/cl/base.cl | 8 + src/cl/carryfused.cl | 43 +++-- src/cl/expand.cl | 4 + src/cl/fftbase.cl | 165 ++++++++++++++--- src/cl/fftheight.cl | 1 + src/cl/ffthin.cl | 14 +- src/cl/fftp.cl | 36 +++- src/cl/fftw.cl | 12 +- src/cl/fftwidth.cl | 1 + src/cl/shufl.cl | 429 ++++++++++++++++++++++++------------------- src/cl/tailmul.cl | 16 +- src/cl/tailsquare.cl | 48 +++-- src/cl/tailutil.cl | 53 +++--- 13 files changed, 548 insertions(+), 282 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index e9cabb57..f055e9a8 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -156,6 +156,14 @@ G_H "group height" == SMALL_HEIGHT / NH #define LDSPAD_H 1 #endif +// By default, LDS access is not shared among workgroups. +#if !defined(LDSMUL_W) +#define LDSMUL_W 1 +#endif +#if !defined(LDSMUL_H) +#define LDSMUL_H 1 +#endif + #if !defined(TABMUL_CHAIN) #define TABMUL_CHAIN 0 #endif diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index c9bc907d..bd492843 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -38,7 +38,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) //if (WMUL == 1) return; #if WMUL > 1 - const u32 lds_i64s = LDS_BYTES / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL line + const u32 lds_i64s = LDS_SHUFL_BYTES(WMUL) / sizeof(i64); // Number of i64s in LDS used by shufl for each WMUL workgroup local i64 *lds = (local i64 *) lds2; // Handle nasty case where we are writing 8-byte quantities but SHUFL_BYTES_W is only 4 bytes @@ -67,9 +67,11 @@ void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) // Write the other half of the carries bar(); if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW/2; ++i) lds[lds_i64s + i * G_W] = carry[i + NW/2]; - // Read carries from our WMUL workgroup LDS area. Compatible with shufl, no trailing bar() needed. + // Read carries from our WMUL workgroup LDS area. Compatible with shufl when no trailing bar() needed. bar(); if (me >= G_W) for (i32 i = 0; i < NW/2; ++i) carry[i + NW/2] = lds[i * G_W]; + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(WMUL)) bar(); } } @@ -81,8 +83,10 @@ void OVERLOAD shufl_carries_up(local void *lds2, i64 *carry, u32 me, u32 lowMe) if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW; ++i) lds[lds_i64s + i * G_W] = carry[i]; // Full barrier needed as we just moved data from one WMUL workgroup LDS area to the another WMUL workgroup's LDS area bar(); - // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl and no trailing bar() is required. + // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl when no trailing bar() is required. if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(WMUL)) bar(); } #endif @@ -95,7 +99,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) //if (WMUL == 1) return; #if WMUL > 1 - const u32 lds_i32s = LDS_BYTES / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL line + const u32 lds_i32s = LDS_SHUFL_BYTES(WMUL) / sizeof(i32); // Number of i32s in LDS used by shufl for each WMUL workgroup local i32 *lds = (local i32 *) lds2; lds += (me / G_W) * lds_i32s + lowMe; // This WMUL workgroup's LDS area @@ -105,8 +109,10 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) if (me < (WMUL-1) * G_W) for (i32 i = 0; i < NW; ++i) lds[lds_i32s + i * G_W] = carry[i]; // Full barrier needed as we just moved data from one WMUL workgroup LDS area to the another WMUL workgroup's LDS area bar(); - // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl and no trailing bar() is required. + // Read carries from our WMUL workgroup's LDS area. This is compatible with shufl when no trailing bar() is required. if (me >= G_W) for (i32 i = 0; i < NW; ++i) carry[i] = lds[i * G_W]; + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(WMUL)) bar(); #endif } @@ -118,7 +124,8 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - local T2 lds[WMUL * LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(WMUL) / sizeof(T2)]; + LDSinit(lds, WMUL); T2 u[NW]; @@ -343,7 +350,8 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - local F2 lds[WMUL * LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(WMUL) / sizeof(F2)]; + LDSinit(lds, WMUL); F2 u[NW]; @@ -570,7 +578,8 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { - local GF31 lds[WMUL * LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(WMUL) / sizeof(GF31)]; + LDSinit(lds, WMUL); GF31 u[NW]; @@ -805,7 +814,8 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { - local GF61 lds[WMUL * LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(WMUL) / sizeof(GF61)]; + LDSinit(lds, WMUL); GF61 u[NW]; @@ -1047,8 +1057,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { - local T2 lds[WMUL * LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(WMUL) / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; + LDSinit(lds, WMUL); T2 u[NW]; GF31 u31[NW]; @@ -1315,8 +1326,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - local F2 ldsF2[WMUL * LDS_BYTES / sizeof(F2)]; + local F2 ldsF2[LDS_BYTES(WMUL) / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; + LDSinit(ldsF2, WMUL); F2 uF2[NW]; GF31 u31[NW]; @@ -1592,8 +1604,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; + LDSinit(lds61, WMUL); F2 uF2[NW]; GF61 u61[NW]; @@ -1870,8 +1883,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { - local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, WMUL); GF31 u31[NW]; GF61 u61[NW]; @@ -2146,9 +2160,10 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { - local GF61 lds61[WMUL * LDS_BYTES / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, WMUL); F2 uF2[NW]; GF31 u31[NW]; diff --git a/src/cl/expand.cl b/src/cl/expand.cl index b60c3161..eb097eaa 100644 --- a/src/cl/expand.cl +++ b/src/cl/expand.cl @@ -8,6 +8,7 @@ #if FFT_FP64 #define T_Z61 T #define T2_GF61 T2 +#define T_F_Z31_Z61 T #define T2_F2_GF31_GF61 T2 #define as_T2_GF61 as_double2 #include INCLUDE_FILE @@ -20,6 +21,7 @@ #if NTT_GF61 #define T_Z61 Z61 #define T2_GF61 GF61 +#define T_F_Z31_Z61 Z61 #define T2_F2_GF31_GF61 GF61 #define as_T2_GF61 as_ulong2 #include INCLUDE_FILE @@ -32,6 +34,7 @@ #if FFT_FP32 #define F_Z31 F #define F2_GF31 F2 +#define T_F_Z31_Z61 F #define T2_F2_GF31_GF61 F2 #include INCLUDE_FILE #undef F_Z31 @@ -42,6 +45,7 @@ #if NTT_GF31 #define F_Z31 Z31 #define F2_GF31 GF31 +#define T_F_Z31_Z61 Z31 #define T2_F2_GF31_GF61 GF31 #include INCLUDE_FILE #undef F_Z31 diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index f0b6d37c..2c55164a 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -3,23 +3,146 @@ #include "fft4.cl" #include "fft8.cl" -// Calculate the LDS bytes used by shufl -#if LDSPAD && SHUFL_BYTES == 16 && RADIX == 8 -#define LDS_BYTES ((WG * RADIX + 7) * SHUFL_BYTES) -#elif LDSPAD && SHUFL_BYTES == 16 && RADIX == 4 -#define LDS_BYTES ((WG * RADIX + 12) * SHUFL_BYTES) -#elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 8 -#define LDS_BYTES ((WG * RADIX + 56) * SHUFL_BYTES) -#elif LDSPAD && SHUFL_BYTES == 8 && RADIX == 4 -#define LDS_BYTES ((WG * RADIX + 12) * SHUFL_BYTES) -#elif LDSPAD && SHUFL_BYTES == 4 && RADIX == 8 -#define LDS_BYTES ((WG * RADIX + 56) * SHUFL_BYTES) -#elif LDSPAD && SHUFL_BYTES == 4 && RADIX == 4 -#define LDS_BYTES ((WG * RADIX + 12) * SHUFL_BYTES) +// NOTE: tailSquare, with its ability to optionally define tailSquareZero, does not allow us to know numWG at #include time. +// Thus, we must define macros that take numWG as in input argument. This could be rectified by making tailSquareZero obey the TAIL_KERNELS setting. + +// This section is not necessary. On TitanV, CUDA 12.9, I see a 0.5% slowdown when not LDS sharing but compiled with the LDS sharing code. + +#if LDSMUL == 1 // Not sharing LDS memory, use simplified code. + +#define SHARING_LDS(numWG) 0 +#define SBMUL(numWG) 1 +#define LDSPAD_COUNT(numWG) (!LDSPAD ? 0 : RADIX == 4 ? 12 : SHUFL_BYTES >= 16 ? 7 : 56) +#define LDS_SHUFL_BYTES(numWG) ((WG * RADIX + LDSPAD_COUNT(numWG)) * SHUFL_BYTES) +#define LDS_BYTES(numWG) (numWG * LDS_SHUFL_BYTES(numWG)) + +void OVERLOAD LDSinit(void local *lds, const u32 numWG) { +} + +local void * OVERLOAD LDSptr(local void *lds, const u32 numWG) { + return (local char *)lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG); +} + +local void * OVERLOAD LDSsharing_ptr(local void *lds, const u32 numWG) { + return LDSptr(lds, numWG); +} + +void OVERLOAD LDSbar(const u32 numWG) { + bar(WG); +} + +void OVERLOAD LDStx_start(local void *lds, const u32 numWG) { + LDSbar(numWG); +} + +void OVERLOAD LDStx_end(local void *lds, const u32 numWG) { +} + + +// This section handles both cases of sharing and not sharing LDS memory + #else -#define LDS_BYTES (WG * RADIX * SHUFL_BYTES) + +// LDS access is shared if the kernel processes multiple independent workgroups, the user settable LDSMUL is more than one, and the GPU allows barriers on a subset of threads +#define SHARING_LDS(numWG) (numWG > 1 && LDSMUL > 1 && (NVIDIAGPU || WG <= WAVEFRONT)) +// If sharing LDS access, LDSMUL sets a limit on how many workgroups share the same LDS memory. Sharing LDS allow shufl to use a multiple of SHUFL_BYTES. +#define SBMUL(numWG) (!SHARING_LDS(numWG) ? 1 : numWG >= LDSMUL ? LDSMUL : numWG) +// Calculate the LDS padding used by shufl +#define LDSPAD_COUNT(numWG) (!LDSPAD ? 0 : RADIX == 4 ? 12 : SBMUL(numWG) * SHUFL_BYTES >= 16 ? 7 : 56) +// LDS_SHUFL_BYTES is the number of LDS bytes *allocated* for each workgroup (SBMUL > 1 means the workgroup can *access* some multiple of LDS_SHUFL_BYTES) +#define LDS_SHUFL_BYTES(numWG) ((WG * RADIX + LDSPAD_COUNT(numWG)) * SHUFL_BYTES) +// The total number of LDS_BYTES allocated by a kernel includes space for 4 semaphores to control workgroup access +#define LDS_BYTES(numWG) (numWG * LDS_SHUFL_BYTES(numWG) + (SHARING_LDS(numWG) ? 16 : 0)) + +//// BUG BUG BUG +///variant 2 partitioned_LDS is a nightmare -- require variant 2 to be LDSMUL=1 until we can figure it out? + +// Initialize access to LDS memory. It may be advantageous to have independent workgroups share access to LDS memory via a lock controlling a critical section. +// This may let a kernel use less LDS memory, or have each workgroup use more LDS memory to perform fewer passes of writing and reading LDS memory. +void OVERLOAD LDSinit(void local *lds, const u32 numWG) { + // Init semaphores to unlocked state + if (SHARING_LDS(numWG)) { + assert(numWG / SBMUL(numWG) <= 4); // LDS_BYTES is hardwired to allocate 4 semaphores + if (get_local_id(0) == 0) { + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + numWG * LDS_SHUFL_BYTES(numWG)); + for (u32 i = 0; i < numWG / SBMUL(numWG); i++) semaphores[i] = 0; + } + bar(); + } +} + +// Return a pointer to the LDS memory allocated for this workgroup. If SBMUL is greater than 1, workgroup may use additional memory by sharing +// with other workgroups and using locks to control access. +local void * OVERLOAD LDSptr(local void *lds, const u32 numWG) { + return (local char *)lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG); +} + +// Return a pointer to the LDS memory this workgroup is allowed to access when sharing with other workgroups. +local void * OVERLOAD LDSsharing_ptr(local void *lds, const u32 numWG) { + if (!SHARING_LDS(numWG)) return LDSptr(lds, numWG); + return (local char *)lds + ((u32)get_local_id(0) / WG / SBMUL(numWG)) * SBMUL(numWG) * LDS_SHUFL_BYTES(numWG); +} + +// Wait for all of a workgroup's threads to arrive. +// NOTE: A "workgroup" is an independent group of threads doing FFT work (see WMUL in carryFused or TAIL_KERNELS=2). +void OVERLOAD LDSbar(const u32 numWG) { + + if (WG <= WAVEFRONT) return; + + // If were not using semaphores to share LDS access, perform a standard bar. The standard bar is free to implement a full bar across + // all threads if that is more efficient than a bar across a subset of threads. + if (!SHARING_LDS(numWG)) { + bar(WG); + return; + } + + // Barrier on a subset of threads. + barsync(numWG, WG); +} + +// Start a new LDS access transaction. This is required for sharing LDS memory with other workgroups. +// Historically, each workgroup had its own LDS area, and shufl routines performed a bar(WG) at the start of accessing LDS but not at the end. +// After calling shufl, a bar(WG) was required before next LDS memory usage. All routines that use LDS memory OBEYED THIS PROTOCOL +// of bar(WG) before LDS use (full bar() if writing outside the workgroup's LDS area) and no bar(WG) after last use (full bar() if reading from +// outside the workgroup's LDS area). If we're not sharing LDS access among multiple workgroups, maintain this historical implementation. +// If sharing LDS we have NEW REQUIREMENTS. LDStx_end performs an LDSbar because workgroups write to more than just their own LDS area. The LDSbar +// ensures the reads have completed before any future writes. When accessing LDS memory without LDStx calls (see shufl_carries_up in carryFused and +// reverseLines in tailutil) they too must perform a bar() after the last read from LDS. +// NOTE: Pass in the original LDS pointer, not the pointer returned by LDSptr or LDSsharing_ptr. +void OVERLOAD LDStx_start(local void *lds, const u32 numWG) { + // If each workgroup has its own LDS area, then no locks are needed to access shared memory. Use the historical model of requiring a barrier before LDS access. + if (!SHARING_LDS(numWG)) { + LDSbar(numWG); + return; + } + // Have first thread in a workgroup lock the semaphore controlling access to LDS memory + if (get_local_id(0) % WG == 0) { + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + numWG * LDS_SHUFL_BYTES(numWG)); + + // Lock semaphore (set to one) to gain access to critical section + while (atomic_cmpxchg(&semaphores[get_local_id(0) / WG / SBMUL(numWG)], 0, 1) == 1); + } + LDSbar(numWG); +} + +// End an LDS access transaction +// NOTE: Pass in the original LDS pointer, not the pointer returned by LDSptr or LDSsharing_ptr. +void OVERLOAD LDStx_end(local void *lds, const u32 numWG) { + // Historically, no trailing LDSbar is required when not sharing LDS memory. + if (!SHARING_LDS(numWG)) return; + + // Since we are sharing LDS areas among multiple workgroups, we must wait for all of a workgroup's threads to finish their LDS access. + LDSbar(numWG); + // Unlock the semaphore + if (get_local_id(0) % WG == 0) { + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + numWG * LDS_SHUFL_BYTES(numWG)); + semaphores[get_local_id(0) / WG / SBMUL(numWG)] = 0; + } +} + #endif + #define INCLUDE_FILE "shufl.cl" #include "expand.cl" @@ -263,7 +386,7 @@ void OVERLOAD tabMul8_4b(Trig trig, T2 *u, u32 f, u32 me) { u32 p = me & ~(f - 1); T2 w = TFLOAD(&trig[p]); -// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data +// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data // T2 w2 = csqTrigFancy(w); // u[2] = cmulFancy(u[2], w2); // T2 w3 = ccubeTrigFancy(w2, w); @@ -275,7 +398,7 @@ void OVERLOAD tabMul8_4b(Trig trig, T2 *u, u32 f, u32 me) { // base = cmulFancy(base, w); // } - u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data + u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data T2 w2 = csqTrig(w); u[2] = cmul(u[2], w2); T2 w3 = ccubeTrig(w2, w); @@ -534,8 +657,7 @@ void finish_tabMul8_fft8(Trig trig, T *preloads, T2 *u, u32 f, u32 numWG, u32 me void OVERLOAD fft_common(local T2 *lds, T2 *u, Trig trig, T2 w, u32 numWG, u32 lowMe, int callnum) { // This line mimics shufl -- partition lds for variant 2 - local T2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2); + local T2* partitioned_lds = LDSptr(lds, numWG); // Variant 0 uses broadcast instructions. Only available on AMD GPUs. @@ -975,7 +1097,7 @@ void OVERLOAD tabMul8_4b(TrigFP32 trig, F2 *u, u32 f, u32 me) { u32 p = me & ~(f - 1); F2 w = TFLOAD(&trig[p]); -// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data +// u[1] = cmulFancy(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data // T2 w2 = csqTrigFancy(w); // u[2] = cmulFancy(u[2], w2); // T2 w3 = ccubeTrigFancy(w2, w); @@ -987,7 +1109,7 @@ void OVERLOAD tabMul8_4b(TrigFP32 trig, F2 *u, u32 f, u32 me) { // base = cmulFancy(base, w); // } - u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data + u[1] = cmul(u[1], w); // GW: - this should use Fancy, but tabmul8_4a does not and it could for half of the data F2 w2 = csqTrig(w); u[2] = cmul(u[2], w2); F2 w3 = ccubeTrig(w2, w); @@ -1256,8 +1378,7 @@ void finish_tabMul8_fft8(TrigFP32 trig, F *preloads, F2 *u, u32 f, u32 numWG, u3 void OVERLOAD fft_common(local F2 *lds, F2 *u, TrigFP32 trig, u32 numWG, u32 lowMe, int callnum) { // This line mimics shufl -- partition lds - local F2* partitioned_lds = lds; - if (numWG > 1) partitioned_lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2); + local F2* partitioned_lds = LDSptr(lds, numWG); // Variant 2 code for SIZE=256, RADIX=4 #if ENABLE_FP32_VARIANT_2 && WG == 64 && RADIX == 4 && VARIANT == 2 diff --git a/src/cl/fftheight.cl b/src/cl/fftheight.cl index 7d835b41..a19d7b2c 100644 --- a/src/cl/fftheight.cl +++ b/src/cl/fftheight.cl @@ -7,6 +7,7 @@ #define LDSPAD LDSPAD_H #define LDSSWIZ LDSSWIZ_H #define SHUFL_BYTES SHUFL_BYTES_H +#define LDSMUL LDSMUL_H #define UNROLL UNROLL_H #define SAVE_ONE_MUL 1 // Radeon VII weirdness where saving one mul was slower (needs retesting!) #define DOING_HEIGHT 1 // Flags to work around any optimizer weirdness where common code performs better in fft_WIDTH and worse in fft_HEIGHT or vice versa diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index e95eb88d..2c104aa9 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -32,7 +32,9 @@ u32 get_line_number(u32 base_lo) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local T2 lds[LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; T2 u[NH]; @@ -65,7 +67,9 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local F2 lds[LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; @@ -102,7 +106,8 @@ KERNEL(G_H) fftHin(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local GF31 lds[LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -130,7 +135,8 @@ KERNEL(G_H) fftHinGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { // Do an FFT Height after an fftMiddleIn (which may not have fully transposed data, leading to non-sequential input) KERNEL(G_H) fftHinGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local GF61 lds[LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); diff --git a/src/cl/fftp.cl b/src/cl/fftp.cl index 620b0734..29c90854 100644 --- a/src/cl/fftp.cl +++ b/src/cl/fftp.cl @@ -11,7 +11,9 @@ // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + T2 u[NW]; u32 g = get_group_id(0); @@ -42,7 +44,9 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 lds[LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + F2 u[NW]; u32 g = get_group_id(0); @@ -83,7 +87,9 @@ KERNEL(G_W) fftP(P(F2) out, CP(Word2) in, TrigFP32 smallTrig, BigTabFP32 THREAD_ // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { - local GF31 lds[LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + GF31 u[NW]; u32 g = get_group_id(0); @@ -138,7 +144,9 @@ KERNEL(G_W) fftP(P(GF31) out, CP(Word2) in, TrigGF31 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { - local GF61 lds[LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + GF61 u[NW]; u32 g = get_group_id(0); @@ -195,8 +203,10 @@ KERNEL(G_W) fftP(P(GF61) out, CP(Word2) in, TrigGF61 smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) { - local T2 lds[LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; + LDSinit(lds, 1); + T2 u[NW]; GF31 u31[NW]; @@ -263,8 +273,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTab THREAD_WEIGHTS) // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local F2 ldsF2[LDS_BYTES / sizeof(F2)]; + local F2 ldsF2[LDS_BYTES(1) / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; + LDSinit(ldsF2, 1); + F2 uF2[NW]; GF31 u31[NW]; @@ -337,8 +349,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[LDS_BYTES / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES(1) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; + LDSinit(lds61, 1); + F2 uF2[NW]; GF61 u61[NW]; @@ -410,8 +424,10 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIG // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { - local GF61 lds61[LDS_BYTES / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES(1) / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, 1); + GF31 u31[NW]; GF61 u61[NW]; @@ -493,9 +509,11 @@ KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig) { // fftPremul: weight words with IBDWT weights followed by FFT-width. KERNEL(G_W) fftP(P(T2) out, CP(Word2) in, Trig smallTrig, BigTabFP32 THREAD_WEIGHTS) { - local GF61 lds61[LDS_BYTES / sizeof(GF61)]; + local GF61 lds61[LDS_BYTES(1) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; local GF31 *lds31 = (local GF31 *) lds61; + LDSinit(lds61, 1); + F2 uF2[NW]; GF31 u31[NW]; GF61 u61[NW]; diff --git a/src/cl/fftw.cl b/src/cl/fftw.cl index 14b4f5fd..43926016 100644 --- a/src/cl/fftw.cl +++ b/src/cl/fftw.cl @@ -10,7 +10,8 @@ // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); T2 u[NW]; u32 g = get_group_id(0); @@ -35,7 +36,8 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { // Do the ending fft_WIDTH after an fftMiddleOut. This is the same as the first half of carryFused. KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); CP(F2) inF2 = (CP(F2)) in; P(F2) outF2 = (P(F2)) out; @@ -63,7 +65,8 @@ KERNEL(G_W) fftW(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF31 KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); P(GF31) out31 = (P(GF31)) (out + DISTGF31); @@ -91,7 +94,8 @@ KERNEL(G_W) fftWGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if NTT_GF61 KERNEL(G_W) fftWGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); P(GF61) out61 = (P(GF61)) (out + DISTGF61); diff --git a/src/cl/fftwidth.cl b/src/cl/fftwidth.cl index f1c65c0e..5dff7e6c 100644 --- a/src/cl/fftwidth.cl +++ b/src/cl/fftwidth.cl @@ -7,6 +7,7 @@ #define LDSPAD LDSPAD_W #define LDSSWIZ LDSSWIZ_W #define SHUFL_BYTES SHUFL_BYTES_W +#define LDSMUL LDSMUL_W #define UNROLL UNROLL_W #define SAVE_ONE_MUL 0 // Radeon VII weirdness where saving one mul in width variant 2 was slower #define DOING_HEIGHT 0 // Flags to work around any optimizer weirdness where common code performs better in fft_WIDTH and worse in fft_HEIGHT or vice versa diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index 0bb6446a..a7a31f07 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -1,5 +1,23 @@ // Copyright (C) Mihai Preda + +// Strongly typed versions of LDSptr and LDSsharing_ptr. On TitanV, CUDA 12.9, this is 1% faster. +local T_F_Z31_Z61 * OVERLOAD LDSptr(local T_F_Z31_Z61 *lds, const u32 numWG) { + return lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG) / sizeof(T_F_Z31_Z61); +} +local T2_F2_GF31_GF61 * OVERLOAD LDSptr(local T2_F2_GF31_GF61 *lds, const u32 numWG) { + return lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG) / sizeof(T2_F2_GF31_GF61); +} +local T_F_Z31_Z61 * OVERLOAD LDSsharing_ptr(local T_F_Z31_Z61 *lds, const u32 numWG) { + if (!SHARING_LDS(numWG)) return LDSptr(lds, numWG); + return lds + ((u32)get_local_id(0) / WG / SBMUL(numWG)) * SBMUL(numWG) * LDS_SHUFL_BYTES(numWG) / sizeof(T_F_Z31_Z61); +} +local T2_F2_GF31_GF61 * OVERLOAD LDSsharing_ptr(local T2_F2_GF31_GF61 *lds, const u32 numWG) { + if (!SHARING_LDS(numWG)) return LDSptr(lds, numWG); + return lds + ((u32)get_local_id(0) / WG / SBMUL(numWG)) * SBMUL(numWG) * LDS_SHUFL_BYTES(numWG) / sizeof(T2_F2_GF31_GF61); +} + + #ifdef T2_GF61 // Shufl two or more fft_WIDTHs or FFT_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. @@ -7,13 +25,9 @@ // even when operating on differernt sized data elements as can happen in an M31+M61 NTT. // WG = workgroup size of a single fft_WIDTH or fft_HEIGHT // n = sizeof array u (nW or nH). n * WG = WIDTH or HEIGHT +// r usually equals RADIX if a full fft_RADIX step was just performed. On occasion u[8] values may do less than an fft8 step. // numWG = number of fft_WIDTHs or fft_HEIGHTs being processed simultaneously // lowMe = me % WG -// r usually equals RADIX if a full fft_RADIX step was just performed. On occasion u[8] values may do less than an fft8 step. -// NOTE: shufl routines perform a bar(WG) at the start but not at the end. After calling shufl, a bar(WG) is required -// before next LDS memory usage. All routines that use LDS memory MUST OBEY THIS PROTOCOL of bar() before LDS use and -// only bar(WG) required before next use. ALSO NOTE: the first shufl call does not need to do bar(WG). A relatively -// minor optimization would be to special case the first shufl call. void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u32 lowMe) { u32 mask = f - 1; @@ -21,9 +35,8 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. // We're writing 16 bytes at a time, which means groups of 8 must have unique LDS banks. - if (SHUFL_BYTES == 16) { - local T2_GF61* lds = lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); + if (SBMUL(numWG) * SHUFL_BYTES >= 16) { + local T2_GF61* lds = LDSsharing_ptr(lds2, numWG); #if LDSPAD // Special case first RADIX == 8 to eliminate LDS bank conflicts. @@ -33,11 +46,12 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 // Pad 1 value every row to eliminate bank conflicts. if (0 && f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe & 7) * (WG + 1) + (lowMe / 8) * 8 + i] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 1) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); return; } @@ -46,21 +60,11 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS that uses a little padding. Pad one value after every row to eliminate bank conflicts. // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 if (f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 1) + lowMe] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 8) + (lowMe / 8) + (lowMe & 7) * (WG + 1)]; } - return; - } - - // Special case second RADIX == 8 to eliminate LDS bank conflicts. - // Input values are the output from previous shufl. For example, WIDTH=512: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 - // No padding of LDS blocks is needed to eliminate bank conflicts! Groups of 8 threads are already in separate LDS banks. - // We could however save a bar() by writing to same locations that the previous shufl wrote to. - if (0 && f == 8 && r == 8 && RADIX == 8) { - // for (u32 i = 0; i < RADIX; ++i) { lds[something] = u[i]; } - bar(WG); - //for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[something]; } + LDStx_end(lds2, numWG); return; } @@ -71,10 +75,11 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 // Pad 1 value every row to eliminate bank conflicts. if (f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 2) & 3) * (WG + 1) + (lowMe / 8) * 8 + (lowMe & 1) * 4 + i] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 4 + (lowMe / 32) * 8 + ((lowMe / 8) & 3) * (WG + 1) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); return; } @@ -85,11 +90,12 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 16... 1.. output[64..127] = +4 // Pad 4 values after every row to eliminate bank conflicts. if (f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); return; } #endif @@ -100,10 +106,11 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 1, 65... lds[64..127] = +8 // Swizzle LDS blocks to eliminate bank conflicts. Swizzle on the first 8 threads written to LDS (multiples of 1) and the first 8 threads read from LDS (multiples of 64). if (f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 7)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 7)]; } + LDStx_end(lds2, numWG); return; } @@ -114,9 +121,11 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. // We can however save a bar() by writing to same locations that previous shufl wrote to. if (f == 8 && r == 8 && RADIX == 8) { + LDStx_start(lds2, numWG); //GRRR.... LDStx_start will do the bar we are trying to save for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } + LDStx_end(lds2, numWG); return; } @@ -126,10 +135,11 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 8 threads written to LDS (4 multiples of 1 and 2 multiples of 4) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 1). if (f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 7)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 7)]; } + LDStx_end(lds2, numWG); return; } @@ -139,26 +149,28 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 8 threads written to LDS (4 multiples of 64 and 2 multiples of 1) and the first 8 threads read from LDS (4 multiples of 64 and 2 multiples of 4). if (f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 4)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 4)]; } + LDStx_end(lds2, numWG); return; } #endif // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; } // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. // We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - else if (SHUFL_BYTES == 8) { - local T_Z61* lds = ((local T_Z61*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); + else if (SBMUL(numWG) * SHUFL_BYTES == 8) { + local T_Z61* lds = LDSsharing_ptr((local T_Z61 *)lds2, numWG); #if LDSPAD // Special case first RADIX == 8 code to eliminate LDS bank conflicts. @@ -166,14 +178,15 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS that uses a little padding. Pad two values after every row to eliminate bank conflicts. // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 448, 1, 65... u[1] = +8 if (f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + LDStx_end(lds2, numWG); return; } @@ -182,16 +195,17 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS with 8 pads after each row. // Read from LDS in output order. In the example: u[0] = 0, 64, ... 448, 8, 72... u[1] = +1 if (f == 8 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 8) * (WG + 8) + (lowMe & 7)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 8) * (WG + 8) + (lowMe & 7)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); return; } @@ -200,14 +214,15 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS that uses a little padding. Pad four values after every other row to eliminate bank conflicts. // Read from LDS in the desired output order. In the example: u[0] = 0, 64, 128, 192, 1, 65... u[1] = +8 if (f == 1 && r == 4 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / 2 * (2 * WG + 4) + (i % 2) * WG + lowMe] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (2 * WG + 4)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / 2 * (2 * WG + 4) + (i % 2) * WG + lowMe] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (2 * WG + 4)]; } + LDStx_end(lds2, numWG); return; } @@ -216,16 +231,17 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS with 4 pads after each row. // Read from LDS in output order. In the example: u[0] = 0, 64, 128, 192, 8, 72... u[1] = +1 if (f == 4 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 32) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 32) * 4 + (lowMe / 4) * (WG + 4) + (lowMe & 3)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 32) * 4 + (lowMe / 32) * 4 + ((lowMe / 4) & 7) * (WG + 4) + (lowMe & 3)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 32) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 32) * 4 + (lowMe / 4) * (WG + 4) + (lowMe & 3)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 32) * 4 + (lowMe / 32) * 4 + ((lowMe / 4) & 7) * (WG + 4) + (lowMe & 3)]; } + LDStx_end(lds2, numWG); return; } @@ -236,16 +252,17 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 // Pad one value after every row to eliminate bank conflicts. if (1 && f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); return; } @@ -254,14 +271,15 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS using a little padding. Pad four values after every row to eliminate bank conflicts. // Read from LDS in the desired output order. In the example: u[0] = 0, 64, ... 192, 1, 65... u[1] = +16 if (0 && f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (WG + 4)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG / 4 + (lowMe / 4) + (lowMe & 3) * (WG + 4)]; } + LDStx_end(lds2, numWG); return; } @@ -272,16 +290,17 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: u[0] = 0...192, 16... 32.. 48.. 1... u[1] = +4 // Pad 4 values after every row to eliminate bank conflicts. if (0 && f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); return; } @@ -290,14 +309,15 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS with 4 pads after each row. // Read from LDS in output order. In the example: u[0] = 0...192, 16... 32.. 48.. 1... u[1] = +4 if (1 && f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 16) * 4 + (lowMe / 16) * 4 + ((lowMe / 4) & 3) * (WG + 4) + (lowMe & 3)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 4) + lowMe] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 16) * 4 + (lowMe / 16) * 4 + ((lowMe / 4) & 3) * (WG + 4) + (lowMe & 3)]; } + LDStx_end(lds2, numWG); return; } #endif @@ -309,16 +329,17 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 1). if (f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + LDStx_end(lds2, numWG); return; } @@ -328,16 +349,17 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and 2 multiples of 8). if (f == 8 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + LDStx_end(lds2, numWG); return; } @@ -347,14 +369,15 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). if (f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + LDStx_end(lds2, numWG); return; } @@ -364,54 +387,58 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). if (f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + LDStx_end(lds2, numWG); return; } #endif // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; } // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! - else if (SHUFL_BYTES == 4) { + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. - local int* lds = (local int*) lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); + local int* lds = (local int*)LDSsharing_ptr(lds2, numWG); - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + LDStx_end(lds2, numWG); + return; } } @@ -427,7 +454,7 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe // Shufl two or more fft_WIDTHs or fft_HEIGHTs operating on 64-bit values using LDS_BYTES of LDS memory. An fft2 is also performed. void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, u32 lowMe) { assert(RADIX == 8); - assert(SHUFL_BYTES >= 8); + assert(SBMUL(numWG) * SHUFL_BYTES >= 8); u32 mask = f - 1; assert((mask & (mask + 1)) == 0); @@ -438,26 +465,26 @@ void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, // val2 is read from 4 * WG + i * WG/2 // If SHUFL_BYTES is 16 we can write the complete T2 value to LDS memory with one instruction. - if (SHUFL_BYTES == 16) { - local T2_GF61* lds = lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T2_GF61); + if (SBMUL(numWG) * SHUFL_BYTES >= 16) { + local T2_GF61* lds = LDSsharing_ptr(lds2, numWG); // Execute the original shufl code with an fft2 add-on. - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { T2_GF61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; T2_GF61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; if (lowMe < WG / 2) u[i] = addq(val1, val2); else u[i] = subq(val1, val2); } + LDStx_end(lds2, numWG); + return; } // If SHUFL_BYTES is 8 we split the T2 values into two T values. These are written to LDS memory with two instructions. - else if (SHUFL_BYTES == 8) { - local T_Z61* lds = ((local T_Z61*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(T_Z61); + else if (SBMUL(numWG) * SHUFL_BYTES == 8) { + local T_Z61* lds = LDSsharing_ptr((local T_Z61 *)lds2, numWG); #if LDSPAD // Special case second RADIX == 8 to eliminate LDS bank conflicts. @@ -466,9 +493,9 @@ void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 if (f == 8 && RADIX == 8) { local T_Z61 *ldsIn; - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 @@ -477,9 +504,9 @@ void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, if (lowMe < WG / 2) u[i].x = addq(val1, val2); else u[i].x = subq(val1, val2); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 @@ -488,59 +515,63 @@ void OVERLOAD shufl_and_fft2(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 numWG, if (lowMe < WG / 2) u[i].y = addq(val1, val2); else u[i].y = subq(val1, val2); } + LDStx_end(lds2, numWG); return; } #endif // Execute the original shufl code with an fft2 add-on. - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { T_Z61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; T_Z61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; if (lowMe < WG / 2) u[i].x = addq(val1, val2); else u[i].x = subq(val1, val2); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { T_Z61 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; T_Z61 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; if (lowMe < WG / 2) u[i].y = addq(val1, val2); else u[i].y = subq(val1, val2); } + LDStx_end(lds2, numWG); + return; } // If SHUFL_BYTES is 4 we split the T2 values into 4 int values. These are written to LDS memory using four instructions. // NOT OPTIMIZED TO REDUCE LDS BANK CONFLICTS!! - else if (SHUFL_BYTES == 4) { + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { // NEEDS WORK!!! // Lower LDS requirements may let the optimizer use fewer VGPRs and increase occupancy for WIDTHs >= 1024. // Alas, the increased occupancy does not offset extra code needed for shufl_int (the assembly // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. - local int* lds = (local int*) lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(int); + local int* lds = (local int*)LDSsharing_ptr(lds2, numWG); - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } + LDStx_end(lds2, numWG); + return; } } @@ -559,9 +590,8 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. // We're writing 8 bytes at a time, which means groups of 16 must have unique LDS banks. - if (SHUFL_BYTES >= 8) { - local F2_GF31* lds = lds2; - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); + if (SBMUL(numWG) * SHUFL_BYTES >= 8) { + local F2_GF31* lds = LDSsharing_ptr(lds2, numWG); #if LDSPAD // Special case first RADIX == 8 to eliminate LDS bank conflicts. @@ -569,10 +599,11 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS that uses a little padding. Pad two values after every row to eliminate bank conflicts. // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 if (f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 2) + lowMe] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG / 8 + (lowMe / 8) + (lowMe & 7) * (WG + 2)]; } + LDStx_end(lds2, numWG); return; } @@ -581,12 +612,13 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 // Pad 8 values after every 64 values to eliminate bank conflicts. if (1 && f == 8 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i]; } else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG + 8) + lowMe]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + LDStx_end(lds2, numWG); return; } @@ -595,11 +627,12 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS with 8 pads after each row. // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 if (0 && f == 8 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } + LDStx_end(lds2, numWG); return; } @@ -610,11 +643,12 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 // Pad one value after every row to eliminate bank conflicts. if (f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 1) + (lowMe / 16) * 16 + (lowMe & 3) * 4 + i] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 1) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 1) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); return; } @@ -625,11 +659,12 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 // Pad 4 values after every row to eliminate bank conflicts. if (f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 3) * (WG + 4) + (lowMe / 16) * 16 + i * 4 + (lowMe & 3)] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 16 + (lowMe / 16) * (WG + 4) + (lowMe & 15)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + (lowMe / 64) * 16 + ((lowMe / 16) & 3) * (WG + 4) + (lowMe & 15)]; } + LDStx_end(lds2, numWG); return; } #endif @@ -641,11 +676,12 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 1 and 2 multiples of 8) and the first 26 threads read from LDS (multiples of 64 and two multiples of 1). if (f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 8 + i) ^ (lowMe & 15)] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((i & 1) * 8) + ((lowMe / 8) & 7))]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ (((lowMe / 8) & 15))]; } + LDStx_end(lds2, numWG); return; } @@ -655,11 +691,12 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (8 multiples of 64 and 2 multiples of 1) and the first 16 threads read from LDS (8 multiples of 64 and two multiples of 8). if (f == 8 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 8 * 64 + i * 8 + (lowMe & 7)) ^ (lowMe & 8)] = u[i]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((i & 1) * 8)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 8) & 8)]; } + LDStx_end(lds2, numWG); return; } @@ -669,10 +706,11 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 1 and 4 multiples of 4) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 1). if (f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe * 4 + i) ^ (lowMe & 15)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 15)]; } + LDStx_end(lds2, numWG); return; } @@ -682,26 +720,28 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Swizzle LDS blocks to eliminate bank conflicts. // Swizzle on the first 16 threads written to LDS (4 multiples of 64 and 4 multiples of 1) and the first 16 threads read from LDS (4 multiples of 64 and 4 multiples of 16). if (f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[(lowMe / 4 * 16 + i * 4 + (lowMe & 3)) ^ (lowMe & 12)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[(i * WG + lowMe) ^ ((lowMe / 4) & 12)]; } + LDStx_end(lds2, numWG); return; } #endif // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; } // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. // We're writing 4 bytes at a time, which means groups of 32 must have unique LDS banks. - else if (SHUFL_BYTES == 4) { - local F_Z31* lds = ((local F_Z31*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { + local F_Z31* lds = LDSsharing_ptr((local F_Z31 *)lds2, numWG); #if LDSPAD // Special case first RADIX == 8 to eliminate LDS bank conflicts. @@ -711,16 +751,17 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 448, 1, 65... output[64..127] = +8 // Pad one value after every row to eliminate bank conflicts. if (f == 1 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 4) & 7) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 3) * 8 + i] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 4) * 32 + (i & 3) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 256) * 32 + ((lowMe / 32) & 7) * (WG + 1) + (lowMe & 31)]; } + LDStx_end(lds2, numWG); return; } @@ -729,18 +770,19 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 // Pad 8 values after every 64 values to eliminate bank conflicts. if (f == 8 && r == 8 && RADIX == 8) { - bar(WG); + LDStx_start(lds2, numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + LDStx_end(lds2, numWG); return; } @@ -751,16 +793,17 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in the desired output order. In the example: output[0..63] = 0, 64, ... 192, 1, 65... output[64..127] = +16 // Pad one value after every row to eliminate bank conflicts. if (f == 1 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 1) + (lowMe / 32) * 32 + (lowMe & 7) * 4 + i] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 1)) + (lowMe / 32) * (WG + 1) + (lowMe & 31)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 1) + (lowMe & 31)]; } + LDStx_end(lds2, numWG); return; } @@ -771,29 +814,32 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0...192, 16... 32.. 48.. 1... lds[64..127] = +4 // Pad 4 values after every row to eliminate bank conflicts. if (f == 4 && r == 4 && RADIX == 4) { - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].x; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 3) * (WG + 4) + (lowMe / 32) * 32 + ((lowMe / 4) & 1) * 16 + i * 4 + (lowMe & 3)] = u[i].y; } - bar(WG); + LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[(i / 2) * 32 + (i & 1) * (2 * (WG + 4)) + (lowMe / 32) * (WG + 4) + (lowMe & 31)]; } else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + (lowMe / 128) * 32 + ((lowMe / 32) & 3) * (WG + 4) + (lowMe & 31)]; } + LDStx_end(lds2, numWG); return; } #endif // Otherwise, execute the original shufl code modified to handle case where a full RADIX fft was not done - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * WG + lowMe]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * WG + lowMe]; } + LDStx_end(lds2, numWG); + return; } } @@ -819,9 +865,8 @@ void OVERLOAD shufl_and_fft2(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, // val2 is read from 4 * WG + i * WG/2 // If SHUFL_BYTES is 8 or more we can write the complete F2 value to LDS memory with one instruction. - if (SHUFL_BYTES >= 8) { - local F2_GF31* lds = ((local F2_GF31*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F2_GF31); + if (SBMUL(numWG) * SHUFL_BYTES >= 8) { + local F2_GF31* lds = LDSsharing_ptr(lds2, numWG); #if LDSPAD // Special case second RADIX == 8 to eliminate LDS bank conflicts. @@ -830,9 +875,9 @@ void OVERLOAD shufl_and_fft2(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, // Read from LDS in output order. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 if (f == 8 && RADIX == 8) { local F2_GF31 *ldsIn; - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { // Read val1 from the standard shufl's i = i/2, lowMe = lowMe % WG/2 + (i&1) * WG/2 // Read val2 from the standard shufl's i = i/2 + 4, lowMe = lowMe % WG/2 + (i&1) * WG/2 @@ -841,46 +886,50 @@ void OVERLOAD shufl_and_fft2(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 numWG, if (lowMe < WG / 2) u[i] = addq(val1, val2); else u[i] = subq(val1, val2); } + LDStx_end(lds2, numWG); return; } #endif // Execute the original shufl code with an fft2 add-on. - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i]; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { F2_GF31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; F2_GF31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; if (lowMe < WG / 2) u[i] = addq(val1, val2); else u[i] = subq(val1, val2); } + LDStx_end(lds2, numWG); + return; } // If SHUFL_BYTES is 4 we split the F2 values into two F values. These are written to LDS memory using two instructions. - else if (SHUFL_BYTES == 4) { - local F_Z31* lds = ((local F_Z31*) lds2); - if (numWG > 1) lds += ((u32) get_local_id(0) / WG) * LDS_BYTES / sizeof(F_Z31); + else if (SBMUL(numWG) * SHUFL_BYTES == 4) { + local F_Z31* lds = LDSsharing_ptr((local F_Z31 *)lds2, numWG); // Execute the original shufl code with an fft2 add-on. - bar(WG); + LDStx_start(lds2, numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].x; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { F_Z31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; F_Z31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; if (lowMe < WG / 2) u[i].x = addq(val1, val2); else u[i].x = subq(val1, val2); } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = u[i].y; } - bar(WG); + LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { F_Z31 val1 = lds[ i * (WG / 2) + lowMe % (WG / 2)]; F_Z31 val2 = lds[4 * WG + i * (WG / 2) + lowMe % (WG / 2)]; if (lowMe < WG / 2) u[i].y = addq(val1, val2); else u[i].y = subq(val1, val2); } + LDStx_end(lds2, numWG); + return; } } diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 15d9076e..112193b3 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -79,7 +79,9 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { - local T2 lds[LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; T2 u[NH], v[NH]; @@ -193,7 +195,9 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s } KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { - local F2 lds[LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; @@ -303,7 +307,9 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar } KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { - local GF31 lds[LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -432,7 +438,9 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar } KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { - local GF61 lds[LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 53d7d13e..4f1e396d 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -93,7 +93,9 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - local T2 lds[LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + T2 u[NH]; const u32 H = ND / SMALL_HEIGHT; @@ -132,7 +134,9 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local T2 lds[LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; T2 u[NH], v[NH]; @@ -239,7 +243,9 @@ void OVERLOAD pairSq2_special(T2 *u, T2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local T2 lds[2 * LDS_BYTES / sizeof(T2)]; + local T2 lds[LDS_BYTES(2) / sizeof(T2)]; + LDSinit(lds, 2); + const u32 H = ND / SMALL_HEIGHT; T2 u[NH]; @@ -370,7 +376,9 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { - local F2 lds[LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + F2 u[NH]; const u32 H = ND / SMALL_HEIGHT; @@ -405,7 +413,9 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local F2 lds[LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; @@ -508,7 +518,9 @@ void OVERLOAD pairSq2_special(F2 *u, F2 base_squared) { } KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local F2 lds[2 * LDS_BYTES / sizeof(F2)]; + local F2 lds[LDS_BYTES(2) / sizeof(F2)]; + LDSinit(lds, 2); + const u32 H = ND / SMALL_HEIGHT; CP(F2) inF2 = (CP(F2)) in; @@ -639,7 +651,9 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF31 lds[LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -692,7 +706,9 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local GF31 lds[LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -790,7 +806,9 @@ void OVERLOAD pairSq2_special(GF31 *u, GF31 base_squared) { } KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local GF31 lds[2 * LDS_BYTES / sizeof(GF31)]; + local GF31 lds[LDS_BYTES(2) / sizeof(GF31)]; + LDSinit(lds, 2); + const u32 H = ND / SMALL_HEIGHT; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -975,7 +993,9 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { - local GF61 lds[LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -1028,7 +1048,9 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local GF61 lds[LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -1126,7 +1148,9 @@ void OVERLOAD pairSq2_special(GF61 *u, GF61 base_squared) { } KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { - local GF61 lds[2 * LDS_BYTES / sizeof(GF61)]; + local GF61 lds[LDS_BYTES(2) / sizeof(GF61)]; + LDSinit(lds, 2); + const u32 H = ND / SMALL_HEIGHT; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); diff --git a/src/cl/tailutil.cl b/src/cl/tailutil.cl index f5d9fe51..5eb3fe4d 100644 --- a/src/cl/tailutil.cl +++ b/src/cl/tailutil.cl @@ -137,34 +137,34 @@ void OVERLOAD reverse2(local T2_GF61 *lds2, T2_GF61 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; - if (SHUFL_BYTES_H >= 8) { - local T2_GF61 *lds = lds2; - if (me >= WG) lds += LDS_BYTES / sizeof(T2_GF61); + if (SBMUL(2) * SHUFL_BYTES_H >= 8) { + local T2_GF61 *lds = LDSsharing_ptr(lds2, 2); // For NH=8, u[0] to u[3] are left unchanged. Write to lds: // u[7]rev u[6]rev u[5]rev u[4]rev // v[7]rev v[6]rev v[5]rev v[4]rev - bar(WG); + LDStx_start(lds2, 2); for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i]; } // For NH=8, read from lds into u[i]: // u[4] = u[7]rev v[7]rev // u[5] = u[6]rev v[6]rev // u[6] = u[5]rev v[5]rev // u[7] = u[4]rev v[4]rev - bar(WG); + LDSbar(2); for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, 2); } - else if (SHUFL_BYTES_H == 4) { - local T_Z61 *lds = (local T_Z61 *) lds2; - if (me >= WG) lds += LDS_BYTES / sizeof(T_Z61); - bar(WG); + else if (SBMUL(2) * SHUFL_BYTES_H == 4) { + local T_Z61 *lds = LDSsharing_ptr((local T_Z61 *)lds2, 2); + LDStx_start(lds2, 2); for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i].x; } - bar(WG); + LDSbar(2); for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].x = lds[i * WG + lowMe]; } - bar(WG); + LDSbar(2); for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i].y; } - bar(WG); + LDSbar(2); for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i].y = lds[i * WG + lowMe]; } + LDStx_end(lds2, 2); } } @@ -177,19 +177,21 @@ void OVERLOAD revCrossLine(local T2_GF61 *lds2, T2_GF61 *u) { if (SHUFL_BYTES_H >= 8) { local T2_GF61 *ldsOut = lds2; local T2_GF61 *ldsIn = lds2; - if (me < WG) ldsOut += LDS_BYTES / sizeof(T2_GF61); // Crossing LDS halves - else ldsIn += LDS_BYTES / sizeof(T2_GF61); // Staying within LDS halves (just like shufl) + if (me < WG) ldsOut += LDS_SHUFL_BYTES(2) / sizeof(T2_GF61); // Crossing LDS halves + else ldsIn += LDS_SHUFL_BYTES(2) / sizeof(T2_GF61); // Staying within LDS halves (just like shufl) bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[WG * i + lowMe]; } + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(2)) bar(); } else if (SHUFL_BYTES_H == 4) { local T_Z61 *ldsOut = (local T_Z61 *) lds2; local T_Z61 *ldsIn = (local T_Z61 *) lds2; - if (me < WG) ldsOut += LDS_BYTES / sizeof(T_Z61); - else ldsIn += LDS_BYTES / sizeof(T_Z61); + if (me < WG) ldsOut += LDS_SHUFL_BYTES(2) / sizeof(T_Z61); + else ldsIn += LDS_SHUFL_BYTES(2) / sizeof(T_Z61); bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].x; } bar(); // we need a full bar because we just crossed halves @@ -198,6 +200,8 @@ void OVERLOAD revCrossLine(local T2_GF61 *lds2, T2_GF61 *u) { for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2].y; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2].y = ldsIn[WG * i + lowMe]; } + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(2)) bar(); } } @@ -346,24 +350,25 @@ void OVERLOAD reverseLine(local F2_GF31 *lds, F2_GF31 *u) { // These versions are for the kernel(s) that use a double-wide workgroup (u in half the workgroup, v in the other half) // -void OVERLOAD reverse2(local F2_GF31 *lds, F2_GF31 *u) { +void OVERLOAD reverse2(local F2_GF31 *lds2, F2_GF31 *u) { u32 me = get_local_id(0); u32 lowMe = me % WG; - if (SHUFL_BYTES_H >= 4) { - if (me >= WG) lds += LDS_BYTES / sizeof(F2); + if (SBMUL(2) * SHUFL_BYTES_H >= 4) { + local F2_GF31 *lds = LDSsharing_ptr(lds2, 2); // For NH=8, u[0] to u[3] are left unchanged. Write to lds: // u[7]rev u[6]rev u[5]rev u[4]rev // v[7]rev v[6]rev v[5]rev v[4]rev - bar(WG); + LDStx_start(lds2, 2); for (u32 i = 0; i < NH/2; ++i) { lds[((NH/2 - i) * WG - (me >= WG ? 1 : 0) - lowMe) % (NH/2 * WG)] = u[NH/2 + i]; } // For NH=8, read from lds into u[i]: // u[4] = u[7]rev v[7]rev // u[5] = u[6]rev v[6]rev // u[6] = u[5]rev v[5]rev // u[7] = u[4]rev v[4]rev - bar(WG); + LDSbar(2); for (u32 i = 0; i < NH/2; ++i) { u[NH/2 + i] = lds[i * WG + lowMe]; } + LDStx_end(lds2, 2); } } @@ -376,12 +381,14 @@ void OVERLOAD revCrossLine(local F2_GF31 *lds2, F2_GF31 *u) { if (SHUFL_BYTES_H >= 4) { local F2_GF31 *ldsOut = lds2; local F2_GF31 *ldsIn = lds2; - if (me < WG) ldsOut += LDS_BYTES / sizeof(F2); - else ldsIn += LDS_BYTES / sizeof(F2); + if (me < WG) ldsOut += LDS_SHUFL_BYTES(2) / sizeof(F2); + else ldsIn += LDS_SHUFL_BYTES(2) / sizeof(F2); bar(); // we need a full bar because we're crossing halves for (u32 i = 0; i < NH/2; ++i) { ldsOut[WG * (NH/2 - 1 - i) + revLowMe] = u[i + NH/2]; } bar(); // we need a full bar because we just crossed halves. LDS reads are compatible with future shufl calls. for (u32 i = 0; i < NH/2; ++i) { u[i + NH/2] = ldsIn[WG * i + lowMe]; } + // One last bar() is needed when sharing LDS memory. This is because when sharing a workgroup will write to more than its own LDS area. + if (SHARING_LDS(2)) bar(); } } From 982b74f7bf45603df2fa2da5e7d7d362572341ff Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 21:48:29 -0600 Subject: [PATCH 165/214] Stop the statistics kernels writing past the sample buffers updateStats() records one sample per kernel invocation at a position it reads and increments from bufROE[0] / bufStatsCarry[0] (kept in the buffer so that CUDA graphs can replay the kernel), with no bound. The host caps only its own counters (carryPos < CARRY_SIZE) and resets the GPU-side position when it reads the samples -- readCarryStats() -- which happens in the PRP loop and in measureCarry, and nowhere else. isPrimeLL and isCERT never call it. So with -use STATS set, an LL or CERT run keeps recording carry statistics into bufStatsCarry (CARRY_SIZE + 2 = 100002 floats) forever: after 100000 iterations every further iteration does an atomic_max past the end of the buffer, into whatever the driver placed after it (buf1 is allocated right after). A PRP run with -log >= 200000 overflows by up to 400 KB every log interval. For ROE the host avoids this by switching to the non-ROE kernel once roePos reaches wantROE; nothing equivalent exists for the carry stats. Pass the buffer capacity to the kernels as STATS_SIZE and stop recording, and stop advancing the position, once it is reached. The ROE path is unchanged in effect (the host already keeps roePos <= ROE_SIZE). The size enum moves above clDefines() so it is visible there. Checked on an Intel UHD iGPU: -prp 5000011 -fft 256:2:256 -use STATS=1 passes its Gerbicz check with the same residue as without STATS, and -ll with -use STATS=1 compiles and runs. Found by an AI code audit (Claude, Anthropic's Claude Code) of the host layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Gpu.cpp | 11 +++++++---- src/cl/carryutil.cl | 20 ++++++++++++-------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ba5e3f79..fd6f4d5f 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -233,6 +233,12 @@ constexpr bool isInList(const string& s, initializer_list list) { return false; } +// Capacity of the ROE and carry statistics sample buffers; passed to the kernels as STATS_SIZE so they stop recording when full. +enum { +ROE_SIZE = 100000, +CARRY_SIZE = 100000 +}; + string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, u64 E, bool doLog, bool &tail_single_wide, bool &tail_single_kernel, u32 &in_place, u32 &pad_size, u32 &wmul) { map config; @@ -377,6 +383,7 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector(ROE_SIZE, CARRY_SIZE))); if (fft.FFT_FP64 || fft.FFT_FP32) { defines += toDefine("WEIGHT_STEP", weightM1(N, E, fft.shape.height * fft.shape.middle, 0, 0, 1)); @@ -776,10 +783,6 @@ string Gpu::kernelDefines(enum WHICH_KERNEL_TYPE which_kernel) { return defines + " "; } -enum { -ROE_SIZE = 100000, -CARRY_SIZE = 100000 -}; Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, bool logFftSize) : shared(s), diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 4a63583d..9d8dda6b 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -161,14 +161,18 @@ void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *b // CUDA graphs don't allow arguments to change. Thus, calculating posROE and storing it in bufROE works better. if (me < num_threads) { posROE = bufROE[0]; - atomic_max(bufROE + posROE + 2, u32RoundMax); - - // The second bufRoe entry is a count of the number atomic_maxes performed. When the last atomic_max is done, increment posROE and clear the counter. - if (me == 0) { - u32 old_value = atomic_add(bufROE + 1, 1); - if (old_value == num_blocks - 1) { - bufROE[0] = posROE + 1; - bufROE[1] = 0; + // The buffer holds STATS_SIZE samples. The host resets the position only when it reads the samples, and the LL and + // CERT loops never read the carry statistics, so once the buffer is full stop recording rather than write past it. + if (posROE < STATS_SIZE) { + atomic_max(bufROE + posROE + 2, u32RoundMax); + + // The second bufRoe entry is a count of the number atomic_maxes performed. When the last atomic_max is done, increment posROE and clear the counter. + if (me == 0) { + u32 old_value = atomic_add(bufROE + 1, 1); + if (old_value == num_blocks - 1) { + bufROE[0] = posROE + 1; + bufROE[1] = 0; + } } } } From 35a03bedb2ef82f3be472334420d6ce8c99d1c6f Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 21:51:01 -0600 Subject: [PATCH 166/214] Fix the L2_STRIPING branch of fftHin's line numbering get_line_number() in ffthin.cl used stripe_group_size two lines before declaring it, and its "if (g linesInOneStripeGroup)" had no comparison operator, so ffthin.cl did not compile at all with -DL2_STRIPING=. Kernels are compiled lazily and fftHin is only used by exponentiate() -- proof generation and verification, and CERT -- so a PRP run with -use INPLACE=1,L2_STRIPING=n completes and then fails when it tries to generate its proof. The mapping the code intends is right as written: unlike tailSquare, fftHin has no Hermitian-pair readiness rule, and by the time it runs for a block fftMiddleIn has produced every line of both the base_lo and base_hi stripe groups, which is exactly the 2 * L2_STRIPING * 16 * MIDDLE lines the host hands it. Declare stripe_group_size first and restore the "<". Checked on an Intel UHD iGPU by verifying a freshly generated proof of M799999 with -fft 256:2:256: -use INPLACE=1 verifies; adding L2_STRIPING=2 failed to compile ffthin.cl before this change and verifies with it. Found by an AI code audit (Claude, Anthropic's Claude Code) of the tail and striping kernels; fixed and tested by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/cl/ffthin.cl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cl/ffthin.cl b/src/cl/ffthin.cl index e95eb88d..516d49cc 100644 --- a/src/cl/ffthin.cl +++ b/src/cl/ffthin.cl @@ -14,13 +14,14 @@ u32 get_line_number(u32 base_lo) { // Old, simple L2 striping code // return g / (L2_STRIPING * 16) * WIDTH + base + g % (L2_STRIPING * 16); - // Process stripe group base_lo or base_hi + // Process stripe group base_lo or base_hi. Unlike tailSquare, there is no Hermitian-pair readiness rule here: + // by the time fftHin runs for a block, fftMiddleIn has produced every line of both stripe groups. + u32 stripe_group_size = L2_STRIPING; u32 base_hi = WIDTH - stripe_group_size * 16 - base_lo; u32 linesInOneStripe = 16 * MIDDLE; - u32 stripe_group_size = L2_STRIPING; u32 linesInOneStripeGroup = stripe_group_size * linesInOneStripe; u32 base; - if (g linesInOneStripeGroup) base = base_lo; + if (g < linesInOneStripeGroup) base = base_lo; else base = base_hi, g -= linesInOneStripeGroup; return g / (L2_STRIPING * 16) * WIDTH + base + g % (L2_STRIPING * 16); #else From 7e6e2089eec8d01a8b6b4149f3cd74cc15df6c85 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Wed, 16 Sep 2026 21:53:27 -0600 Subject: [PATCH 167/214] Do not lose a finished PRP result over proof generation Three related problems at the end of isPrimePRP: 1. saveProof() throws "bad proof generation" once its retries and reducePower() are exhausted, and the throw escapes the only place the result (res64, res2048, isPrime) is returned. No verified savefile is written for k >= kEnd, so a restart redoes the tail and, if the bad residue is one every power shares (e.g. a bit-rotted /proof/<(E+1)/2>, whose size still passes fileExists and which isValidTo does not CRC), fails the same way forever. The test's answer is known and correct and is never reported. 2. getProofPower() returning 0 logs "Proof generation disabled!" and then ProofSet{E, 0} throws "Invalid proof power", so a resumed run whose (E+1)/2 residue is missing can never proceed; the worker leaves its task loop on every restart. 3. The k == E proof residue is written by the Background thread and computeProof() reads it back after fewer than blockSize squarings with no join, so on a slow disk it can see a missing or half-written file. Hold the ProofSet in an optional and only construct it for power > 0; wait for the background writes before generating the proof; and if generation fails, log it and return the result with an empty proofPath (writeResultPRP already omits the "proof" object then). A stop request during proof generation is still propagated. The normal path is unchanged (checked with a short PRP on an Intel UHD iGPU); the failure paths themselves were not exercised. Found by an AI code audit (Claude, Anthropic's Claude Code) of the host layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Gpu.cpp | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ba5e3f79..0e26c0f9 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -2592,8 +2592,11 @@ PRPResult Gpu::isPrimePRP([[maybe_unused]] const Task& task) { assert(checkStep % blockSize == 0); u32 const power = getProofPower(k); - - ProofSet proofSet{E, power}; + + // power == 0 means "proof generation disabled" (no complete set of residues can be built from here on). ProofSet does + // not accept 0, so only construct one when there is a proof to make. + std::optional proofSet; + if (power) { proofSet.emplace(E, power); } bool isPrime = false; @@ -2612,7 +2615,7 @@ PRPResult Gpu::isPrimePRP([[maybe_unused]] const Task& task) { bool skipNextCheckUpdate = false; - u64 persistK = proofSet.next(k); + u64 persistK = proofSet ? proofSet->next(k) : u64(-1); enum LEAD_TYPE leadIn = LEAD_NONE; assert(k % blockSize == 0); @@ -2655,7 +2658,7 @@ PRPResult Gpu::isPrimePRP([[maybe_unused]] const Task& task) { goto reload; } (*background)([=, E=this->E] { ProofSet::save(E, power, k, compactBits(rawData, E)); }); - persistK = proofSet.next(k); + persistK = proofSet->next(k); } if (k == kEnd) { @@ -2714,9 +2717,21 @@ PRPResult Gpu::isPrimePRP([[maybe_unused]] const Task& task) { doBigLog(k, res, ok, secsPerIt, kEndEnd, nErrors); if (k >= kEndEnd) { - fs::path const proofFile = saveProof(args, proofSet); + // The test is complete; nothing after this point may lose the result. Make sure the final proof residue, + // written by the background thread at k == E, is on disk before computeProof reads it back, and if the proof + // cannot be generated report the result without one rather than throw it away. + fs::path proofFile; + if (proofSet) { + background->waitEmpty(); + try { + proofFile = saveProof(args, *proofSet); + } catch (...) { + if (Signal::stopRequested()) { throw; } + log("Proof generation failed; reporting the result without a proof\n"); + } + } return {.isPrime=isPrime, .res64=finalRes64, .nErrors=nErrors, .proofPath=proofFile.string(), .res2048=toHex(res2048)}; - } + } } else { ++nErrors; doBigLog(k, res, ok, secsPerIt, kEndEnd, nErrors); From 480af43af10589b1f459973598cb098bc172ec8a Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:35:50 -0600 Subject: [PATCH 168/214] Tag every multiplication's ROE sample as a multiplication Gpu::mul() records the current ROE slot in mulRoePos so that readROE() can split the samples into squaring and multiplication statistics. The stored value is in the "+ 2" format (an index into the raw bufROE vector, whose first two entries are bookkeeping) but the dedup test compared against the raw roePos, so after one multiplication back() == roePos + 1 > roePos and the next consecutive multiplication was not recorded. writeState, doCheck and exponentiate issue runs of back-to-back modMul calls, so all but the first were folded into the squaring statistics that drive the Z-score, the "Danger ROE!" warning and measureROE. readROE() also returned early when roePos was 0 without clearing mulRoePos, so indices recorded while sampling was off carried into the next window. Compare in the same format and clear on both paths. Found by an AI code audit (Claude, Anthropic's Claude Code) of the host layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Gpu.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ba5e3f79..2149a4e1 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -1673,6 +1673,7 @@ pair Gpu::readROE() { mulRoePos.clear(); return {roeStat(squareRoe), roeStat(mulRoe)}; } else { + mulRoePos.clear(); // indices recorded while ROE sampling was off must not tag the next window return {}; } } @@ -1759,7 +1760,8 @@ void Gpu::mul(Buffer& ioA, Buffer& inB, Buffer& tmp1, bool fftW(buf3, tmp1); // Register the current ROE pos as multiplication (vs. a squaring) - if (mulRoePos.empty() || mulRoePos.back() < roePos) { mulRoePos.push_back(roePos + 2); } + // mulRoePos holds indices in the "+ 2" format of the raw bufROE vector, so compare in that format too. + if (mulRoePos.empty() || mulRoePos.back() != roePos + 2) { mulRoePos.push_back(roePos + 2); } if (mul3) { carryM(ioA, buf3); } else { carryA(ioA, buf3); } carryB(ioA); From 26a9f9306de491dc4c0570f075245b53a6775844 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:38:52 -0600 Subject: [PATCH 169/214] Skip a Cert worktodo line whose .cert start file is missing bestTask() always prefers a Cert task over PRP/LL, and isCERT() opens M.cert with File::openReadThrow, which throws fs::filesystem_error when the file is absent. That is a std::exception, caught in gpuWorker(), which then returns: the worker's task loop ends. The Cert line is only removed after success, so every restart picks the same line, throws, and exits -- all PRP/LL work behind it is blocked with no recovery. The file goes missing on its own if anything fails between fs::remove(fname) and deleteTask at the end of a CERT, and with a hand-edited worktodo or a partial download. Skip such a line in bestTask() with a log message, so the worker moves on to the other tasks; the line stays for when the file appears (AutoPrimeNet adds the line only after the download). Before/after with a Cert line, no M.cert, and a PRP line behind it: the worker exited on the filesystem_error and the PRP never ran; now the line is skipped and the PRP runs. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Worktodo.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 44a5b075..eae976ea 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -12,6 +12,8 @@ #include #include #include +#include +#include namespace { @@ -97,6 +99,12 @@ static std::optional bestTask(const fs::path& fileName, bool smallest) { optional best; for (const string& line : File::openRead(fileName)) { optional task = parse(line); + // A Cert line whose start-value file is not here cannot run: isCERT would throw and end the worker, and since + // Cert lines take priority over PRP/LL the worker would be wedged for good. Skip the line until the file appears. + if (task && task->kind == Task::CERT && !std::filesystem::exists("M" + to_string(task->exponent) + ".cert")) { + log("Cert start file M%" PRIu64 ".cert not found; skipping that worktodo line for now\n", task->exponent); + continue; + } if (task && (!best || (best->kind != Task::CERT && task->kind == Task::CERT) || ((best->kind != Task::CERT || task->kind == Task::CERT) && smallest && task->exponent < best->exponent))) { From 32c3d7ed80eb6e22bcd75738d4f26ae4985449ea Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:38:52 -0600 Subject: [PATCH 170/214] Make file errors std::exceptions so top-level handlers catch them CRCError, ReadError, WriteError (File.h) and BadHeaderError (Saver.cpp) were plain structs. Only Saver::load catches them by type; the top-level handlers in main(), gpuWorker() and Background::run() catch const char*, std::string and std::exception, so any of these escaping ends in std::terminate: - a worktodo line without a trailing newline, or longer than 1023 characters, makes File::readLine throw ReadError inside bestTask -> the process aborts ("terminate called after throwing an instance of 'ReadError'") instead of reporting the line; - a full disk while the Background thread writes a savefile or proof residue -> WriteError -> terminate of the whole process, all workers; - ENOSPC appending to results-N.txt -> WriteError -> terminate. Derive them from std::runtime_error (keeping the `name` member and the brace-init throw sites), and add a std::exception handler to main(), which had none. Existing catch-by-type sites are unaffected. Before/after on a worktodo-0.txt whose only line lacks a newline: exit 134 with the terminate message; now "Exception ReadError: read error: worktodo-0.txt" and a clean exit. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/File.h | 17 ++++++++--------- src/Saver.cpp | 5 ++++- src/main.cpp | 3 +++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/File.h b/src/File.h index 03c7a86e..86f96a2e 100644 --- a/src/File.h +++ b/src/File.h @@ -12,6 +12,7 @@ #include #endif #include +#include #include #include #include @@ -42,17 +43,15 @@ namespace fs = std::filesystem; -struct CRCError { - std::string name; -}; - -struct ReadError { - std::string name; -}; - -struct WriteError { +// File errors derive from std::exception so that the top-level handlers in main(), gpuWorker() and +// Background::run() log them and carry on instead of letting them reach std::terminate. +struct FileError : std::runtime_error { std::string name; + FileError(const char* kind, std::string n) : std::runtime_error(std::string(kind) + ": " + n), name(std::move(n)) {} }; +struct CRCError : FileError { explicit CRCError(std::string n) : FileError("CRC error", std::move(n)) {} }; +struct ReadError : FileError { explicit ReadError(std::string n) : FileError("read error", std::move(n)) {} }; +struct WriteError : FileError { explicit WriteError(std::string n) : FileError("write error", std::move(n)) {} }; class File { FILE* f = nullptr; diff --git a/src/Saver.cpp b/src/Saver.cpp index dfb2b314..6bfd7db4 100644 --- a/src/Saver.cpp +++ b/src/Saver.cpp @@ -30,7 +30,10 @@ static constexpr const char *LL_v1 = "OWL LL 1 E=%" PRIu64 " k=%" PRIu64 " CRC=% // Push version number to sync it with PRP. static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " time=%lf\n"; -struct BadHeaderError { string name; }; +struct BadHeaderError : std::runtime_error { + string name; + explicit BadHeaderError(string n) : std::runtime_error("bad savefile header: " + n), name(std::move(n)) {} +}; bool startsWith(const string& s, const string& prefix) { return s.starts_with(prefix); diff --git a/src/main.cpp b/src/main.cpp index becccf9b..7a2f1e8c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -144,6 +144,9 @@ int main(int argc, char **argv) { } catch (const string& mes) { log("Exiting because \"%s\"\n", mes.c_str()); exitCode = isCleanExit(mes.c_str()) ? 0 : 1; + } catch (const std::exception& e) { + log("Exiting because of exception %s: %s\n", typeName(e), e.what()); + exitCode = 1; } log("Bye\n"); From 554af7e44b2de626cb9d0a8c1d621b7b2695f034 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:42:28 -0600 Subject: [PATCH 171/214] Only accept Mersenne "1,2,E,-1" shapes on PRP worktodo lines Worktodo::parse took the exponent from parts[2] when a PRP/PRPDC line had the "1,2,E,-1" shape and otherwise fell back to parts[0]. That fallback exists for Test=/DoubleCheck= lines, whose first field is the exponent, but it was applied to PRP lines too, so an assignment for a different number, e.g. PRP=,1234567,2,100003,-1 (k*2^n-1) or one with c=+1 or base 3, was accepted as a Mersenne PRP of exponent k and reported under that AID -- and since Task::execute retargets a composite exponent to the previous prime, the line above actually ran M1234547. Use the bare "E,..." fallback only for LL lines; a PRP line that is not "1,2,E,-1" is now reported as ignored. Before/after on the line above: it ran ("Exponent 1234567 is not prime. Using exponent 1234547 instead"); now "worktodo.txt line ignored", while a Test= line behind it still parses and runs. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Worktodo.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index 44a5b075..06209827 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -58,8 +58,10 @@ std::optional parse(const std::string& line) { parts.erase(parts.begin()); } - string const s = (parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && (parts[3] == "-1" || parts[3] == "-1\n")) ? parts[2] - : (!parts.empty() ? parts[0] : ""); + // PRP lines are "k,b,n,c,..." and only k=1, b=2, c=-1 is a Mersenne number; anything else (k*2^n-1, 2^n+1, base 3) + // is not ours and must not be run as exponent k. The bare "E,..." form is only used by Test=/DoubleCheck= lines. + bool const mersenne = parts.size() >= 4 && parts[0] == "1" && parts[1] == "2" && (parts[3] == "-1" || parts[3] == "-1\n"); + string const s = mersenne ? parts[2] : ((isLL && !parts.empty()) ? parts[0] : ""); const char *end = s.c_str() + s.size(); u64 exp{}; From ef82c28e5ed9466e8b6f7488a5db3e2d77b585a9 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:42:33 -0600 Subject: [PATCH 172/214] Validate -log, -save and -maxAlloc values and "!" config lines -log accepted 0 (a multiple of 1000) and the PRP loop then divides by it (k % logStep) -> SIGFPE. -save accepted 0, which makes Saver::trimFiles index v[-1] on the first checkpoint; the assert guarding it is debug-only. -maxAlloc with no value did s.back() on an empty string (an assert in debug, undefined in release). "!" per-FFT config lines ignored the sscanf result, so a line with fewer than two tokens used uninitialised buffers. Reject each with a message instead. (Non-numeric values to stoi/stod throw std::invalid_argument, which main() now reports instead of terminating on, since the file-error change.) Found by an AI code audit (Claude, Anthropic's Claude Code) of argument handling; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Args.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Args.cpp b/src/Args.cpp index d4744d68..29e83aff 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -271,7 +271,10 @@ void Args::parse(const string& line) { // conditional defines predicated on a FFT char fftBuf[32]; char configBuf[256]; - sscanf(line.c_str(), "! %31s %255s", fftBuf, configBuf); + if (sscanf(line.c_str(), "! %31s %255s", fftBuf, configBuf) != 2) { // otherwise the buffers are uninitialised + log("config line ignored (expected \"! \"): \"%s\"\n", line.c_str()); + return; + } string const fft = fftBuf; string const config = configBuf; perFftConfig[fft] = splitUses(config); @@ -372,7 +375,10 @@ void Args::parse(const string& line) { } } else if (key == "-maxAlloc" || key == "-maxalloc") { - assert(!s.empty()); + if (s.empty()) { // s.back() below would be undefined + log("-maxAlloc expects a value, e.g. -maxAlloc 4G\n"); + throw "-maxAlloc "; + } u32 multiple = (s.back() == 'G') ? (1u << 30) : (1u << 20); maxAlloc = size_t(stod(s) * multiple + .5); } @@ -402,8 +408,8 @@ void Args::parse(const string& line) { } } else if (key == "-log") { logStep = stoi(s); - if (logStep % 1000 != 0) { - log("-log must be a multiple of 1000\n"); + if (logStep == 0 || logStep % 1000 != 0) { // 0 would divide by zero in the PRP loop + log("-log must be a positive multiple of 1000\n"); throw "invalid log size"; } } else if (key == "-use") { @@ -417,7 +423,12 @@ void Args::parse(const string& line) { } else if (key == "-unsafeMath") { safeMath = false; } else if (key == "-save") { - nSavefiles = stoi(s); + int const n = stoi(s); + if (n < 1) { // 0 makes Saver::trimFiles index v[-1] + log("-save must be at least 1\n"); + throw "invalid -save value"; + } + nSavefiles = n; } else { log("Argument '%s' '%s' not understood\n", key.c_str(), s.c_str()); throw "args"; From a04a25510e43902ace19a5f33d1c5c53c82a1dcc Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:43:52 -0600 Subject: [PATCH 173/214] Escape strings in the result JSON Task::json(const string&) wrapped the value in quotes with no escaping. -user, -uid (or the derived tailDir) and the uname release/arch strings go straight into the results line, so a value containing a quote or a backslash produced an unparseable JSON record. Escape the quote, the backslash and control characters (as \uXXXX). Found by an AI code audit (Claude, Anthropic's Claude Code) of result reporting; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Task.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/Task.cpp b/src/Task.cpp index 9d8d3648..7ec35f23 100644 --- a/src/Task.cpp +++ b/src/Task.cpp @@ -15,6 +15,7 @@ #include #include +#include #include namespace { @@ -103,7 +104,17 @@ string json(const vector& v) { return {isFirst ? ""s : (s + '}')}; } -string json(const string& s) { return '"' + s + '"'; } +// JSON string literal: escape the quote, the backslash and control characters, so a user name or OS string +// containing them cannot produce an unparseable results line. +string json(const string& s) { + string out = "\""; + for (unsigned char c : s) { + if (c == '"' || c == '\\') { out += '\\'; out += char(c); } + else if (c < 0x20) { char buf[8]; snprintf(buf, sizeof(buf), "\\u%04x", c); out += buf; } + else { out += char(c); } + } + return out + '"'; +} string json(int x) { return to_string(x); } string json(u32 x) { return to_string(x); } string json(u64 x) { return to_string(x); } From 3cc46732bcddd6f8680b2f9597bf89c12782103c Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:46:25 -0600 Subject: [PATCH 174/214] Check the savefile header's exponent before sizing the residue read readState() parsed the exponent out of the savefile header and used it to size the residue read (nWords(exponent)) before anyone compared it with the exponent being run; Saver::load() only checks state.exponent afterwards. nWords() returns u32, so a corrupt or foreign header controls a resize(): N=1*2^137438953376 gives nWords = 2^32 - 1, a 16 GB allocation, and the resulting std::bad_alloc / length_error is not one of the errors load() catches, so it escapes instead of taking the "move the bad file aside and use the previous checkpoint" path. (Other bogus values wrap to a small nWords and end in a CRC error, which is handled.) Pass the expected exponent into readState() and reject a mismatching header with BadHeaderError before any read, with a log line saying what was found. Checked on an Intel UHD iGPU with a savefile whose header says N=1*2^137438953471-1 for exponent 5000011: the header is reported, the file is moved to .bad and the run restarts from the previous state. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Saver.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Saver.cpp b/src/Saver.cpp index dfb2b314..68f8ff90 100644 --- a/src/Saver.cpp +++ b/src/Saver.cpp @@ -32,6 +32,14 @@ static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 struct BadHeaderError { string name; }; +// The header's exponent sizes the residue read that follows; a corrupt header must not be allowed to size it. +void checkExponent(u64 got, u64 want, const string& header) { + if (got != want) { + log("savefile header exponent %" PRIu64 " does not match %" PRIu64 ": \"%s\"\n", got, want, rstripNewline(header).c_str()); + throw BadHeaderError{header}; + } +} + bool startsWith(const string& s, const string& prefix) { return s.starts_with(prefix); } @@ -88,7 +96,7 @@ fs::path findLast(const fs::path& dir, const string& prefix, const string& kind) return path; } -PRPState readState([[maybe_unused]] const PRPState& dummy, File fi) { +PRPState readState(const PRPState& expected, File fi) { u64 exponent{}, k{}; u32 blockSize{}, nErrors{}; u64 res64{}; @@ -97,11 +105,13 @@ PRPState readState([[maybe_unused]] const PRPState& dummy, File fi) { string const header = fi.readLine(); if (sscanf(header.c_str(), PRP_v13, &exponent, &k, &blockSize, &res64, &nErrors, &elapsed) == 6) { + checkExponent(exponent, expected.exponent, header); return {.exponent=exponent, .k=k, .blockSize=blockSize, .res64=res64, .check=fi.readChecked(nWords(exponent)), .nErrors=nErrors, .elapsed=elapsed}; } u32 crc{}; if (sscanf(header.c_str(), PRP_v12, &exponent, &k, &blockSize, &res64, &nErrors, &crc) == 6) { + checkExponent(exponent, expected.exponent, header); return {.exponent=exponent, .k=k, .blockSize=blockSize, .res64=res64, .check=fi.readWithCRC(nWords(exponent), crc), .nErrors=nErrors, .elapsed=0}; } @@ -109,18 +119,20 @@ PRPState readState([[maybe_unused]] const PRPState& dummy, File fi) { throw BadHeaderError{fi.name}; } -LLState readState([[maybe_unused]] const LLState& dummy, File fi) { +LLState readState(const LLState& expected, File fi) { u64 exponent{}, k{}; double elapsed{}; string const header = fi.readLine(); if (sscanf(header.c_str(), LL_v13, &exponent, &k, &elapsed) == 3) { + checkExponent(exponent, expected.exponent, header); return {.exponent=exponent, .k=k, .data=fi.readChecked(nWords(exponent)), .elapsed=elapsed}; } u32 crc{}; if (sscanf(header.c_str(), LL_v1, &exponent, &k, &crc) == 3) { + checkExponent(exponent, expected.exponent, header); return {.exponent=exponent, .k=k, .data=fi.readWithCRC(nWords(exponent), crc), .elapsed=0}; } @@ -226,7 +238,9 @@ State Saver::load() { if (File fi{File::openRead(path)}; fi) { try { - State state = readState(State{}, std::move(fi)); + State expected{}; + expected.exponent = exponent; + State state = readState(expected, std::move(fi)); assert(state.exponent == exponent); if (state.exponent == exponent) { return state; From 72804e45386b7a80c336c752fb41db7a51b90c41 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:49:56 -0600 Subject: [PATCH 175/214] CycleFile: do not promote a partial file when the write threw ~CycleFile unconditionally renamed ".new" over "". If the destructor runs during stack unwinding -- File::write threw WriteError because the disk filled up half way through a savefile -- the partial .new file replaced the previous good checkpoint. The next load then fails its CRC, moves the file aside and falls back one more checkpoint, so the damage is bounded, but a good file was destroyed for nothing. Record std::uncaught_exceptions() at construction and, if more exceptions are in flight at destruction, delete the .new file instead of renaming it. Not exercised (needs a full disk); the change is confined to the destructor. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/CycleFile.cpp | 9 +++++++++ src/CycleFile.h | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/src/CycleFile.cpp b/src/CycleFile.cpp index 0b1db8fd..c2488093 100644 --- a/src/CycleFile.cpp +++ b/src/CycleFile.cpp @@ -3,6 +3,9 @@ #include "CycleFile.h" #include "fs.h" +#include +#include + CycleFile::CycleFile(const fs::path& name) : name{name}, f{File::openWrite(name + ".new")} @@ -12,6 +15,12 @@ CycleFile::CycleFile(const fs::path& name) : CycleFile::~CycleFile() { if (!f) { return; } f.reset(); + if (std::uncaught_exceptions() > uncaughtAtStart) { + // The write threw (WriteError on a full disk, say): keep the previous file, drop the partial one. + std::error_code ec; + fs::remove(name + ".new", ec); + return; + } fancyRename(name + ".new", name); } diff --git a/src/CycleFile.h b/src/CycleFile.h index 816c3456..e02e8163 100644 --- a/src/CycleFile.h +++ b/src/CycleFile.h @@ -2,6 +2,8 @@ #pragma once +#include + #include "File.h" #include @@ -21,4 +23,8 @@ class CycleFile { // Cancel the rename void reset(); + + // Exceptions in flight when this object was created; if more are in flight when it is destroyed, the write + // is being unwound (e.g. disk full) and the partial .new file must not replace the previous good file. + int uncaughtAtStart = std::uncaught_exceptions(); }; From b4c23409e7fcb089b42f512dddf51e926eb92d08 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:50:02 -0600 Subject: [PATCH 176/214] Two file-utility edge cases: empty readAll(), deleteLine temp cleanup File::readAll() on an empty file called read(0), whose fread of zero bytes returns 0 and is reported as a ReadError; a legitimately empty file (the only caller reads kernel source/include files) was an error. Return an empty string. deleteLine() creates "-" and, when the line is not found or the size check fails, returned without removing it, leaving a temp file on disk for every failed claim attempt. Remove it on those paths. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/File.h | 1 + src/fs.cpp | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/File.h b/src/File.h index 03c7a86e..9e28e2cf 100644 --- a/src/File.h +++ b/src/File.h @@ -264,6 +264,7 @@ class File { string readAll() { u64 const sz = size(); + if (sz == 0) { return {}; } // fread of 0 bytes would be reported as a ReadError return {read(sz).data(), sz}; } }; diff --git a/src/fs.cpp b/src/fs.cpp index 8eefa860..7a4c4d6f 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -63,7 +63,11 @@ bool deleteLine(const fs::path& path, const string& targetLine, u64 initialSize) fs::path const tmp = path + ("-"s + toString(this_thread::get_id())); - if (!copyWithout(targetLine, path, tmp) || !sizeMatches(path, initialSize)) { return false; } + if (!copyWithout(targetLine, path, tmp) || !sizeMatches(path, initialSize)) { + error_code ec; + fs::remove(tmp, ec); // do not leave "-" behind on every failed attempt + return false; + } fancyRename(tmp, path); return true; From fa0ab045188fb137c02223043a969325971a9030 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:51:28 -0600 Subject: [PATCH 177/214] tune.txt: skip an unparseable line instead of using an uninitialised spec TuneEntry::readTuneFile logged an unparseable tune.txt line as "ignored" but did not continue, so an FFTConfig was then built from the uninitialised specBuf. A blank line or a "#" comment in tune.txt killed startup with "Invalid FFT spec". Found by an AI code audit (Claude, Anthropic's Claude Code) of FFT selection; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/TuneEntry.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TuneEntry.cpp b/src/TuneEntry.cpp index da4e6dad..dd741105 100644 --- a/src/TuneEntry.cpp +++ b/src/TuneEntry.cpp @@ -60,6 +60,7 @@ vector TuneEntry::readTuneFile(const Args& args) { double cost{}; if (sscanf(line.c_str(), "%lf %31s", &cost, specBuf) < 2) { log("tune.txt line '%s' ignored\n", line.c_str()); + continue; // otherwise specBuf below is uninitialised } FFTConfig const fft{specBuf}; assert(cost >= prevCost && fft.maxExp() > prevMaxExp); From d79f2a2b5fc1e2a3f297011b1b42722f5d9969e3 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:52:21 -0600 Subject: [PATCH 178/214] FFTConfig::maxExp must use this config's maxBpw, not the shape's best Since 9860255 FFTConfig::maxExp() returned shape.maxExp(), the limit of the shape's best variant, ignoring the config's own variant and carry width. FFTConfig::maxBpw() already accounts for both (variant lookup/interpolation and the CARRY_32 clamp). With the shape-wide value, bestFit() accepts a tune.txt row for an exponent past that row's real bits-per-word limit, TuneEntry's cost/maxExp frontier collapses to one variant per shape, and tune.cpp always skips the variant_M=1 / CARRY_32 candidates because their maxExp no longer differs from the base variant's. Restore u64(maxBpw() * shape.size()). Found by an AI code audit (Claude, Anthropic's Claude Code) of FFT selection; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/FFTConfig.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FFTConfig.h b/src/FFTConfig.h index fd8a471e..1fac5f86 100644 --- a/src/FFTConfig.h +++ b/src/FFTConfig.h @@ -94,7 +94,7 @@ struct FFTConfig { [[nodiscard]] std::string spec() const; [[nodiscard]] u32 size() const { return shape.size(); } - [[nodiscard]] u64 maxExp() const { return shape.maxExp(); } + [[nodiscard]] u64 maxExp() const { return u64(maxBpw() * shape.size()); } // this config's variant and carry, not the shape's best [[nodiscard]] float minBpw() const { return shape.minBpw(); } [[nodiscard]] float maxBpw() const; From 1c96a99341061ca808f1cbeb6b62c1c2b3159578 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 10:08:27 -0600 Subject: [PATCH 179/214] Default INPLACE=1 on nVidia GPUs -tune already assumes INPLACE=1 is the right choice on nVidia: tune.cpp sets time_inplace_only = NVIDIAGPU, which forces INPLACE=1 and skips timing the options that only apply when INPLACE=0. But the compiled-in default is INPLACE=0, so a user who has not run -tune, or who runs without a config.txt, gets the layout the tuner never picks on nVidia. Make the default vendor-dependent, the same way PAD already is: - Gpu.cpp: in_place = isNvidiaGpu(id) ? 1 : 0 - base.cl: #define INPLACE 1 when NVIDIAGPU and INPLACE was not given Both sides are needed and must agree. The C++ in_place only sizes the data buffers; the kernels take INPLACE from the -D flags, which are emitted only for keys the user actually passed, so a C++-only change would have sized the buffers for one layout while compiling the kernels for the other. An explicit -use INPLACE=n still wins on either vendor, and -tune is unaffected since it always sets the flag explicitly. Co-Authored-By: Claude Opus 5 --- src/Gpu.cpp | 2 +- src/cl/base.cl | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ba5e3f79..f2da79d3 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -251,7 +251,7 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector Date: Thu, 17 Sep 2026 10:08:52 -0600 Subject: [PATCH 180/214] Validate FFT shape and variant at runtime FFTShape's fallback for unlisted shapes manipulates width/middle/height towards a shape with known bits-per-word. For a too-small shape (reachable through -tune / -info / a size range, which bypass FFTConfig's spec checks) "while (h < 256) { h *= 2; m /= 2; }" drives middle to 0, the recursion re-enters with m == 0 and "while (m < 9) { m *= 2; ... }" never terminates: "-tune -fft 128:2:128" hangs. Apply the same limits FFTConfig applies to a full spec in the FFTShape constructor, and never let the fallback produce a middle below 2. The variant digits were only asserted; in a release build an out-of-range digit (the shipped tune.txt has rows in the old encoding with a 3) indexes past bpw[] in maxBpw() and selects kernel variants that do not exist. Check them and report. Found by an AI code audit (Claude, Anthropic's Claude Code) of FFT selection; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/FFTConfig.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index e1687af6..fec3bc47 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -132,6 +132,13 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : // Un-initialized shape, don't set BPW if (w == 1 && m == 1 && h == 1) { return; } + // Same limits FFTConfig applies to a full spec. Shapes can also arrive here from -tune / -info / size ranges, and a + // too-small one (e.g. 128:2:128) drives middle to 0 in the fallback below and then loops forever. + if ((w != 256 && w != 512 && w != 1024 && w != 4096) || m < 2 || m > 16 || (h != 256 && h != 512 && h != 1024)) { + log("Invalid FFT shape %u:%u:%u (width 256/512/1024/4096, middle 2..16, height 256/512/1024)\n", w, m, h); + throw "Invalid FFT shape"; + } + string const s = spec(); if (auto it = BPW.find(s); it != BPW.end()) { bpw = it->second; @@ -147,7 +154,7 @@ FFTShape::FFTShape(enum FFT_TYPES t, u32 w, u32 m, u32 h) : while (w >= 4*h) { w /= 2; h *= 2; } while (w < h || w < 256 || w == 2048) { w *= 2; h /= 2; } while (h < 256) { h *= 2; m /= 2; } - if (m == 1) m = 2; + if (m < 2) m = 2; // Make up some defaults (should only happen for experimental FFT types (t >= 52) if (w == orig_w && m == orig_m && h == orig_h) { @@ -266,9 +273,12 @@ FFTConfig::FFTConfig(FFTShape shape, u32 variant, enum CARRY_KIND carry) : variant{variant}, carry{carry} { - assert(variant_W(variant) < N_VARIANT_W); - assert(variant_M(variant) < N_VARIANT_M); - assert(variant_H(variant) < N_VARIANT_H); + // Checked at runtime, not only asserted: an out-of-range digit indexes past bpw[] in maxBpw() and selects kernel + // variants that do not exist (the shipped tune.txt predates this encoding and has such rows). + if (variant_W(variant) >= N_VARIANT_W || variant_M(variant) >= N_VARIANT_M || variant_H(variant) >= N_VARIANT_H) { + log("Invalid FFT variant %u for %s (digits must be < %u%u%u)\n", variant, shape.spec().c_str(), N_VARIANT_W, N_VARIANT_M, N_VARIANT_H); + throw "Invalid FFT variant"; + } if (shape.fft_type == FFT64) FFT_FP64 = true, FFT_FP32 = false, NTT_GF31 = false, NTT_GF61 = false, WordSize = 4; else if (shape.fft_type == FFT3161) FFT_FP64 = false, FFT_FP32 = false, NTT_GF31 = true, NTT_GF61 = true, WordSize = 8; From d3d440ba6d42c88d0baae0ef2d7f009481c95863 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 10:08:52 -0600 Subject: [PATCH 181/214] Honour the FFT type for size-only -fft specs and prefixed size ranges A bare size ("-fft 8M") took FFTShape::multiSpec(spec).front(). multiSpec resolves a size through allShapes(), which enumerates every FFT type and sorts by size, width and height only, so for 1M/2M/4M/8M the first entry is an FP32+GF61 (or FP32+GF31+GF61) NTT and the FP64 run the user asked for silently became an integer-NTT run. The type prefix ("-fft 1:8M") was parsed and then discarded on that path; worse, the size range was split from the original spec, so "1:8M" was read as the size 1. In FFTConfig, filter the size's shapes to the requested type (FP64 unless a prefix is given) and report if none exists; in multiSpec, parse the range from the part after the prefix and, when a prefix is present, keep only that type. An unprefixed range in -tune still means every type, as before. Found by an AI code audit (Claude, Anthropic's Claude Code) of FFT selection; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/FFTConfig.cpp | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/FFTConfig.cpp b/src/FFTConfig.cpp index e1687af6..02dda39b 100644 --- a/src/FFTConfig.cpp +++ b/src/FFTConfig.cpp @@ -52,10 +52,12 @@ vector FFTShape::multiSpec(const string& iniSpec) { for (const string &spec : split(iniSpec, ',')) { enum FFT_TYPES fft_type = FFT64; + bool hasTypePrefix = false; auto parts = split(spec, ':'); if (parseInt(parts[0]) < 60) { // Look for a prefix specifying the FFT type fft_type = (enum FFT_TYPES) parseInt(parts[0]); parts = vector(next(parts.begin()), parts.end()); + hasTypePrefix = true; } assert(parts.size() <= 3); if (parts.size() == 3) { @@ -67,11 +69,14 @@ vector FFTShape::multiSpec(const string& iniSpec) { } assert(parts.size() == 1); - parts = split(spec, '-'); - assert(!parts.empty() && parts.size() <= 2); - u32 const sizeFrom = parseInt(parts[0]); - u32 const sizeTo = parts.size() == 2 ? parseInt(parts[1]) : sizeFrom; + // Parse the size range from the part after the type prefix (splitting the whole spec would read "1:8M" as 1). + auto range = split(parts[0], '-'); + assert(!range.empty() && range.size() <= 2); + u32 const sizeFrom = parseInt(range[0]); + u32 const sizeTo = range.size() == 2 ? parseInt(range[1]) : sizeFrom; auto shapes = allShapes(sizeFrom, sizeTo); + // allShapes() enumerates every FFT type; an explicit prefix asks for one of them. + if (hasTypePrefix) { std::erase_if(shapes, [fft_type](const FFTShape& sh) { return sh.fft_type != fft_type; }); } if (shapes.empty()) { log("Could not find a FFT config for '%s'\n", spec.c_str()); throw "Invalid FFT spec"; @@ -247,7 +252,15 @@ FFTConfig::FFTConfig(const string& spec) { } if (v.size() == 1) { - *this = {FFTShape::multiSpec(spec).front(), LAST_VARIANT, CARRY_AUTO}; + // A bare size ("8M") means an FFT of that size of the requested type -- FP64 unless a prefix says otherwise. + // multiSpec() returns every type for an unprefixed size, sorted without regard to type, so pick ours explicitly. + auto shapes = FFTShape::multiSpec(spec); + std::erase_if(shapes, [fft_type](const FFTShape& sh) { return sh.fft_type != fft_type; }); + if (shapes.empty()) { + log("No FFT of type %d with size '%s'\n", int(fft_type), v[0].c_str()); + throw "Invalid FFT spec"; + } + *this = {shapes.front(), LAST_VARIANT, CARRY_AUTO}; } else if (v.size() == 3) { *this = {FFTShape{fft_type, v[0], v[1], v[2]}, LAST_VARIANT, CARRY_AUTO}; } else if (v.size() == 4) { From 74935f50d12ec24a4b0bcabb721631cefb9e9498 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 10:11:33 -0600 Subject: [PATCH 182/214] Do not use CUDA graphs under -profile use_graphs was enabled whenever the device supports graphs and GRAPHS=1 (the default), regardless of -profile. With profiling on, the four bottom-half kernels are launched once under stream capture -- their start/end events are recorded into the capturing stream and never execute -- and every later squaring is a cuGraphLaunch that produces no events at all. The profile therefore lists fftMidIn / tailSquare / fftMidOut / carryFused as about one call of ~0 ns while they are most of the runtime, which defeats the purpose of -profile on the CUDA build. Disable graphs while profiling. This trades away the graph launch overhead in the profiled numbers, which is the usual trade-off for per-kernel timing. Not run here (no CUDA hardware); the change is one condition. Found by an AI code audit (Claude, Anthropic's Claude Code) of the CUDA paths; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Gpu.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ba5e3f79..9039388c 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -1004,8 +1004,10 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo auxQueues.push_back(Queue{*shared.context, args.profile, true}); } - // Set flag indicating we're going to use CUDA graphs - use_graphs = graph_square[0].isSupported(shared.context->deviceId()) && args.value("GRAPHS", 1); + // Set flag indicating we're going to use CUDA graphs. Not under -profile: a graph replays the four bottom-half + // kernels without per-kernel events, and the events recorded while capturing the graph never execute, so the + // profile would show those kernels -- most of an iteration -- as one call of ~0 ns. + use_graphs = graph_square[0].isSupported(shared.context->deviceId()) && args.value("GRAPHS", 1) && !args.profile; // Set L1 cache configuration. Really we should only do this once rather than once per worker. // However, the current way PRPLL is organized would then make this option hard to tune. From 52429286bcb9cc7316b5d3d5dc73af15dcb2d126 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 10:23:37 -0600 Subject: [PATCH 183/214] Checkpoint CERT assignments isCERT ran its squarings from k = 0 every time: there was no savefile, so stopping a certification meant redoing all of it, and nothing on disk told AutoPrimeNet how far along it was. PRP and LL already have Saver with a per-kind savefile ("-/-."); give CERT the same. CERTState is LLState plus the assignment's squaring count, written with a header "OWL CERT 1 N=1*2^E-1 k=K squarings=S time=T" followed by the residue with the usual checksum. isCERT loads the most recent checkpoint, starts from the M.cert file only when there is none (or when the checkpoint's squaring count does not match the assignment), saves at every log point and on a stop request, and removes its checkpoints when the certification completes. The log cadence follows -log as LL does, instead of a fixed 100000, so checkpoints land at the same points. Checked on an Intel UHD iGPU with a random 5000011 start value and 60000 squarings: a run interrupted with SIGINT after the 20000 checkpoint left "cert-5000011/5000011-000020000.cert" and a checkpoint at the stop; the next run logged "CERT loaded @ ", finished, wrote the result, and removed the checkpoints, the .cert start file and the worktodo line. The resumed run's residues at 40000 and 60000 and its SHA-3 match an uninterrupted run on the same start value. Requested by Teal on gwoltman/gpuowl#23; implemented by Claude (Anthropic's Claude Code) working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Gpu.cpp | 23 +++++++++++++++++++---- src/Saver.cpp | 28 ++++++++++++++++++++++++++++ src/Saver.h | 12 ++++++++++++ 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index ba5e3f79..390811ab 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -2833,22 +2833,34 @@ array Gpu::isCERT(const Task& task) { // AutoPrimenet.py does not add the cert entry to worktodo.txt until it has successfully downloaded the .cert file. - { // Enclosing this code in braces ensures the file will be closed by the File destructor. The later file deletion requires the file be closed in Windows. + // Resume from a checkpoint if there is one for this assignment; otherwise start from the .cert file. + Saver saver{E, 1000, args.nSavefiles}; + CERTState state = saver.load(); + if (!state.data.empty() && state.squarings != task.squarings) { + log("CERT checkpoint is for %" PRIu64 " squarings, assignment says %u; starting over\n", state.squarings, task.squarings); + state = CERTState{.exponent=E, .k=0, .squarings=0, .data={}, .elapsed=0}; + } + + if (!state.data.empty()) { + writeIn(bufData, state.data); + log("CERT loaded @ %" PRIu64 "\n", state.k); + } else { // Enclosing this code in braces ensures the file will be closed by the File destructor. The later file deletion requires the file be closed in Windows. File fi = File::openReadThrow(fname); u32 const nBytes = u32((E - 1) / 8 + 1); Words const B = fi.readBytesLE(nBytes); writeIn(bufData, B); } + double const elapsedBefore = state.elapsed; Timer elapsedTimer; elapsedTimer.reset(); - u32 const startK = 0; + u32 const startK = u32(state.k); IterationTimer iterationTimer{startK}; - u32 k = 0; + u32 k = startK; u32 const kEnd = task.squarings; enum LEAD_TYPE leadIn = LEAD_NONE; @@ -2861,7 +2873,7 @@ array Gpu::isCERT(const Task& task) { log("Stopping, please wait..\n"); } - bool const doLog = (k % 100'000 == 0) || doStop; + bool const doLog = (k % args.logStep == 0) || doStop; // same cadence as LL; every log point is also a checkpoint enum LEAD_TYPE const leadOut = doLog || useLongCarry ? LEAD_NONE : LEAD_WIDTH; squareCERT(bufData, leadIn, leadOut); @@ -2879,9 +2891,12 @@ array Gpu::isCERT(const Task& task) { if (k >= kEnd) { fs::remove (fname); + Saver::clear(E); return std::move(SHA3{}.update(data.data(), u32((E-1)/8+1))).finish(); } + saver.save({.exponent=E, .k=k, .squarings=kEnd, .data=std::move(data), .elapsed=elapsedBefore + elapsedTimer.at()}); + if (doStop) { throw "stop requested"; } } } diff --git a/src/Saver.cpp b/src/Saver.cpp index dfb2b314..c7794b84 100644 --- a/src/Saver.cpp +++ b/src/Saver.cpp @@ -29,6 +29,7 @@ static constexpr const char *LL_v1 = "OWL LL 1 E=%" PRIu64 " k=%" PRIu64 " CRC=% // Anticipated next version. // Push version number to sync it with PRP. static constexpr const char *LL_v13 = "OWL LL 13 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " time=%lf\n"; +static constexpr const char *CERT_v1 = "OWL CERT 1 N=1*2^%" PRIu64 "-1 k=%" PRIu64 " squarings=%" PRIu64 " time=%lf\n"; struct BadHeaderError { string name; }; @@ -136,6 +137,28 @@ void writeState(const File& fo, const PRPState& state) { fo.writeChecked(state.check); } +CERTState readState([[maybe_unused]] const CERTState& dummy, File fi) { + u64 exponent{}, k{}, squarings{}; + double elapsed{}; + + string const header = fi.readLine(); + + if (sscanf(header.c_str(), CERT_v1, &exponent, &k, &squarings, &elapsed) == 4) { + return {.exponent=exponent, .k=k, .squarings=squarings, .data=fi.readChecked(nWords(exponent)), .elapsed=elapsed}; + } + + log("Loading CERT from '%s': bad header '%s'\n", fi.name.c_str(), header.c_str()); + throw BadHeaderError{fi.name}; +} + +void writeState(const File& fo, const CERTState& state) { + assert(state.data.size() == nWords(state.exponent)); + if (fo.printf(CERT_v1, state.exponent, state.k, state.squarings, state.elapsed) <= 0) { + throw WriteError{fo.name}; + } + fo.writeChecked(state.data); +} + void writeState(const File& fo, const LLState& state) { assert(state.data.size() == nWords(state.exponent)); if (fo.printf(LL_v13, state.exponent, state.k, state.elapsed) <= 0) { @@ -167,6 +190,10 @@ template<> LLState Saver::initState() { return {.exponent=exponent, .k=0, .data=makeWords(exponent, 4), .elapsed=0}; } +template<> CERTState Saver::initState() { + return {.exponent=exponent, .k=0, .squarings=0, .data={}, .elapsed=0}; // no checkpoint: caller starts from the .cert file +} + // ---- Saver ---- @@ -297,3 +324,4 @@ void Saver::dropMostRecent() { template class Saver; template class Saver; +template class Saver; diff --git a/src/Saver.h b/src/Saver.h index 07fc44a6..359552fc 100644 --- a/src/Saver.h +++ b/src/Saver.h @@ -31,6 +31,18 @@ struct LLState { double elapsed{}; }; +// A CERT (certification) in progress: the current residue after k of `squarings` squarings. An empty `data` +// means "no checkpoint": start from the M.cert file. +struct CERTState { + static const constexpr char* KIND = "cert"; + + u64 exponent; + u64 k; + u64 squarings; + vector data; + double elapsed{}; +}; + template class Saver { u64 exponent; From 50a2dafda130d2fe3a45881ac0d18be870c9d4bf Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:38:52 -0600 Subject: [PATCH 184/214] Serialize the global-worktodo claim across worker threads getWork() claims a task from the shared worktodo.txt with a read-size / rewrite / compare-size protocol that the comment describes as protection against other processes. Within one process nothing serializes it: with -workers N all workers start together, find their local worktodo-N.txt empty, and hit the global file at once. Two of them read the same size, pick the same (deterministically chosen) task, each append it to their own local file, each rewrite the global file without it, each pass the size check because neither has renamed yet, and both rename. The global file ends up consistent, but the task is now in two local files: two workers run the same exponent in the same / directory, writing the same savefile paths from two threads, and produce two results for one assignment. Take a process-wide mutex around the claim. The cross-process heuristic is unchanged. Not exercised here (needs -workers > 1 with a pool directory on a GPU that can run two instances); the change is a static std::mutex and a lock_guard. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Worktodo.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index eae976ea..710572b0 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -14,6 +14,7 @@ #include #include #include +#include namespace { @@ -146,6 +147,11 @@ optional getWork(Args& args, i32 instance) { 8. start again (from step 1) */ + // The size heuristic below guards against other processes. Within this process the workers start together and + // would all read the same file and pick the same task, so serialize the claim itself. + static std::mutex claimMutex; + std::lock_guard const claimLock(claimMutex); + for (int retry = 0; retry < 2; ++retry) { u64 const initialSize = fileSize(worktodo); if (!initialSize) { return {}; } From 8bce298d977b0d998251fcf8e367e0709e40ba6a Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 09:47:47 -0600 Subject: [PATCH 185/214] Ignore worktodo lines whose exponent is not prime Task::execute replaces a composite exponent with the previous prime and only warns. That is meant for "-prp " timing runs, but it also applied to worktodo assignments: a PRP=/Test= line with a composite exponent ran a different exponent and reported the result under the line's AID. (The proof VERIFY path hits assert(proof.E == exponent) for the same reason.) Reject such lines in Worktodo::parse with a message, the way malformed lines are, so the retarget only ever applies to command-line exponents. Primes is a small sieve and cheap to construct per line. Before/after with PRP=,1,2,1234567,-1 followed by a Test= line: the composite line ran as M1234547; now it is ignored and the Test= line runs. Found by an AI code audit (Claude, Anthropic's Claude Code) of the persistence layer; fixed by Claude working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/Worktodo.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Worktodo.cpp b/src/Worktodo.cpp index eae976ea..ddf3fd86 100644 --- a/src/Worktodo.cpp +++ b/src/Worktodo.cpp @@ -7,6 +7,7 @@ #include "common.h" #include "Args.h" #include "fs.h" +#include "Primes.h" #include #include @@ -67,6 +68,13 @@ std::optional parse(const std::string& line) { u64 exp{}; auto [ptr, _] = from_chars(s.c_str(), end, exp, 10); if (ptr != end) { exp = 0; } + // Task::execute silently retargets a composite exponent to the previous prime. That is a convenience for + // "-prp " timing runs; an assignment line with a composite exponent is a mistake, and running a + // different exponent under its AID would report the wrong result. Ignore the line instead. + if (exp > 1000 && !Primes{}.isPrime(exp)) { + log("worktodo.txt line ignored, exponent %" PRIu64 " is not prime: \"%s\"\n", exp, rstripNewline(line).c_str()); + return {}; + } if (exp > 1000) { return {{.kind=isPRP ? Task::PRP : Task::LL, .exponent=exp, .AID=AID, .line=line, .squarings=0}}; } } if (isCERT) { From dfecaf7b4c792b382482ffa883f4c1ddbf658693 Mon Sep 17 00:00:00 2001 From: george Date: Thu, 17 Sep 2026 19:05:11 +0000 Subject: [PATCH 186/214] Changed AI fix for disabling GRAPHS for -time. You'll get a warning message now (MULTI_Q too). --- src/Gpu.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 9039388c..486420f5 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -329,6 +329,21 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector& extraConf, boo auxQueues.push_back(Queue{*shared.context, args.profile, true}); } - // Set flag indicating we're going to use CUDA graphs. Not under -profile: a graph replays the four bottom-half - // kernels without per-kernel events, and the events recorded while capturing the graph never execute, so the - // profile would show those kernels -- most of an iteration -- as one call of ~0 ns. - use_graphs = graph_square[0].isSupported(shared.context->deviceId()) && args.value("GRAPHS", 1) && !args.profile; + // Set flag indicating we're going to use CUDA graphs. + use_graphs = graph_square[0].isSupported(shared.context->deviceId()) && args.value("GRAPHS", 1); // Set L1 cache configuration. Really we should only do this once rather than once per worker. // However, the current way PRPLL is organized would then make this option hard to tune. From 4b2e1817f6ff484935bf5b763ea65d969f0791db Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 20:00:50 -0600 Subject: [PATCH 187/214] Fall back to OpenCL C 3.0 where 2.0 is unavailable; pass no compile options to the linker The kernels were always compiled with -cl-std=CL2.0, and clLinkProgram was handed "-cl-finite-math-only". Every GPU driver we run on accepts both, but an OpenCL 3.0 implementation is not required to offer OpenCL C 2.0, and POCL (and likely Mesa rusticl) does not: it rejects the option outright, so no kernel ever compiled there -- Build option -cl-std specified OpenCL C version 2.0, but device cpu doesn't support that OpenCL C version. -- and its linker rejects a compile-only option (INVALID_LINKER_OPTIONS), which the GPU vendors silently ignore. Probe once at KernelCompiler construction: compile an empty kernel with -cl-std=CL2.0 and, only if that is rejected and -cl-std=CL3.0 is accepted, use CL3.0 for all kernels and say so. OpenCL C 3.0 carries what the kernels use from 2.0 (generic address space, memory-order atomics) as optional features, which such devices report. Devices that accept CL2.0 -- all the GPUs -- see no change at all. Pass no options to clLinkProgram; the finite-math option stays on the compile step, where it acts. Checked: on an Intel UHD iGPU the probe keeps CL2.0 and a 256:2:256 run gives the same residue as before (2213bb0fe8602a74 at 2000). On a POCL CPU device the fallback triggers, every kernel compiles, and 10000 iterations pass the Gerbicz check with residues identical to the iGPU's (2213bb0fe8602a74, 49e39cc3b65f3399). Under -cl-std=CL3.0 the GF61 tail kernels that hit OUT_OF_RESOURCES on the Intel iGPU under CL2.0 also compile and run there. Found while adding a kernel-compile check to CI (#65); implemented by Claude (Anthropic's Claude Code) working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/KernelCompiler.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/KernelCompiler.cpp b/src/KernelCompiler.cpp index ab3144d5..d384a6f1 100644 --- a/src/KernelCompiler.cpp +++ b/src/KernelCompiler.cpp @@ -27,17 +27,37 @@ static_assert(sizeof(Program) == sizeof(cl_program)); // * -fno-bin-llvmir // * various: -fno-bin-source -fno-bin-amdil +// Does the device's compiler accept this -cl-std? Compiles an empty kernel with just that option. +static bool acceptsClStd(cl_context context, cl_device_id deviceId, const string& clStd) { + Program probe = loadSource(context, "kernel void probe() {}\n"); + if (!probe) { return false; } + string const opts = "-cl-std=" + clStd; + return clCompileProgram(probe.get(), 1, &deviceId, opts.c_str(), 0, nullptr, nullptr, nullptr, nullptr) == CL_SUCCESS; +} + KernelCompiler::KernelCompiler(const Args& args, const Context* context, const string& clArgs) : cacheDir{args.cacheDir.string()}, context{context->get()}, - linkArgs{"-cl-finite-math-only " }, - baseArgs{linkArgs + "-cl-std=CL2.0 " + clArgs}, + linkArgs{}, // no compile-only options here: clLinkProgram accepts only linker options (POCL enforces it) + baseArgs{}, dump{args.dump}, useCache{args.useCache}, verbose{args.verbose}, deviceId{context->deviceId()} { + // Every GPU driver we run on accepts -cl-std=CL2.0. Some OpenCL 3.0 implementations (POCL; likely Mesa rusticl) + // offer no OpenCL C 2.0 at all and reject the option, but do offer OpenCL C 3.0, whose optional features cover what + // the kernels use from 2.0 (generic address space, memory-order atomics). Probe once and fall back. + string clStd = "CL2.0"; +#ifndef CUDA_BACKEND + if (!acceptsClStd(context->get(), deviceId, "CL2.0") && acceptsClStd(context->get(), deviceId, "CL3.0")) { + clStd = "CL3.0"; + log("OpenCL C 2.0 is not available on this device; compiling the kernels as OpenCL C 3.0\n"); + } +#endif + baseArgs = "-cl-finite-math-only -cl-std=" + clStd + ' ' + clArgs; + string const hw = getDriverVersion(deviceId) + ':' + getDeviceName(deviceId); if (args.verbose) { log("OpenCL: %s, args %s\n", hw.c_str(), baseArgs.c_str()); } From 36bf699e933daed5084a818249f1942459ffedad Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Thu, 17 Sep 2026 19:23:08 -0600 Subject: [PATCH 188/214] CI: compile the kernels and pass a Gerbicz check on a POCL CPU device The Linux OpenCL jobs install POCL and build the host binary, but only run "prpll -h". The kernels in src/cl are compiled at first use, on the device, so no .cl file was ever compiled by CI; that is how 436bdb4's #error in base.cl, which broke every non-NVIDIA OpenCL device, went unnoticed until PR #35. Add one job, first in the workflow, on a single ubuntu-latest runner: build the release binary, run 400 iterations of a 256:2:256 FP64 test on the POCL CPU device and require the Gerbicz check at the end to report OK. This compiles the FP64 kernel set for a non-NVIDIA OpenCL target on every push and pull request, once -- the cost is kernel compilation, so running it in every matrix cell would only multiply that. The integer-NTT types are left out (far too slow on a CPU device), and aarch64 POCL is not tried yet. Also cancel a superseded run of the same branch or PR (concurrency group) rather than pay for both. Needs the OpenCL C 3.0 fallback (POCL offers no OpenCL C 2.0), on which this is stacked. POCL's compiler is clang, so this catches #error, syntax and OpenCL C semantic breakage -- not a vendor compiler's idiosyncrasies. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d038df19..05e187d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,36 @@ on: schedule: - cron: '0 0 1 * *' +# A new push to the same branch or PR cancels the previous run instead of paying for both. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: + Kernel-Smoke: + name: Kernel compile + Gerbicz check (POCL CPU) + # The kernels in src/cl are compiled at first use, on the device, so the build jobs below never see them. + # This compiles the FP64 kernel set on POCL's CPU device -- a non-NVIDIA OpenCL target -- and ends in a + # Gerbicz check. One runner, on purpose: the cost is kernel compilation, and it need only happen once. + # The integer-NTT types are left out (far too slow on a CPU device); aarch64 POCL is not tried yet. + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v7 + - name: Install + run: | + sudo apt-get update -y + sudo apt-get install -y ocl-icd-opencl-dev pocl-opencl-icd + - name: Build + run: make -O -j "$(nproc)" + - name: Compile the kernels and pass a Gerbicz check + run: | + cd build-release + ./prpll -h + timeout 20m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400 + cat gpuowl-0.log + grep -q "OK *400 " gpuowl-0.log + Linux-OpenCL: name: Linux OpenCL From 5cf29bf70fab6cb83f357c0de22b8123fe3d734d Mon Sep 17 00:00:00 2001 From: george Date: Sat, 19 Sep 2026 01:12:31 +0000 Subject: [PATCH 189/214] CUDA shim: don't trust CUBIN for --maxrregcount, use the PTX path instead NVRTC accepts --maxrregcount=N without complaint but its CUBIN's actual register usage ignores it (verified: requesting 24 regs for carryFused still yielded 64). Since the CUBIN-preferring change in 57947a1, every kernel that asks for a register cap (CARRYFUSED, TAIL, MIDIN, MIDOUT, and any -use REG*= override) silently got whatever registers ptxas felt like using instead. The PTX path already carries a spliced-in .maxnreg directive that the driver's JIT does honor correctly, so drop the CUBIN specifically for capped kernels and fall back to that. Kernels with no cap requested (the majority) keep using the CUBIN fast path. Verified on a Titan V, FFT 512:16:512:202: tailSquare/carryFused now hit their intended default caps (88/80) and any -use REGTS64=/REGCF64= override is honored exactly, instead of being ignored. Co-Authored-By: Claude Sonnet 5 --- src/cuda/clwrap_cuda.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/cuda/clwrap_cuda.cpp b/src/cuda/clwrap_cuda.cpp index 585a49f5..6006278b 100644 --- a/src/cuda/clwrap_cuda.cpp +++ b/src/cuda/clwrap_cuda.cpp @@ -472,9 +472,13 @@ int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id return CL_COMPILE_PROGRAM_FAILURE; } - // NVRTC applies --maxrregcount when it runs ptxas itself, i.e. in the CUBIN. - // PTX carries no register cap and the driver JIT never sees the option, so - // the PTX fallback gets a .maxnreg directive spliced in ahead of every entry. + // --maxrregcount is supposed to be applied by NVRTC's own ptxas when it builds the CUBIN, with the PTX + // fallback getting a spliced-in .maxnreg directive for the driver JIT path instead. In practice (verified + // against this toolkit/driver: CUDA 13.0, NVRTC accepts --maxrregcount without warning but the resulting + // CUBIN's register usage is unaffected by it -- e.g. requesting 24 regs for carryFused still yields 64). + // The .maxnreg-spliced PTX + driver JIT does honor the cap correctly. So for any kernel that asked for a + // register cap, drop the (silently non-compliant) CUBIN and force the PTX path, which is known to work. + // Kernels with no cap requested keep using the CUBIN fast path this shim was added for. if (maxregcount) { string const maxntidPattern = ".maxntid "; @@ -485,6 +489,7 @@ int clCompileProgram(cl_program prog, unsigned /*nDevices*/, const cl_device_id prog->ptx.insert(pos, maxnregPattern); startpos = pos + 20; } + prog->cubin.clear(); } return CL_SUCCESS; From 34ac865a744b50b2eba68d287a11989a936d92bf Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sat, 19 Sep 2026 10:39:04 -0600 Subject: [PATCH 190/214] carryFused: qualify the ready-flag atomics as global MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inter-workgroup carry hand-off casts `global u32*` ready flags to a plain `atomic_uint*`. Without an address-space qualifier that pointee lives in the generic address space, so the cast only compiles where OpenCL C 2.0's generic address space is available. On a device without it — POCL's pthread CPU device on aarch64 (Raspberry Pi 4, Debian 12, POCL 3.1) reports "Generic address space support: No" — clang rejects every one of the 45 casts: carryfused.cl:242:60: casting '__global u32 *' to type '__private atomic_uint *' changes address space of pointer and carryFused fails to compile, so the program cannot run at all. The generic address space is also an optional feature in OpenCL C 3.0, which the kernels fall back to (#66), so the kernels should not depend on it. Cast to `global atomic_uint*` instead. Same code on devices that do have the generic address space: Intel iGPU and POCL 7.2 on x86 still give `OK 400 0d42972a9d02005c`. The CUDA translation strips standalone `global` tokens, so the NVRTC path sees the same source as before. Written by Claude (Anthropic's Claude Code) working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/cl/carryfused.cl | 90 ++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index bd492843..8d857586 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -224,7 +224,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -236,9 +236,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -271,7 +271,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -282,7 +282,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -455,7 +455,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -467,9 +467,9 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -494,7 +494,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -505,7 +505,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -691,7 +691,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -703,9 +703,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -731,7 +731,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -742,7 +742,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -932,7 +932,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -944,9 +944,9 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -972,7 +972,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -983,7 +983,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -1189,7 +1189,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -1201,9 +1201,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -1236,7 +1236,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -1247,7 +1247,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -1472,7 +1472,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -1484,9 +1484,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -1511,7 +1511,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -1522,7 +1522,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -1751,7 +1751,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -1763,9 +1763,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -1790,7 +1790,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -1801,7 +1801,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -2024,7 +2024,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -2036,9 +2036,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -2064,7 +2064,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -2075,7 +2075,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); @@ -2331,7 +2331,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut sync(); // Make sure all lanes have completed the CSSTORE if (lowMe % WAVEFRONT == 0) { u32 pos = gr * (G_W / WAVEFRONT) + lowMe / WAVEFRONT; - atomic_store((atomic_uint *) &ready[pos], 1); + atomic_store((global atomic_uint *) &ready[pos], 1); } #endif } @@ -2343,9 +2343,9 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut if (gr < H / WMUL) { bar(G_W); #if WMUL == 1 - if (lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #else - if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((atomic_uint *) &ready[gr], 1); } + if (me >= (WMUL-1) * G_W && lowMe == 0) { atomic_store((global atomic_uint *) &ready[gr], 1); } #endif } #endif @@ -2370,7 +2370,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // Wait until our carries are ready. The barrier below must be reached by every work-item of the // workgroup, so the spin-wait and the barrier sit outside the "me < G_W" guard. #if OLD_FENCE - if (me == 0) { do { spin(); } while(!atomic_load_explicit((atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } + if (me == 0) { do { spin(); } while(!atomic_load_explicit((global atomic_uint *) &ready[gr - 1], memory_order_relaxed, memory_scope_device)); } bar(); #endif if (me < G_W) { @@ -2381,7 +2381,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut #else u32 pos = (gr - 1) * (G_W / WAVEFRONT) + me / WAVEFRONT; if (me % WAVEFRONT == 0) { - do { spin(); } while(atomic_load_explicit((atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); + do { spin(); } while(atomic_load_explicit((global atomic_uint *) &ready[pos], memory_order_relaxed, memory_scope_device) == 0); } sync(); read_mem_fence(CLK_GLOBAL_MEM_FENCE); From fd8def34a5ace8cba9cac88068b170d3fa97ec82 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sat, 19 Sep 2026 10:42:06 -0600 Subject: [PATCH 191/214] main: exit non-zero when a worker dies on an exception MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gpuWorker() catches whatever its task throws, logs it and returns, and main() then reports success: a run that ends with Compiling 'transpose.cl' error COMPILE_PROGRAM_FAILURE (-15) Exception "Can't compile transpose.cl" Bye exits 0. Only exceptions that escape to main() itself (no device, bad arguments) set the exit code. Launchers, scripts and CI cannot tell such a run from a finished one without parsing the log — the kernel smoke job in CI has to grep for its "OK" line for exactly this reason. Record the failure in a flag from the worker's catch blocks and let main() turn it into exit code 1, unless the exception is one of the clean stops ("stop requested", "help", "version") that main() already treats as success. Nothing else changes: the other workers still run to completion, the log is the same. A forced compile failure now exits 1; a run that completes its iterations (which ends through "stop requested") still exits 0. Written by Claude (Anthropic's Claude Code) working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- src/main.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main.cpp b/src/main.cpp index 7a2f1e8c..74ce6ad7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -16,12 +16,19 @@ #include "Gpu.h" #include "tune.h" +#include #include #include #include #include // #include from GCC-13 onwards +// Set when a worker dies on an exception, so that main() can report the failure through the exit code +// even though the other workers (and the process) carry on to a normal end. +static std::atomic workerFailed{false}; + +static bool isCleanExit(const char* reason); + static void gpuWorker(GpuCommon shared, i32 instance) { // LogContext context{(instance ? shared.args->tailDir() : ""s) + to_string(instance) + ' '}; // log("Starting worker %d\n", instance); @@ -34,10 +41,13 @@ static void gpuWorker(GpuCommon shared, i32 instance) { while (auto task = Worktodo::getTask(*shared.args, instance)) { task->execute(shared, instance); } } catch (const char *mes) { log("Exception \"%s\"\n", mes); + if (!isCleanExit(mes)) { workerFailed = true; } } catch (const string& mes) { log("Exception \"%s\"\n", mes.c_str()); + if (!isCleanExit(mes.c_str())) { workerFailed = true; } } catch (const std::exception& e) { log("Exception %s: %s\n", typeName(e), e.what()); + workerFailed = true; } } @@ -149,6 +159,8 @@ int main(int argc, char **argv) { exitCode = 1; } + if (workerFailed && exitCode == 0) { exitCode = 1; } + log("Bye\n"); return exitCode; } From b2c1022820b97b48f5aaf510a2cbeb0b2fee3262 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sat, 19 Sep 2026 11:00:15 -0600 Subject: [PATCH 192/214] CI: make the kernel smoke job fast, informative, and run it on arm64 too The Kernel-Smoke job from #65 never completed on a runner: POCL's default work-group method vectorizes across work-items, and on these kernels that takes 15-25 minutes on the AVX-512 runner CPUs, so the step ran until the job-level timeout cancelled it -- and printed nothing, because the log dump came after the run. Ubuntu's own POCL (5.0 on 24.04, 6.0 on 26.04) is slow on these kernels even with the vectorizer off, and on the arm runners it aborts with an internal assertion (WorkitemLoops.cc: "Cannot add context restore for a PHI node at the region entry!"). conda-forge's POCL 7.1 computes wrong results. So the job builds POCL 7.2 from source against the distro LLVM 21 on ubuntu-26.04, kept in actions/cache (a cache miss costs about 90 s, a hit a few seconds), selected through OCL_ICD_VENDORS without touching the system, and runs with POCL_WORK_GROUP_METHOD=loops. The whole job takes about a minute on a cache hit; the check step itself 20 s. Two matrix entries, ubuntu-26.04 and ubuntu-26.04-arm: the arm one runs the kernels through LLVM's AArch64 backend and the host code on arm64 at runtime, which the build-only arm jobs never do. POCL 7.2 compiles the kernels on aarch64 without the assertion, both on the runner and on a Raspberry Pi 4 (Debian 12, LLVM 18): OK 400 0d42972a9d02005c in both work-group methods. Review suggestions from #65 (tdulcet): no workflow-level concurrency group (it cancelled the PR's own run), no job-level timeout (a slow apt mirror should not fail the job) -- a step-level timeout-minutes plus `timeout -s KILL` on prpll, which ignores SIGTERM inside an OpenCL compile, bound the cost instead; DEBUG=1 build so the runtime asserts are exercised. The log is dumped before the exit status is checked so a timeout or crash still shows how far the kernels got. One product line: the debug-only assert(iters % 10000 == 0) in Args.cpp becomes iters > 0. Release builds never enforced it, and a short run is exactly what a smoke test is. Written by Claude (Anthropic's Claude Code) working with Mark Rose. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 57 +++++++++++++++++++++++++++++++--------- src/Args.cpp | 2 +- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05e187d8..b174b00f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,34 +6,65 @@ on: schedule: - cron: '0 0 1 * *' -# A new push to the same branch or PR cancels the previous run instead of paying for both. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - jobs: Kernel-Smoke: - name: Kernel compile + Gerbicz check (POCL CPU) + name: Kernel compile + Gerbicz check (POCL CPU, ${{ matrix.os }}) # The kernels in src/cl are compiled at first use, on the device, so the build jobs below never see them. # This compiles the FP64 kernel set on POCL's CPU device -- a non-NVIDIA OpenCL target -- and ends in a # Gerbicz check. One runner, on purpose: the cost is kernel compilation, and it need only happen once. - # The integer-NTT types are left out (far too slow on a CPU device); aarch64 POCL is not tried yet. - runs-on: ubuntu-latest - timeout-minutes: 25 + # Debug build so the runtime asserts are exercised too. The integer-NTT types are left out (far too slow + # on a CPU device); aarch64 POCL is not tried yet. + # + # POCL_WORK_GROUP_METHOD=loops: with the default (vectorizing) work-group method POCL spends 15-25 minutes + # compiling these kernels on the AVX-512 runner CPUs; with "loops" and POCL 7.2 the whole check takes about + # 10 seconds. Ubuntu's own POCL packages (5.0 on 24.04, 6.0 on 26.04) are slow even with "loops" and the + # conda-forge 7.1 build computes wrong results, so POCL 7.2 is built from source against the distro LLVM + # and cached; a cache hit costs seconds. + # The arm64 entry runs the same kernels through LLVM's AArch64 backend and the host code on arm64, + # which the build-only arm jobs above never do. Same commands: 26.04-arm has the same LLVM 21. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-26.04, ubuntu-26.04-arm] + env: + POCL_WORK_GROUP_METHOD: loops + POCL_VERSION: "7.2" steps: - uses: actions/checkout@v7 - name: Install + # The LLVM packages are needed at run time too (the cached libpocl links against them), not only to build POCL. run: | sudo apt-get update -y - sudo apt-get install -y ocl-icd-opencl-dev pocl-opencl-icd + sudo apt-get install -y ocl-icd-opencl-dev cmake ninja-build llvm-21-dev libclang-21-dev libclang-cpp21-dev clang-21 libhwloc-dev + - name: Restore POCL + id: pocl-cache + uses: actions/cache@v4 + with: + path: ~/pocl + key: pocl-${{ env.POCL_VERSION }}-${{ runner.os }}-${{ runner.arch }}-${{ matrix.os }}-llvm21 + - name: Build POCL + if: steps.pocl-cache.outputs.cache-hit != 'true' + run: | + curl -Ls "https://api.github.com/repos/pocl/pocl/tarball/v$POCL_VERSION" | tar -xz + cmake -G Ninja -S pocl-pocl-* -B pocl-build -DCMAKE_BUILD_TYPE=Release -DWITH_LLVM_CONFIG=/usr/bin/llvm-config-21 \ + -DENABLE_ICD=ON -DENABLE_TESTS=OFF -DENABLE_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX="$HOME/pocl" -DPOCL_INSTALL_ICD_VENDORDIR="$HOME/pocl/etc/OpenCL/vendors" + ninja -C pocl-build install + - name: Use POCL + run: echo "OCL_ICD_VENDORS=$HOME/pocl/etc/OpenCL/vendors" >> "$GITHUB_ENV" - name: Build - run: make -O -j "$(nproc)" + run: make DEBUG=1 -O -j "$(nproc)" - name: Compile the kernels and pass a Gerbicz check + # Step-level bound (not job-level, so a slow apt mirror cannot fail the job): a correct run takes + # well under a minute. prpll ignores SIGTERM while inside an OpenCL compile, hence -s KILL. + timeout-minutes: 12 run: | - cd build-release + cd build-debug ./prpll -h - timeout 20m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400 + # Dump the log whatever happened, so a timeout or crash still shows how far the kernels got. + timeout -s KILL 10m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400; rc=$? cat gpuowl-0.log + test "$rc" -eq 0 grep -q "OK *400 " gpuowl-0.log Linux-OpenCL: diff --git a/src/Args.cpp b/src/Args.cpp index 29e83aff..cec0ef76 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -382,7 +382,7 @@ void Args::parse(const string& line) { u32 multiple = (s.back() == 'G') ? (1u << 30) : (1u << 20); maxAlloc = size_t(stod(s) * multiple + .5); } - else if (key == "-iters") { iters = stoi(s); assert(iters && (iters % 10000 == 0)); } + else if (key == "-iters") { iters = stoi(s); assert(iters > 0); } // any positive count; release never enforced the old multiple-of-10000 rule else if (key == "-prp" || key == "-PRP") { prpExp = stoll(s); } else if (key == "-ll" || key == "-LL") { llExp = stoll(s); } else if (key == "-smallest") { smallest = true; } From ae8aa0c2c1d5202167610246060939d9af9da1b7 Mon Sep 17 00:00:00 2001 From: george Date: Sun, 20 Sep 2026 03:47:14 +0000 Subject: [PATCH 193/214] tailMul: add TAIL_KERNELS support (double-wide / split-kernel), matching tailSquare tailMul never got the TAIL_KERNELS treatment tailSquare has: it was always the old single-wide, single-kernel implementation, which is why it was burning ~160-250 registers regardless of GPU. Since nearly all GPUs prefer the double-wide layout (TAIL_KERNELS=2), this brings tailMul's register usage down to parity with tailSquare (e.g. 162 -> 96 registers on one FP64 shape tested). Adds, for FP64, FP32, GF31 and GF61: - tailMulZero / tailMulLowZero kernels (handle lines 0 and H/2 separately, for split-kernel mode), mirroring tailSquareZero. - A double-wide tailMul variant (G_H*2 threads, one line per half-workgroup instead of two lines per thread) with a pairMul2_special helper for the self-paired line-0/H/2 case, mirroring tailSquare's double-wide kernel. Gpu.cpp/Gpu.h wire up the new Zero kernels and give tailMul/tailMulLow the same 4-way work-size formula tailSquare already uses for TAIL_KERNELS in {0,1,2,3}. Verified by temporarily substituting tailMul(x,x) for tailSquare(x) in the real PRP squaring path (multiplying a value by itself is mathematically squaring) and comparing residues after 10000 real iterations: all four TAIL_KERNELS settings reproduce the tailSquare baseline exactly, for FP64 (1K:13:256), FP32 (2:256:2:256), and GF31/GF61 (1:256:2:256 hybrid, and each in isolation via 3:256:2:256 / 52:256:2:256). This caught a real bug in the first pass -- the GF31/GF61 zero-kernels were missing a SINGLE_WIDE-specific trig-table index used by tailSquareZeroGF31/61, which only broke TAIL_KERNELS=1 -- since fixed and reverified. Co-Authored-By: Claude Sonnet 5 --- src/Gpu.cpp | 66 ++++- src/Gpu.h | 6 + src/cl/tailmul.cl | 667 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 714 insertions(+), 25 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 51ed5283..3464d75a 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -842,9 +842,19 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, kernelDefines(KFP) + numCudaRegisters(TAIL)), // Single-wide tailSquare with one kernel - K(ktailMul, "tailmul.cl", "tailMul", hN / nH / 2, kernelDefines(KFP)), - K(ktailMulLow, "tailmul.cl", "tailMul", hN / nH / 2, kernelDefines(KFP) + "-DMUL_LOW=1"), + hN / nH / 2, kernelDefines(KFP) + numCudaRegisters(TAIL)), // Single-wide tailSquare with one kernel + K(ktailMulZero, "tailmul.cl", "tailMulZero", SMALL_H / nH * 2, kernelDefines(KFP)), + K(ktailMulLowZero, "tailmul.cl", "tailMulZero", SMALL_H / nH * 2, kernelDefines(KFP) + "-DMUL_LOW=1"), + K(ktailMul, "tailmul.cl", "tailMul", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(KFP)), // Single-wide tailMul with one kernel + K(ktailMulLow, "tailmul.cl", "tailMul", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(KFP) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numCudaRegisters(MIDOUT)), K(kfftW, "fftw.cl", "fftW", hN / nW, kernelDefines(KFP)), @@ -855,9 +865,19 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, kernelDefines(K31) + numCudaRegisters(TAIL31)), // Single-wide tailSquare with one kernel - K(ktailMulGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, kernelDefines(K31)), - K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", hN / nH / 2, kernelDefines(K31) + "-DMUL_LOW=1"), + hN / nH / 2, kernelDefines(K31) + numCudaRegisters(TAIL31)), // Single-wide tailSquare with one kernel + K(ktailMulZeroGF31, "tailmul.cl", "tailMulZeroGF31", SMALL_H / nH * 2, kernelDefines(K31)), + K(ktailMulLowZeroGF31, "tailmul.cl", "tailMulZeroGF31", SMALL_H / nH * 2, kernelDefines(K31) + "-DMUL_LOW=1"), + K(ktailMulGF31, "tailmul.cl", "tailMulGF31", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K31)), // Single-wide tailMul with one kernel + K(ktailMulLowGF31, "tailmul.cl", "tailMulGF31", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K31) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numCudaRegisters(MIDOUT31)), K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW, kernelDefines(K31)), @@ -868,9 +888,19 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, kernelDefines(K61) + numCudaRegisters(TAIL61)), // Single-wide tailSquare with one kernel - K(ktailMulGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, kernelDefines(K61)), - K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", hN / nH / 2, kernelDefines(K61) + "-DMUL_LOW=1"), + hN / nH / 2, kernelDefines(K61) + numCudaRegisters(TAIL61)), // Single-wide tailSquare with one kernel + K(ktailMulZeroGF61, "tailmul.cl", "tailMulZeroGF61", SMALL_H / nH * 2, kernelDefines(K61)), + K(ktailMulLowZeroGF61, "tailmul.cl", "tailMulZeroGF61", SMALL_H / nH * 2, kernelDefines(K61) + "-DMUL_LOW=1"), + K(ktailMulGF61, "tailmul.cl", "tailMulGF61", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K61)), // Single-wide tailMul with one kernel + K(ktailMulLowGF61, "tailmul.cl", "tailMulGF61", + !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailMul with two kernels + !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel + !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels + hN / nH / 2, kernelDefines(K61) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numCudaRegisters(MIDOUT61)), K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW, kernelDefines(K61)), @@ -970,6 +1000,8 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo kfftHin.setFixedArgs(3, bufTrigH); ktailSquareZero.setFixedArgs(2, bufTrigH); ktailSquare.setFixedArgs(3, bufTrigH); + ktailMulZero.setFixedArgs(3, bufTrigH); + ktailMulLowZero.setFixedArgs(3, bufTrigH); ktailMulLow.setFixedArgs(4, bufTrigH); ktailMul.setFixedArgs(4, bufTrigH); kfftMidOut.setFixedArgs(3, bufTrigM); @@ -981,6 +1013,8 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo kfftHinGF31.setFixedArgs(3, bufTrigH); ktailSquareZeroGF31.setFixedArgs(2, bufTrigH); ktailSquareGF31.setFixedArgs(3, bufTrigH); + ktailMulZeroGF31.setFixedArgs(3, bufTrigH); + ktailMulLowZeroGF31.setFixedArgs(3, bufTrigH); ktailMulLowGF31.setFixedArgs(4, bufTrigH); ktailMulGF31.setFixedArgs(4, bufTrigH); kfftMidOutGF31.setFixedArgs(3, bufTrigM); @@ -992,6 +1026,8 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo kfftHinGF61.setFixedArgs(3, bufTrigH); ktailSquareZeroGF61.setFixedArgs(2, bufTrigH); ktailSquareGF61.setFixedArgs(3, bufTrigH); + ktailMulZeroGF61.setFixedArgs(3, bufTrigH); + ktailMulLowZeroGF61.setFixedArgs(3, bufTrigH); ktailMulLowGF61.setFixedArgs(4, bufTrigH); ktailMulGF61.setFixedArgs(4, bufTrigH); kfftMidOutGF61.setFixedArgs(3, bufTrigM); @@ -1501,6 +1537,12 @@ void Gpu::replay_one(enum BOTTOM_HALF_KERNELS kern, int cache_group, int arg, Qu // If not in place, the output is to the scratch buffer Buffer const *in1 = buf; Buffer const *out = in_place ? buf : &buf3; + if (!tail_single_kernel && base == 0) { + if (cache_group == 1) { ktailMulZero.setQueue(q); ktailMulZero(*out, *in1, *in2); } + if (cache_group == 2) { ktailMulZeroGF31.setQueue(q); ktailMulZeroGF31(*out, *in1, *in2); } + if (cache_group == 3) { ktailMulZeroGF61.setQueue(q); ktailMulZeroGF61(*out, *in1, *in2); } + if (kernelsToExecuteX) kernelsToExecuteX--; + } if (cache_group == 1) { ktailMul.setQueue(q); ktailMul.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMul(*out, *in1, *in2, base); } if (cache_group == 2) { ktailMulGF31.setQueue(q); ktailMulGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulGF31(*out, *in1, *in2, base); } if (cache_group == 3) { ktailMulGF61.setQueue(q); ktailMulGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulGF61(*out, *in1, *in2, base); } @@ -1512,6 +1554,12 @@ void Gpu::replay_one(enum BOTTOM_HALF_KERNELS kern, int cache_group, int arg, Qu // If not in place, the output is to the scratch buffer Buffer const *in1 = buf; Buffer const *out = in_place ? buf : &buf3; + if (!tail_single_kernel && base == 0) { + if (cache_group == 1) { ktailMulLowZero.setQueue(q); ktailMulLowZero(*out, *in1, *in2); } + if (cache_group == 2) { ktailMulLowZeroGF31.setQueue(q); ktailMulLowZeroGF31(*out, *in1, *in2); } + if (cache_group == 3) { ktailMulLowZeroGF61.setQueue(q); ktailMulLowZeroGF61(*out, *in1, *in2); } + if (kernelsToExecuteX) kernelsToExecuteX--; + } if (cache_group == 1) { ktailMulLow.setQueue(q); ktailMulLow.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLow(*out, *in1, *in2, base); } if (cache_group == 2) { ktailMulLowGF31.setQueue(q); ktailMulLowGF31.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLowGF31(*out, *in1, *in2, base); } if (cache_group == 3) { ktailMulLowGF61.setQueue(q); ktailMulLowGF61.setKernelsToExecute(kernelsToExecuteX, kernelsToExecuteY); ktailMulLowGF61(*out, *in1, *in2, base); } diff --git a/src/Gpu.h b/src/Gpu.h index ba350c71..15a44c85 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -116,6 +116,8 @@ class Gpu { Kernel kfftHin; Kernel ktailSquareZero; Kernel ktailSquare; + Kernel ktailMulZero; + Kernel ktailMulLowZero; Kernel ktailMul; Kernel ktailMulLow; Kernel kfftMidOut; @@ -126,6 +128,8 @@ class Gpu { Kernel kfftHinGF31; Kernel ktailSquareZeroGF31; Kernel ktailSquareGF31; + Kernel ktailMulZeroGF31; + Kernel ktailMulLowZeroGF31; Kernel ktailMulGF31; Kernel ktailMulLowGF31; Kernel kfftMidOutGF31; @@ -136,6 +140,8 @@ class Gpu { Kernel kfftHinGF61; Kernel ktailSquareZeroGF61; Kernel ktailSquareGF61; + Kernel ktailMulZeroGF61; + Kernel ktailMulLowZeroGF61; Kernel ktailMulGF61; Kernel ktailMulLowGF61; Kernel kfftMidOutGF61; diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 112193b3..facd6273 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -12,6 +12,13 @@ // If L2 striping, process lines output by fftMiddleIn. fftMiddleIn outputs 16 * MIDDLE tailSquare lines. u32 get_line_number(u32 base) { u32 g = get_group_id(0); +#if !SINGLE_KERNEL +#if L2_STRIPING + if (base == 0) g = g + 1; +#else + g = g + 1; +#endif +#endif #if L2_STRIPING // Old, simple L2 striping code // return get_group_id(1) * WIDTH + base + g; @@ -78,6 +85,60 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s } } +#if !SINGLE_KERNEL +// The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local T2 lds[LDS_BYTES(1) / sizeof(T2)]; + LDSinit(lds, 1); + + T2 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulFP64 which must dependentLaunchWait before reading data from fftMiddleInFP64 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + + readTailFusedLine(in, u, line, me); + +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * me); +#else + T2 w = slowTrig_N(H * me, ND / NH); +#endif + +#if MUL_LOW + read(G_H, NH, p, a, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); +#else + readTailFusedLine(a, p, line, me); + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + fft_HEIGHT1(lds, p, smallTrig, w, 1, me); +#endif + + T2 trig = slowTrig_N(line + me * H, ND / NH); + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT1(lds, u, smallTrig, w, 1, me); + writeTailFusedLine(u, out, memline, me); +} +#endif + +#if SINGLE_WIDE + KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES(1) / sizeof(T2)]; LDSinit(lds, 1); @@ -122,6 +183,7 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { T2 trig = slowTrig_N(line1 + me * H, ND / NH); +#if SINGLE_KERNEL if (line1 == 0) { reverse(lds, u + NH/2, true); reverse(lds, p + NH/2, true); @@ -134,6 +196,9 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); reverse(lds, v + NH/2, false); } else { +#else + if (1) { +#endif reverseLine(lds, v); reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); @@ -148,6 +213,101 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { writeTailFusedLine(u, out, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(T2 *u, T2 *p, T2 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(2 * foo2(u[0], p[0])); + u[NH/2] = SWAP_XY(4 * cmul(u[NH/2], p[NH/2])); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + T2 new_base_squared = mul_t4(base_squared); + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], new_base_squared); + } +} + +KERNEL(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local T2 lds[LDS_BYTES(2) / sizeof(T2)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + T2 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP64 that launched dependents before writing FP64 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(in, u, line, lowMe); + +#if FFT_VARIANT_H != 0 + T2 w; +#elif NH == 8 + T2 w = fancyTrig_N(H * lowMe); +#else + T2 w = slowTrig_N(H * lowMe, ND / NH); +#endif + +#if MUL_LOW + read(G_H, NH, p, a, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig, w, 2, lowMe); +#else + readTailFusedLine(a, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrig, w, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrig, w, 2, lowMe); +#endif + + T2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutFP64 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrig, w, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, out, memline, lowMe); +} + +#endif + #endif @@ -194,6 +354,57 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s } } +#if !SINGLE_KERNEL +// The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local F2 lds[LDS_BYTES(1) / sizeof(F2)]; + LDSinit(lds, 1); + + F2 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + CP(F2) inF2 = (CP(F2)) in; + CP(F2) aF2 = (CP(F2)) a; + P(F2) outF2 = (P(F2)) out; + TrigFP32 smallTrigF2 = (TrigFP32) smallTrig; + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulFP32 which must dependentLaunchWait before reading data from fftMiddleInFP32 + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + + readTailFusedLine(inF2, u, line, me); + +#if MUL_LOW + read(G_H, NH, p, aF2, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); +#else + readTailFusedLine(aF2, p, line, me); + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + fft_HEIGHT1(lds, p, smallTrigF2, 1, me); +#endif + + F2 trig = slowTrig_N(line + me * H, ND / NH); + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT1(lds, u, smallTrigF2, 1, me); + writeTailFusedLine(u, outF2, memline, me); +} +#endif + +#if SINGLE_WIDE + KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES(1) / sizeof(F2)]; LDSinit(lds, 1); @@ -235,6 +446,7 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { F2 trig = slowTrig_N(line1 + me * H, ND / NH); +#if SINGLE_KERNEL if (line1 == 0) { reverse(lds, u + NH/2, true); reverse(lds, p + NH/2, true); @@ -247,6 +459,9 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); reverse(lds, v + NH/2, false); } else { +#else + if (1) { +#endif reverseLine(lds, v); reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); @@ -261,6 +476,98 @@ KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { writeTailFusedLine(u, outF2, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(F2 *u, F2 *p, F2 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(2 * foo2(u[0], p[0])); + u[NH/2] = SWAP_XY(4 * cmul(u[NH/2], p[NH/2])); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + F2 new_base_squared = mul_t4(base_squared); + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], new_base_squared); + } +} + +KERNEL(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local F2 lds[LDS_BYTES(2) / sizeof(F2)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + CP(F2) inF2 = (CP(F2)) in; + CP(F2) aF2 = (CP(F2)) a; + P(F2) outF2 = (P(F2)) out; + TrigFP32 smallTrigF2 = (TrigFP32) smallTrig; + + F2 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInFP32 that launched dependents before writing FP32 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(inF2, u, line, lowMe); + +#if MUL_LOW + read(G_H, NH, p, aF2, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrigF2, 2, lowMe); +#else + readTailFusedLine(aF2, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrigF2, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrigF2, 2, lowMe); +#endif + + F2 trig = slowTrig_N(line + H * lowMe, ND / NH * 2); + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutFP32 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrigF2, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, outF2, memline, lowMe); +} + +#endif + #endif @@ -306,6 +613,74 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar } } +#if !SINGLE_KERNEL +// The kernel tailMulZeroGF31 handles the special cases in tailMulGF31, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZeroGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; + LDSinit(lds, 1); + + GF31 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); + CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); + P(GF31) out31 = (P(GF31)) (out + DISTGF31); + TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulGF31 which must dependentLaunchWait before reading data from fftMiddleInGF31 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + + readTailFusedLine(in31, u, line, me); + +#if MUL_LOW + read(G_H, NH, p, a31, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); +#else + readTailFusedLine(a31, p, line, me); + fft_HEIGHT1(lds, u, smallTrig31, 1, me); + fft_HEIGHT1(lds, p, smallTrig31, 1, me); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS31 >= 1 + GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); +#if SINGLE_WIDE + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line]); +#else + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + which]); +#endif + trig = cmul(trig, mult); +#else +#if SINGLE_WIDE + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line*G_H + me]); +#else + GF31 trig = TOLOAD(&smallTrig31[height_trigs + which*G_H + me]); +#endif +#endif + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT2(lds, u, smallTrig31, 1, me); + writeTailFusedLine(u, out31, memline, me); +} +#endif + +#if SINGLE_WIDE + KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; LDSinit(lds, 1); @@ -350,20 +725,13 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS31 >= 1 GF31 trig = TFLOAD(&smallTrig31[height_trigs + me]); // Trig values for line zero, should be cached -#if SINGLE_WIDE GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line1]); -#else - GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line1 * 2]); -#endif trig = cmul(trig, mult); #else -#if SINGLE_WIDE GF31 trig = TOLOAD(&smallTrig31[height_trigs + line1*G_H + me]); -#else - GF31 trig = TOLOAD(&smallTrig31[height_trigs + line1*2*G_H + me]); -#endif #endif +#if SINGLE_KERNEL if (line1 == 0) { reverse(lds, u + NH/2, true); reverse(lds, p + NH/2, true); @@ -376,6 +744,9 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); reverse(lds, v + NH/2, false); } else { +#else + if (1) { +#endif reverseLine(lds, v); reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); @@ -390,6 +761,106 @@ KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig writeTailFusedLine(u, out31, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(GF31 *u, GF31 *p, GF31 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(mul2(foo2(u[0], p[0]))); + u[NH/2] = SWAP_XY(shl(cmul(u[NH/2], p[NH/2]), 2)); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], mul_t4(base_squared)); + } +} + +KERNEL(G_H * 2) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local GF31 lds[LDS_BYTES(2) / sizeof(GF31)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); + CP(GF31) a31 = (CP(GF31)) (a + DISTGF31); + P(GF31) out31 = (P(GF31)) (out + DISTGF31); + TrigGF31 smallTrig31 = (TrigGF31) (smallTrig + DISTHTRIGGF31); + + GF31 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF31 that launched dependents before writing GF31 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(in31, u, line, lowMe); + +#if MUL_LOW + read(G_H, NH, p, a31, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig31, 2, lowMe); +#else + readTailFusedLine(a31, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrig31, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrig31, 2, lowMe); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS31 >= 1 + GF31 trig = TFLOAD(&smallTrig31[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF31 mult = TSLOAD(&smallTrig31[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. + trig = cmul(trig, mult); +#else + GF31 trig = TOLOAD(&smallTrig31[height_trigs + line_u*G_H*2 + me]); +#endif + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutGF31 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrig31, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, out31, memline, lowMe); +} + +#endif + #endif @@ -437,6 +908,74 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar } } +#if !SINGLE_KERNEL +// The kernel tailMulZeroGF61 handles the special cases in tailMulGF61, i.e. the lines 0 and H/2 +// This kernel is launched with 2 workgroups (handling line 0, resp. H/2) +KERNEL(G_H) tailMulZeroGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { + local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; + LDSinit(lds, 1); + + GF61 u[NH], p[NH]; + const u32 H = ND / SMALL_HEIGHT; + + CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); + CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); + P(GF61) out61 = (P(GF61)) (out + DISTGF61); + TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); + + // This kernel in executed in two workgroups. + u32 which = get_group_id(0); + assert(which < 2); + + u32 line = which ? (H/2) : 0; + u32 memline = transPos(line, MIDDLE, WIDTH); + u32 me = get_local_id(0); + + dependentLaunch(); // Next kernel will be tailMulGF61 which must dependentLaunchWait before reading data from fftMiddleInGF61 + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + + readTailFusedLine(in61, u, line, me); + +#if MUL_LOW + read(G_H, NH, p, a61, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); +#else + readTailFusedLine(a61, p, line, me); + fft_HEIGHT1(lds, u, smallTrig61, 1, me); + fft_HEIGHT1(lds, p, smallTrig61, 1, me); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS61 >= 1 + GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); +#if SINGLE_WIDE + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line]); +#else + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + which]); +#endif + trig = cmul(trig, mult); +#else +#if SINGLE_WIDE + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line*G_H + me]); +#else + GF61 trig = TOLOAD(&smallTrig61[height_trigs + which*G_H + me]); +#endif +#endif + + reverse(lds, u + NH/2, !which); + reverse(lds, p + NH/2, !which); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, !which); + reverse(lds, u + NH/2, !which); + + fft_HEIGHT2(lds, u, smallTrig61, 1, me); + writeTailFusedLine(u, out61, memline, me); +} +#endif + +#if SINGLE_WIDE + KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; LDSinit(lds, 1); @@ -481,20 +1020,13 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig u32 height_trigs = SMALL_HEIGHT*1; #if TAIL_TRIGS61 >= 1 GF61 trig = TFLOAD(&smallTrig61[height_trigs + me]); // Trig values for line zero, should be cached -#if SINGLE_WIDE GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line1]); -#else - GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line1 * 2]); -#endif trig = cmul(trig, mult); #else -#if SINGLE_WIDE GF61 trig = TOLOAD(&smallTrig61[height_trigs + line1*G_H + me]); -#else - GF61 trig = TOLOAD(&smallTrig61[height_trigs + line1*2*G_H + me]); -#endif #endif +#if SINGLE_KERNEL if (line1 == 0) { reverse(lds, u + NH/2, true); reverse(lds, p + NH/2, true); @@ -507,6 +1039,9 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig pairMul(NH/2, v, v + NH/2, q, q + NH/2, trig2, false); reverse(lds, v + NH/2, false); } else { +#else + if (1) { +#endif reverseLine(lds, v); reverseLine(lds, q); pairMul(NH, u, v, p, q, trig, false); @@ -521,4 +1056,104 @@ KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig writeTailFusedLine(u, out61, memline1, me); } + +// +// Create a kernel that uses a double-wide workgroup (u in half the workgroup, v in the other half) +// We hope to get better occupancy with the reduced register usage +// + +#else + +// Special pairMul for double-wide line 0: both halves compute their own self-pairing (u with p), +// there is no cross-half data since line_u == 0 and line_v == H/2 both pair with themselves. +void OVERLOAD pairMul2_special(GF61 *u, GF61 *p, GF61 base_squared) { + u32 me = get_local_id(0); + for (i32 i = 0; i < NH / 4; ++i, base_squared = mul_t8(base_squared)) { + if (i == 0 && me == 0) { + u[0] = SWAP_XY(mul2(foo2(u[0], p[0]))); + u[NH/2] = SWAP_XY(shl(cmul(u[NH/2], p[NH/2]), 2)); + } else { + onePairMul(&u[i], &u[NH/2+i], &p[i], &p[NH/2+i], base_squared); + } + onePairMul(&u[i+NH/4], &u[NH/2+i+NH/4], &p[i+NH/4], &p[NH/2+i+NH/4], mul_t4(base_squared)); + } +} + +KERNEL(G_H * 2) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { + local GF61 lds[LDS_BYTES(2) / sizeof(GF61)]; + LDSinit(lds, 2); + + const u32 H = ND / SMALL_HEIGHT; + + CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); + CP(GF61) a61 = (CP(GF61)) (a + DISTGF61); + P(GF61) out61 = (P(GF61)) (out + DISTGF61); + TrigGF61 smallTrig61 = (TrigGF61) (smallTrig + DISTHTRIGGF61); + + GF61 u[NH], p[NH]; + + u32 line_u = get_line_number(base); + u32 line_v = line_u ? H - line_u : (H / 2); + u32 me = get_local_id(0); + u32 lowMe = me % G_H; // lane-id in one of the two halves (half-workgroups). + + // We're going to call the halves "first-half" and "second-half". + bool isSecondHalf = me >= G_H; + + u32 line = !isSecondHalf ? line_u : line_v; + u32 memline = transPos(line, MIDDLE, WIDTH); + + dependentLaunchWait(); // Previous kernel was fftMiddleInGF61 that launched dependents before writing GF61 data + + // Read line u (own half's line) and p (own half's multiplier line) + readTailFusedLine(in61, u, line, lowMe); + +#if MUL_LOW + read(G_H, NH, p, a61, memline * SMALL_HEIGHT); + fft_HEIGHT1(lds, u, smallTrig61, 2, lowMe); +#else + readTailFusedLine(a61, p, line, lowMe); + fft_HEIGHT1(lds, u, smallTrig61, 2, lowMe); + fft_HEIGHT1(lds, p, smallTrig61, 2, lowMe); +#endif + + // Calculate number of trig values used by fft_HEIGHT (see genSmallTrigCombo in trigBufCache.cpp) + // The trig values used here are pre-computed and stored after the fft_HEIGHT trig values. + u32 height_trigs = SMALL_HEIGHT*1; +#if TAIL_TRIGS61 >= 1 + GF61 trig = TFLOAD(&smallTrig61[height_trigs + lowMe]); // Trig values for line zero, should be cached + GF61 mult = TSLOAD(&smallTrig61[height_trigs + G_H + line_u*2 + isSecondHalf]); // Two multipliers. One for line u, one for line v. + trig = cmul(trig, mult); +#else + GF61 trig = TOLOAD(&smallTrig61[height_trigs + line_u*G_H*2 + me]); +#endif + +#if SINGLE_KERNEL + // Line 0 and H/2 are special: they pair with themselves, line 0 is offseted by 1. + if (line_u == 0) { + reverse2(lds, u); + reverse2(lds, p); + pairMul2_special(u, p, trig); + reverse2(lds, u); + } + else { +#else + if (1) { +#endif + revCrossLine(lds, u); + revCrossLine(lds, p); + pairMul(NH/2, u, u + NH/2, p, p + NH/2, trig, false); + revCrossLine(lds, u); + } + + dependentLaunch(); // Next kernel will be fftMiddleOutGF61 which must dependentLaunchWait before reading data + + fft_HEIGHT2(lds, u, smallTrig61, 2, lowMe); + + // Write line u (own half's line) + writeTailFusedLine(u, out61, memline, lowMe); +} + +#endif + #endif From f7fc0fa668ebeb4b6146a2dddd65776ee8fa0ee9 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 09:47:21 -0600 Subject: [PATCH 194/214] CI: fix the kernel-smoke log dump, which never ran on failure Review follow-up to #67 (thanks @tdulcet). Steps run under "bash -e", so timeout -s KILL 10m ./prpll ... ; rc=$? cat gpuowl-0.log test "$rc" -eq 0 does not do what it looks like: when prpll fails or is killed the shell exits at that first line, and the log dump -- the whole point of the arrangement -- never happens. The step then reports the failure with no indication of how far the kernels got, which is exactly the hole #67 set out to close. Guard the run with "|| rc=$?" so the dump is reached, and tolerate a missing log (prpll writes none when it exits before opening one, as it does when no OpenCL device is found). Also from the review: - Download the POCL tarball to a file with "curl -sSLf" instead of piping it into tar. Piped, a 404 reached tar as garbage on stdin and the step's exit status was tar's, not curl's. The release tag archive also extracts to a predictable "pocl-$POCL_VERSION", so the "pocl-pocl-*" glob goes away. - The job comment still said one runner and no aarch64; it has been two runners since the arm64 entry was added in the same PR. - Say why the LLVM packages are installed unconditionally even though the 26.04 images already carry most of them: the cached libpocl links against them, so they are needed on a cache hit too. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b174b00f..0a082ad3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,17 +11,16 @@ jobs: name: Kernel compile + Gerbicz check (POCL CPU, ${{ matrix.os }}) # The kernels in src/cl are compiled at first use, on the device, so the build jobs below never see them. # This compiles the FP64 kernel set on POCL's CPU device -- a non-NVIDIA OpenCL target -- and ends in a - # Gerbicz check. One runner, on purpose: the cost is kernel compilation, and it need only happen once. - # Debug build so the runtime asserts are exercised too. The integer-NTT types are left out (far too slow - # on a CPU device); aarch64 POCL is not tried yet. + # Gerbicz check. Two runners: the arm64 entry puts the kernels through LLVM's AArch64 backend and runs + # the host code on arm64, which the build-only arm jobs above never do. Debug build so the runtime + # asserts are exercised too. The integer-NTT types are left out (far too slow on a CPU device). # # POCL_WORK_GROUP_METHOD=loops: with the default (vectorizing) work-group method POCL spends 15-25 minutes # compiling these kernels on the AVX-512 runner CPUs; with "loops" and POCL 7.2 the whole check takes about # 10 seconds. Ubuntu's own POCL packages (5.0 on 24.04, 6.0 on 26.04) are slow even with "loops" and the # conda-forge 7.1 build computes wrong results, so POCL 7.2 is built from source against the distro LLVM # and cached; a cache hit costs seconds. - # The arm64 entry runs the same kernels through LLVM's AArch64 backend and the host code on arm64, - # which the build-only arm jobs above never do. Same commands: 26.04-arm has the same LLVM 21. + # Both runners are handled by the same commands: 26.04-arm has the same LLVM 21 as 26.04. runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -33,7 +32,10 @@ jobs: steps: - uses: actions/checkout@v7 - name: Install - # The LLVM packages are needed at run time too (the cached libpocl links against them), not only to build POCL. + # Installed unconditionally: the cached libpocl links against the LLVM runtime libraries, so they are + # needed on a cache hit too, not only to build POCL. The 26.04 images already carry llvm-21-dev and + # clang-21, but not libclang-cpp21-dev (which POCL needs); naming them all keeps this independent of + # what a future image happens to ship. run: | sudo apt-get update -y sudo apt-get install -y ocl-icd-opencl-dev cmake ninja-build llvm-21-dev libclang-21-dev libclang-cpp21-dev clang-21 libhwloc-dev @@ -46,8 +48,9 @@ jobs: - name: Build POCL if: steps.pocl-cache.outputs.cache-hit != 'true' run: | - curl -Ls "https://api.github.com/repos/pocl/pocl/tarball/v$POCL_VERSION" | tar -xz - cmake -G Ninja -S pocl-pocl-* -B pocl-build -DCMAKE_BUILD_TYPE=Release -DWITH_LLVM_CONFIG=/usr/bin/llvm-config-21 \ + curl -sSLf "https://github.com/pocl/pocl/archive/refs/tags/v$POCL_VERSION.tar.gz" -o pocl.tar.gz + tar -xzf pocl.tar.gz + cmake -G Ninja -S "pocl-$POCL_VERSION" -B pocl-build -DCMAKE_BUILD_TYPE=Release -DWITH_LLVM_CONFIG=/usr/bin/llvm-config-21 \ -DENABLE_ICD=ON -DENABLE_TESTS=OFF -DENABLE_EXAMPLES=OFF -DCMAKE_INSTALL_PREFIX="$HOME/pocl" -DPOCL_INSTALL_ICD_VENDORDIR="$HOME/pocl/etc/OpenCL/vendors" ninja -C pocl-build install - name: Use POCL @@ -62,8 +65,12 @@ jobs: cd build-debug ./prpll -h # Dump the log whatever happened, so a timeout or crash still shows how far the kernels got. - timeout -s KILL 10m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400; rc=$? - cat gpuowl-0.log + # Steps run under "bash -e", so the run has to be guarded: unguarded, a failing run ends the step + # right there and the dump never happens. prpll's own stdout is block-buffered into the runner's + # pipe and is lost when it is killed, so the log file is the only record. + rc=0 + timeout -s KILL 10m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400 || rc=$? + cat gpuowl-0.log || true test "$rc" -eq 0 grep -q "OK *400 " gpuowl-0.log From 1fe537a0107eb89830ec08bc8028108037aacf01 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 11:46:25 -0600 Subject: [PATCH 195/214] Do not skip the sub-group barrier where the hardware is not in lock-step bar(WG) and barsync() return without doing anything when WG <= WAVEFRONT, on the assumption that a wavefront advances in lock-step so the threads exchanging data through LDS immediately afterwards need no barrier and no fence. That holds on AMD GCN. It does not hold on nVidia Volta and later: Independent Thread Scheduling lets the threads of a warp drift apart, and the CUDA programming guide is explicit that intra-warp communication through shared memory is a race without bar.warp.sync. It holds nowhere else either -- an Intel GPU or a CPU device under POCL has no lock-step wavefront to lean on at all. Today the branch is only ever taken on AMD, where it is sound: with the shipped nW()/nH() rule and widths and heights of 256 and up, G_W and G_H are never below 64, so on nVidia (WAVEFRONT 32) the skip never happens. It goes live the moment any config yields G_W or G_H == 32, which is a one-line change to nW()/nH() away -- and the failure would be silent wrong results, not a crash. So decide by what the hardware actually guarantees: - AMD, and nVidia before Volta: return, exactly as now. The AMD fast path (FAST_BARRIER, WAVEFRONT 64, taken today at width 256) is untouched. - nVidia sm_60 and later: sync() to reconverge the warp, then mem_fence(CLK_LOCAL_MEM_FENCE) to order the LDS traffic. Both already exist -- carryFused uses sync() for exactly this purpose, and the CUDA shim maps mem_fence to __threadfence_block() -- and together they cost a fraction of a barrier. - Anything else: fall through to the real barrier below. barsync() gets the same treatment except that it cannot fall through, as substituting a barrier over all threads is not allowed there. Verified by forcing WAVEFRONT to 1024 locally, which makes every bar(WG) take the branch: master then fails immediately on both an Intel iGPU and POCL with "EE 0 on-load: 0000000000000000", while this patch gives the correct "OK 400 0d42972a9d02005c" on both. Unforced, the residue is unchanged on both devices, as the branch is not reached. Co-Authored-By: Claude Opus 5 (1M context) --- src/cl/base.cl | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/cl/base.cl b/src/cl/base.cl index 69a81875..c2e18547 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -815,7 +815,21 @@ void OVERLOAD bar(void) { // Create a barrier across a subset of threads OR across all threads if that is faster. void OVERLOAD bar(const u32 WG) { - if (WG <= WAVEFRONT) return; + // A group no larger than a wavefront can skip the barrier only where the hardware really does run a whole + // wavefront in lock-step. AMD GCN does, and so did nVidia before Volta. Volta and later do not: + // Independent Thread Scheduling lets the threads of a warp drift apart, and every caller of bar(WG) + // exchanges data through LDS right afterwards, so the warp has to be reconverged and its LDS traffic + // ordered -- which is what sync() plus the fence do, for a fraction of the cost of a barrier. Anything + // else (Intel, a CPU device under POCL) offers no lock-step guarantee at all: use a real barrier there. + if (WG <= WAVEFRONT) { +#if HAS_PTX >= 600 // bar.warp.sync requires sm_60 or higher; ITS arrived in sm_70 + sync(); + mem_fence(CLK_LOCAL_MEM_FENCE); + return; +#elif AMDGPU || HAS_PTX >= 200 + return; +#endif + } #if ENABLE_BARSYNC && HAS_PTX >= 200 // bar.sync with thread count requires sm_20 support or higher. Slower on TitanV, need to try on later nVidia GPUs. __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); // The above is GROSSLY slow on an RTX 5070Ti. The code below is much faster (may need to be expanded to handle more than four named barriers). @@ -834,7 +848,15 @@ void OVERLOAD bar(const u32 WG) { // barsync() fails to compile at the call site instead of the whole of base.cl failing whether or not it is used. #if HAS_PTX >= 200 void OVERLOAD barsync(const u32 numWG, const u32 WG) { - if (WG <= WAVEFRONT) return; + // As in bar(WG) above, except that substituting a barrier over all threads is not allowed here, so on + // Volta and later the warp-wide sync is the only option. (This routine is nVidia-only to begin with.) + if (WG <= WAVEFRONT) { +#if HAS_PTX >= 600 + sync(); + mem_fence(CLK_LOCAL_MEM_FENCE); +#endif + return; + } #if USE_REGISTER_BARSYNC // bar.sync with a register is horribly slow on an RTX 5070Ti. __asm("bar.sync %0, %1;" : : "r"(get_local_id(0) / WG + 1), "n"(WG)); #else // WARNING, WARNING, WARNING: On TitanV using CUDA 12.9 tools and driver 580, this branch does not work in openCL (but works in CUDA build). From c337b3c138151c49533ef2dc13246938f26013d5 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 15:36:17 -0600 Subject: [PATCH 196/214] shufl: the 4-byte path must honour r, like the 8- and 16-byte paths The 64-bit shufl(lds, u, f, r, numWG, lowMe) takes r because a caller may have done only part of an fft_RADIX step, and the 16- and 8-byte paths use it: lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] The 4-byte path instead hard-wires RADIX in place of r: lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] which is the same expression only when r == RADIX. The 32-bit sibling at the bottom of the file honours r, so the two implementations disagree and one of them has to be wrong. The one caller that passes r != RADIX is the SIZE=256/RADIX=8 width FFT (fft_common's "WG == 32 && NW == 8" branch, which does fft8_4 then shufl(..., 1, 4, ...)). Reaching it needs nW() to return 8 at width 256, which it does not today, so this is dormant -- but nW() has been changed before, and the failure is a silently wrong FFT rather than anything noisy. Use the general index. For r == RADIX it is arithmetically identical -- i / 1 is i, i % 1 is 0, and (lowMe & ~mask) * r is (lowMe & ~mask) * RADIX -- so nothing reachable today changes. Verified with nW() forced to return 8, which makes width 256 take that branch, on an Intel iGPU, -fft 256:2:256 -block 200 -iters 400. Note the branch also needs the sub-group barrier fix (the other PR) to work at all there, because G_W is then 32, which is the wavefront size: master 8-byte: EE 4-byte: EE master + barrier fix 8-byte: OK 4-byte: EE <- this bug, alone master + barrier fix + this 8-byte: OK 4-byte: OK where OK is the expected residue 0d42972a9d02005c at iteration 400. Unforced, that residue is unchanged on the iGPU and on POCL with the default shufl width and with SHUFL_BYTES_W/H=4, as expected since r equals RADIX everywhere reachable. Co-Authored-By: Claude Opus 5 (1M context) --- src/cl/shufl.cl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index 0f16c2f7..8548a2bd 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -421,20 +421,24 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // code generated is not pretty). This might not be true for nVidia or future ROCm optimizers. local int* lds = (local int*)LDSsharing_ptr(lds2, numWG); + // Use the same write index as the 8- and 16-byte paths: it honours r, which is smaller than RADIX when + // the caller has done only a partial fft_RADIX step (fft8_4 on the SIZE=256/RADIX=8 path). For r == RADIX + // this is identical to the i * f + (lowMe & ~mask) * RADIX + (lowMe & mask) it replaces. + LDStx_start(lds2, numWG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).x; } + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).x; } LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.x = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } LDSbar(numWG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).y; } + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).y; } LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.y = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } LDSbar(numWG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).z; } + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).z; } LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.z = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } LDSbar(numWG); - for (u32 i = 0; i < RADIX; ++i) { lds[i * f + (lowMe & ~mask) * RADIX + (lowMe & mask)] = as_int4(u[i]).w; } + for (u32 i = 0; i < RADIX; ++i) { lds[i / (RADIX / r) * f + i % (RADIX / r) * WG * r + (lowMe & ~mask) * r + (lowMe & mask)] = as_int4(u[i]).w; } LDSbar(numWG); for (u32 i = 0; i < RADIX; ++i) { int4 tmp = as_int4(u[i]); tmp.w = lds[i * WG + lowMe]; u[i] = as_T2_GF61(tmp); } LDStx_end(lds2, numWG); From dc56c31d77cfee5ed7ab0c5aabcf413dddf5217e Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 15:46:43 -0600 Subject: [PATCH 197/214] shufl: fix the LDSSWIZ read index for workgroups other than 64 The "second RADIX == 8" case of the 16-byte swizzled shufl writes to the natural slot and reads back from a permuted one: lds[i * WG + lowMe] = u[i]; ... u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; That read is the inverse of the write only when WG is 64. The 64 in it is really WG, and the 8 that multiplies i is really WG / 8, so at WG == 128, 256 or 512 threads collect each other's values: enumerating the mapping against the generic (non-swizzled) shufl gives 1008 of 1024 elements wrong at WG 128 and 4032 of 4096 at WG 512. The case is reached by -use LDSSWIZ_W=1,SHUFL_BYTES_W=16, and WG == 512 is fft_common's "WG == 512 && RADIX == 8 && VARIANT == 2" branch, i.e. a width of 4096. On a POCL CPU device, -prp 20000047 -fft 4K:2:256 -block 200 -iters 400: no swizzle OK 400 dc61d68c2b5a6155 LDSSWIZ_W=1,SHUFL_BYTES_W=16 EE 400 c02f878d4ddb764b (before) LDSSWIZ_W=1,SHUFL_BYTES_W=16 OK 400 dc61d68c2b5a6155 (after) Write the index so it inverts the write for any WG. At WG == 64 it is the same expression as before, so the only configuration that worked before still produces the same residue -- checked on an Intel iGPU at widths 512 and 1024 and at height 512 with the swizzle on, and with the default settings. The other three swizzled cases (the two RADIX == 4 ones and the first RADIX == 8) already invert correctly at every WG; only this one hard-wired the workgroup size. Co-Authored-By: Claude Opus 5 (1M context) --- src/cl/shufl.cl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index 0f16c2f7..b28624ef 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -119,12 +119,14 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Output to LDS in the order we expect to read. In the example: lds[0..63] = 0, 64, ... 448, 8, 72... lds[64..127] = +1 // No swizzle of LDS blocks is needed to eliminate bank conflicts. The first 8 threads written to LDS (multiples of 64) and // the first 8 threads read from LDS (multiples of 64) are already in separate LDS banks. + // The read index must be the inverse of the natural write for every WG, not just WG == 64; at WG == 64 it + // reduces to lowMe / 8 * 64 + i * 8 + (lowMe & 7). // We can however save a bar() by writing to same locations that previous shufl wrote to. if (f == 8 && r == 8 && RADIX == 8) { LDStx_start(lds2, numWG); //GRRR.... LDStx_start will do the bar we are trying to save for (u32 i = 0; i < RADIX; ++i) { lds[i * WG + lowMe] = u[i]; } LDSbar(numWG); - for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[lowMe / 8 * 64 + i * 8 + (lowMe & 7)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[((lowMe / 8) & 7) * WG + i * (WG / 8) + (lowMe / 64) * 8 + (lowMe & 7)]; } LDStx_end(lds2, numWG); return; } From e41a3e1201d1ac83799ef5a8d4012b20f20fadb4 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 15:52:09 -0600 Subject: [PATCH 198/214] Disabling MULTI_Q under -time must reach the kernels too clDefines() copies args.flags into its local "config" map at the top, and config is what the kernels are compiled from. The guard that turns MULTI_Q off when profiling writes only args.flags, which by then is no longer what anyone downstream reads: - the kernels keep -DMULTI_Q=1 and go on believing there is a second queue splitting the width, which changes which stripes fftMiddleIn and fftMiddleOut treat specially; - the L2_STRIPING limit a few lines below reads args.value("MULTI_Q"), now 0, so it allows WIDTH/64 -- double what the comment there says is allowed with MULTI_Q, WIDTH/128. The result is a kernel set that is internally inconsistent, and adding -time to a working command line then breaks the run outright: -use MULTI_Q=1,INPLACE=1,L2_STRIPING=4 OK 200 0b944634573d637e -time -use MULTI_Q=1,INPLACE=1,L2_STRIPING=4 EE 0 on-load: 001a940000006003 vs. 0000000000000003 Exception "Error on load" (-fft 256:2:256 on an Intel iGPU. Without -time the host caps L2_STRIPING at WIDTH/128 = 2 because MULTI_Q is on, and all is well.) Set config["MULTI_Q"] as well. After that, -time gives the kernels MULTI_Q=0 and L2_STRIPING=4, the same residue as every other combination, and runs without -time are untouched. The GRAPHS guard below does not need this: GRAPHS is not used by any kernel. Co-Authored-By: Claude Opus 5 (1M context) --- src/Gpu.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 3464d75a..2ad5e6ae 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -338,6 +338,10 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector Date: Sun, 20 Sep 2026 15:54:50 -0600 Subject: [PATCH 199/214] TrigBufCache: give each key field its own bits instead of adding them Every field of the small/middle trig cache key is a number type's "in use" flag plus that type's TAIL_TRIGS setting, added together into two bits: ((b)+(tt)) << 2) + (b31)+(tt31) ... Adding them throws away the distinction. FP64 in use with TAIL_TRIGS=1 and FP64 unused with TAIL_TRIGS=2 both give 2, so they produce the same key; the same holds for the GF31, FP32 and GF61 fields. A TAIL_TRIGS of 3 or more also carries out of its two bits into the neighbouring field. The consequence is a Gpu handing another Gpu's table back: the cache is per process and shared by all workers, so with two -workers whose tasks pick different number types at the same width and nW -- say one FP64 and one M31, with -use TAIL_TRIGS=1,TAIL_TRIGS31=1 -- one of them can be given the twiddles generated for the other. Nothing checks the contents, so it shows up as a Gerbicz failure every block, if it is noticed at all. Pack each field as one bit for the flag and three for TAIL_TRIGS. Enumerating the whole domain (both flag values and TAIL_TRIGS 0..3 for all four types, times tail_single_wide) gives 8192 configurations: the old key maps them onto 682 distinct values, the new one onto 8192. The key is internal to the cache lookup, so nothing else changes; residues are unchanged with the default settings and with -use TAIL_TRIGS=1 and -use TAIL_TRIGS=0,TAIL_TRIGS31=1. Co-Authored-By: Claude Opus 5 (1M context) --- src/TrigBufCache.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/TrigBufCache.cpp b/src/TrigBufCache.cpp index f9e532c5..85586786 100644 --- a/src/TrigBufCache.cpp +++ b/src/TrigBufCache.cpp @@ -1046,7 +1046,14 @@ static vector genMiddleTrig(FFTConfig fft, u32 smallH, u32 middle, u32 /* Code to manage a cache of trigBuffers */ /********************************************************/ -#define make_key_part(b,tt,b31,tt31,b32,tt32,b61,tt61,tk) ((((((((((b)+(tt)) << 2) + (b31)+(tt31)) << 2) + (b32)+(tt32)) << 2) + (b61)+(tt61)) << 2) + (tk)) +// Each field of the key is one "is this number type in use" flag and that type's TAIL_TRIGS setting. They +// need separate bits: added together, as they used to be, a type in use with TAIL_TRIGS=n is indistinguishable +// from that type unused with TAIL_TRIGS=n+1, and a TAIL_TRIGS of 3 or more carries into the neighbouring +// field. Two Gpus in one process can then share a cached table generated for the other one's number type, +// which is silently wrong twiddles -- caught, if at all, only by the Gerbicz check. +#define make_key_field(b, tt) (((((b) != 0) << 3) | ((tt) & 7))) +#define make_key_part(b,tt,b31,tt31,b32,tt32,b61,tt61,tk) \ + ((((((((make_key_field(b, tt) << 4) | make_key_field(b31, tt31)) << 4) | make_key_field(b32, tt32)) << 4) | make_key_field(b61, tt61)) << 1) | ((tk) != 0)) TrigBufCache::~TrigBufCache() = default; From 1186c6f2bf864b6cff41e8afb379a7d7ea255327 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 16:01:39 -0600 Subject: [PATCH 200/214] LDSbar: drop the lock-step early return, as bar(WG) and barsync() did The LDS-sharing variant of LDSbar still starts with if (WG <= WAVEFRONT) return; which is the assumption 0e67b41 removed from bar(WG) and barsync(): that a group no larger than a wavefront needs no barrier because the hardware runs it in lock-step. Returning here bypasses both of the routines that now decide this properly, so on nVidia Volta and later the warp is neither reconverged nor its LDS traffic ordered before the caller reads what the other lanes wrote. Delete the early return. bar(WG) and barsync() already handle WG <= WAVEFRONT by what the hardware guarantees: AMD returns immediately as before, sm_60 and later do sync() plus a local fence, and barsync() is the one used when the LDS area is shared, where substituting a barrier over all threads would not be allowed. Only the LDSMUL > 1 definition of LDSbar has this; the default one is just bar(WG) and is untouched, so nothing changes unless LDS sharing is asked for. Residues are unchanged on an Intel iGPU and on POCL with the default settings. I could not exercise the changed path: LDSbar's sharing variant calls barsync(), which exists only where HAS_PTX >= 200, so LDSMUL > 1 does not compile on anything but nVidia -- on an Intel device -use LDSMUL_W=2 fails with "implicit declaration of function 'barsync'" whether or not this patch is applied. Worth knowing on its own, since SHARING_LDS's "NVIDIAGPU || WG <= WAVEFRONT" reads as though non-nVidia hardware with a small workgroup were meant to be able to share. Co-Authored-By: Claude Opus 5 (1M context) --- src/cl/fftbase.cl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 2c55164a..65aae0c6 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -87,7 +87,9 @@ local void * OVERLOAD LDSsharing_ptr(local void *lds, const u32 numWG) { // NOTE: A "workgroup" is an independent group of threads doing FFT work (see WMUL in carryFused or TAIL_KERNELS=2). void OVERLOAD LDSbar(const u32 numWG) { - if (WG <= WAVEFRONT) return; + // No early return for WG <= WAVEFRONT here: bar(WG) and barsync() below both decide that for themselves, + // by what the hardware guarantees rather than by size alone, and returning early would skip the warp + // reconvergence and LDS fence they do on hardware that is not in lock-step. // If were not using semaphores to share LDS access, perform a standard bar. The standard bar is free to implement a full bar across // all threads if that is more efficient than a bar across a subset of threads. From 703359898d7281e950208ea368d49f1b4e60e50f Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 16:21:47 -0600 Subject: [PATCH 201/214] KernelCompiler: print the linked program's build log, not the compiled one's After clLinkProgram, compile() asks p1 for its build log again -- the same program whose log was already printed a few lines above, right after clCompileProgram. So the linker's own diagnostics are never shown, and a link failure reports only Linking 'carryfused.cl' error LINK_PROGRAM_FAILURE (-17) (args ) with nothing about why. That is not hypothetical: it is exactly what an aarch64 POCL device printed while the kernels were being brought up there, and the reason had to be found by other means. The same line also repeats every compile warning. With a #warning injected into transpose.cl, an Intel iGPU run prints it 8 times before this change and 4 after. Ask p2 instead, guarded, since a failed clLinkProgram may return no program at all and then there is nothing to query. Residues unchanged on an Intel iGPU and on POCL. Co-Authored-By: Claude Opus 5 (1M context) --- src/KernelCompiler.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/KernelCompiler.cpp b/src/KernelCompiler.cpp index d384a6f1..33337762 100644 --- a/src/KernelCompiler.cpp +++ b/src/KernelCompiler.cpp @@ -107,7 +107,10 @@ Program KernelCompiler::compile(const string& fileName, const string& extraArgs) Program p2{clLinkProgram(context, 1, &deviceId, linkArgs.c_str(), 1, (cl_program *) &p1, nullptr, nullptr, &err)}; - if (string const mes = getBuildLog(p1.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } + // The linker's diagnostics live on the linked program. Asking p1 again instead says nothing about the link + // -- and repeats the compile log that was already printed above. A failed clLinkProgram may hand back no + // program at all, and then there is nothing to query. + if (p2) { if (string const mes = getBuildLog(p2.get(), deviceId); !mes.empty()) { log("%s\n", mes.c_str()); } } if (err != CL_SUCCESS) { log("Linking '%s' error %s (args %s)\n", fileName.c_str(), errMes(err).c_str(), linkArgs.c_str()); } From 1a1f8e7dc5b1d9296d5a9bc42c6920194c0f7d68 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 16:26:14 -0600 Subject: [PATCH 202/214] Refuse OLD_FENCE=0 where no wavefront runs in lock-step carryFused's !OLD_FENCE carry hand-off has one lane per wavefront publish the ready flag on behalf of its wavefront, and one lane per wavefront spin on it, with sync() as the only thing ordering the other lanes around it. That works on AMD, where a wavefront advances in lock-step and sync() can be free, and on nVidia, where sync() is bar.warp.sync. On anything else sync() compiles to nothing, the hand-off races, and the answer is silently wrong -- on an Intel iGPU, -use OLD_FENCE=0 returns EE 400 8f64862427ff0b6f where every other configuration of the same run returns 0d42972a9d02005c. A barrier is not a substitute: the sync() at carryfused.cl:224 sits inside "if (gr < H / WMUL && me >= (WMUL-1) * G_W)", which the whole workgroup does not enter, so a workgroup barrier deadlocks there -- tried, and the run dies with OUT_OF_RESOURCES on kCarryFused. The protocol wants a wavefront-scoped primitive, and off AMD and nVidia there is not one. base.cl already defaults OLD_FENCE to 1 when not AMDGPU. Make that hold when it is set explicitly as well, the same way MULTI_Q and GRAPHS are overridden under -time. After this, -use OLD_FENCE=0 on an Intel iGPU logs a line and returns the right residue. Found by sweeping every -use knob against the invariant that none of them may change the residue. It was the only knob that did. Co-Authored-By: Claude Opus 5 (1M context) --- src/Gpu.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 3464d75a..04a21af6 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -340,6 +340,19 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vectorsecond.c_str()) == 0) { + it->second = to_string(1); + log("OLD_FENCE=0 needs an AMD or nVidia device; using OLD_FENCE=1.\n"); + } + } + // GRAPHS are not allowed when profiling with -time. GRAPH replays the four bottom-half // kernels without per-kernel events, and the events recorded while capturing the graph never execute, so the // profile would show those kernels -- most of an iteration -- as one call of ~0 ns. From 0776fee8bf55619157b9f976b4b3714ee0602146 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 16:51:48 -0600 Subject: [PATCH 203/214] CI: require the residue to be the same under several -use settings The kernel smoke job runs one configuration, so it can only catch a kernel that is broken everywhere. The failures this code actually has are the other kind: correct in the shipped configuration and wrong in another one, silently, because nothing ever runs that path. There is a cheap invariant for exactly that. A -use knob is a performance setting, so it must not change the answer: after a fixed number of iterations the residue depends only on the exponent. Run the same check under a few knobs and require one residue. TAIL_KERNELS=0 swaps the double-wide tail kernels for single-wide ones and WMUL=1 changes how carryFused partitions its workgroup, so between them the three runs cover rather more of the kernel set than one did. Each run starts from scratch, since otherwise prpll resumes from the previous run's savefile and the second and third runs would have nothing to do. Verified on a POCL CPU device: the three configurations agree on 0d42972a9d02005c, and adding a knob that does change the residue (OLD_FENCE=0, which races where no wavefront is in lock-step) makes the step exit 1 with the two residues in the message. Costs two more runs of about 20 s each on a warm cache. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 42 +++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a082ad3..5efe77ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,20 +59,40 @@ jobs: run: make DEBUG=1 -O -j "$(nproc)" - name: Compile the kernels and pass a Gerbicz check # Step-level bound (not job-level, so a slow apt mirror cannot fail the job): a correct run takes - # well under a minute. prpll ignores SIGTERM while inside an OpenCL compile, hence -s KILL. - timeout-minutes: 12 + # well under a minute each. prpll ignores SIGTERM while inside an OpenCL compile, hence -s KILL. + # + # The same check runs under a few -use settings and the residues must all agree. A -use knob is a + # performance setting, so it cannot change the answer: after a fixed number of iterations the residue + # depends only on the exponent. That makes a one-line invariant out of the failure this code is most + # prone to -- a kernel that is correct in the shipped configuration and computes the wrong thing in + # another -- which a single-configuration run cannot see at all. TAIL_KERNELS=0 swaps the double-wide + # tail kernels for single-wide ones, and WMUL=1 changes how carryFused partitions its workgroup. + timeout-minutes: 25 run: | cd build-debug ./prpll -h - # Dump the log whatever happened, so a timeout or crash still shows how far the kernels got. - # Steps run under "bash -e", so the run has to be guarded: unguarded, a failing run ends the step - # right there and the dump never happens. prpll's own stdout is block-buffered into the runner's - # pipe and is lost when it is killed, so the log file is the only record. - rc=0 - timeout -s KILL 10m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400 || rc=$? - cat gpuowl-0.log || true - test "$rc" -eq 0 - grep -q "OK *400 " gpuowl-0.log + want="" + for cfg in "" "-use TAIL_KERNELS=0" "-use WMUL=1"; do + # Start each run from scratch: left alone, prpll would resume from the previous run's savefile. + rm -rf gpuowl-0.log 5000011 + # Dump the log whatever happened, so a timeout or crash still shows how far the kernels got. + # Steps run under "bash -e", so the run has to be guarded: unguarded, a failing run ends the step + # right there and the dump never happens. prpll's own stdout is block-buffered into the runner's + # pipe and is lost when it is killed, so the log file is the only record. + rc=0 + timeout -s KILL 6m ./prpll -device 0 -prp 5000011 -fft 256:2:256 -block 200 -iters 400 $cfg || rc=$? + cat gpuowl-0.log || true + test "$rc" -eq 0 + res=$(grep -aoE 'OK +400 +[0-9a-f]{16}' gpuowl-0.log | grep -oE '[0-9a-f]{16}$') + test -n "$res" + echo "residue with '${cfg:-defaults}': $res" + if [ -z "$want" ]; then + want=$res + elif [ "$res" != "$want" ]; then + echo "::error::-use changed the residue: '$cfg' gives $res, defaults give $want" + exit 1 + fi + done Linux-OpenCL: name: Linux OpenCL From 515a27aa8d1af8d732bf21de832165dfc2b6bde9 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 17:46:14 -0600 Subject: [PATCH 204/214] tailMul: read the "low" line with the half-workgroup lane, not get_local_id The double-wide tailMul kernels run G_H*2 threads over two lines and work out lowMe = me % G_H for the lane within a line. Every access uses lowMe except the MUL_LOW read, which calls read(G_H, NH, p, a, memline * SMALL_HEIGHT); and read() indexes with get_local_id(0) -- which in these kernels runs 0..2*G_H-1. The second half-workgroup therefore reads the line shifted by one i step, and at i == NH-1 it reads into the next memline, past the end of the buffer for the last one. The single-wide kernels are fine: there me == lowMe. This breaks proof generation outright, and only proof generation, because tailMulLow is reached only from Gpu::exponentiate via Proof. A PRP run and its Gerbicz check never touch it, so everything looks healthy. On a 7900 XTX, a complete PRP of 786433 on current master: 786433 proof: invalid (316d17434d034dc2 expected 0abb7aeca5890b24) 786433 Proof 'proof-tmp/786433-10.proof' verification FAILED 786433 Proof generation failed; reporting the result without a proof at every proof power from 10 down to 4. With -use TAIL_KERNELS=0 or 1 (single-wide) the same run verifies OK, which puts it on the double-wide path; with this fix the default verifies OK for FFT types 0, 3 and 4, and the PRP residue is bfb6553c004222ba either way, since only the proof path is affected. Add an overload of read() that takes the lane explicitly and use it at the four sites, one per number type. A failed -verify used to exit 0, so nothing but the log could tell a good proof from a bad one. VERIFY tasks come only from the command line, so that is always a one-shot job: throw, which the existing handling in main() turns into exit 1. Add a CI step that would have caught the kernel bug: verify a known-good proof once per FFT type. Verification runs tailSquare and tailMulLow for a few thousand iterations, and the six types are the ones allShapes() enumerates, so every number type and hybrid is covered. A proof is just a list of residues, so one file serves all six. On a POCL CPU device the whole step takes about two minutes, and on an unpatched tree it fails. test/786433-10.proof is 1.0 MB: 11 residues of a 786433-bit exponent, the smallest an FP64 FFT can do at its 3.0 bits-per-word minimum. Proof power 10 keeps verification short (the final exponentiation is E/1024 squarings). It was generated on a 7900 XTX and verifies on POCL and an Intel iGPU too. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ src/Task.cpp | 4 ++++ src/cl/middle.cl | 7 +++++++ src/cl/tailmul.cl | 8 ++++---- test/786433-10.proof | Bin 0 -> 1081411 bytes 5 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 test/786433-10.proof diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a082ad3..fcd43126 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,27 @@ jobs: test "$rc" -eq 0 grep -q "OK *400 " gpuowl-0.log + - name: Verify a proof under every FFT type + # The Gerbicz check above never reaches tailMulLow: that kernel is used only when a proof is built, + # so it can be wrong while everything above stays green. That is not hypothetical -- the double-wide + # tailMul read a whole line with the wrong lane and no test here noticed, because a PRP run and its + # Gerbicz check are both perfectly happy without it. + # + # Verifying a known-good proof closes that hole cheaply: it runs tailSquare and tailMulLow for a few + # thousand iterations, once per FFT type, and the six here are the ones allShapes() enumerates, so + # every number type (FP64, FP32, GF31, GF61) and every hybrid of them gets exercised. The proof is + # type-agnostic: it is a list of residues, so one file checks all six. + timeout-minutes: 30 + run: | + cd build-debug + for t in 0 1 2 3 4 51; do + rm -rf gpuowl-0.log 786433 + rc=0 + timeout -s KILL 10m ./prpll -device 0 -verify ../test/786433-10.proof -fft "$t:256:2:256" || rc=$? + cat gpuowl-0.log || true + test "$rc" -eq 0 + done + Linux-OpenCL: name: Linux OpenCL diff --git a/src/Task.cpp b/src/Task.cpp index 7ec35f23..8c1726e6 100644 --- a/src/Task.cpp +++ b/src/Task.cpp @@ -251,6 +251,10 @@ void Task::execute(GpuCommon shared, u32 instance) { assert(proof.E == exponent); bool const ok = proof.verify(gpu.get()); log("proof '%s' %s\n", verifyPath.c_str(), ok ? "verified" : "failed"); + // -verify is a one-shot job (VERIFY tasks come only from the command line), so a proof that does not + // check out is a failed run and has to be visible as one: without this the process exits 0 and a + // caller cannot tell a good proof from a bad one except by reading the log. + if (!ok) { throw "proof verification failed"; } } else if (kind == PRP || kind == LL) { bool isPrime; diff --git a/src/cl/middle.cl b/src/cl/middle.cl index 90951dc5..b52cea27 100644 --- a/src/cl/middle.cl +++ b/src/cl/middle.cl @@ -42,6 +42,13 @@ void OVERLOAD read(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, const global T2_F2_GF31 for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG_SZ]); } } +// Same, but for a kernel whose workgroup spans more than one line: get_local_id(0) is then not the lane +// within the line, and the caller has to say which lane it means. +void OVERLOAD read(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, const global T2_F2_GF31_GF61 *in, u32 base, u32 lane) { + in += base + lane; + for (u32 i = 0; i < N; ++i) { u[i] = FFTLOAD(&in[i * WG_SZ]); } +} + void OVERLOAD write(u32 WG_SZ, u32 N, T2_F2_GF31_GF61 *u, global T2_F2_GF31_GF61 *out, u32 base) { out += base + (u32) get_local_id(0); for (u32 i = 0; i < N; ++i) { FFTSTORE(&out[i * WG_SZ], u[i]); } diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index facd6273..761c9046 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -270,7 +270,7 @@ KERNEL(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig #endif #if MUL_LOW - read(G_H, NH, p, a, memline * SMALL_HEIGHT); + read(G_H, NH, p, a, memline * SMALL_HEIGHT, lowMe); fft_HEIGHT1(lds, u, smallTrig, w, 2, lowMe); #else readTailFusedLine(a, p, line, lowMe); @@ -530,7 +530,7 @@ KERNEL(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig readTailFusedLine(inF2, u, line, lowMe); #if MUL_LOW - read(G_H, NH, p, aF2, memline * SMALL_HEIGHT); + read(G_H, NH, p, aF2, memline * SMALL_HEIGHT, lowMe); fft_HEIGHT1(lds, u, smallTrigF2, 2, lowMe); #else readTailFusedLine(aF2, p, line, lowMe); @@ -814,7 +814,7 @@ KERNEL(G_H * 2) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig small readTailFusedLine(in31, u, line, lowMe); #if MUL_LOW - read(G_H, NH, p, a31, memline * SMALL_HEIGHT); + read(G_H, NH, p, a31, memline * SMALL_HEIGHT, lowMe); fft_HEIGHT1(lds, u, smallTrig31, 2, lowMe); #else readTailFusedLine(a31, p, line, lowMe); @@ -1109,7 +1109,7 @@ KERNEL(G_H * 2) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig small readTailFusedLine(in61, u, line, lowMe); #if MUL_LOW - read(G_H, NH, p, a61, memline * SMALL_HEIGHT); + read(G_H, NH, p, a61, memline * SMALL_HEIGHT, lowMe); fft_HEIGHT1(lds, u, smallTrig61, 2, lowMe); #else readTailFusedLine(a61, p, line, lowMe); diff --git a/test/786433-10.proof b/test/786433-10.proof new file mode 100644 index 0000000000000000000000000000000000000000..c0f96f996115b0c792f87bfd51de98d91b8e2ead GIT binary patch literal 1081411 zcmV(#K;*wrQcxgJQcq7t3RXo@Q%O%wJu(VNK~qRmNm@lcHZ%%QPgg}!Juxr}PE}1p zMN&OYH#jylGcyW}I8y=~|7gAV38EN$yGm`No5IKpUzyztaQ{hghCngaCz=(nkNesD zCyEz{lc-+@Z=MJsNMF>?HJvMqjh<*HBdFMJyXT9Qk-ZAN!xcAmnXn#`o&_EVV}^$; zcSft}@4{uE(g4foBNm@x4esQ=jcJH{n=_u{uI9qkvkh1R2^A(AW?<;Bt@lD<3YkNg zetQhwxk~14%DrViVkdv+xlb9;R9PUecB(BzdufF6!O~1{V*84cd2BH2Ufq6vExi3!JxgXFAA6!5O^}}3sw3EA3u>- z(+)^FT*cKD+v;h4B4Xy0$y~qFr${e!l zE$J6odnA#Er0bgway9=SdnOOXb_U~l>lK zUOV4aalZ{@g}{R3SJc1e64uEpS-XyIn(%xiO*3}z?;%g9i(HiOUy5S|RL$V4HFbAc zB-k^-g|t~!EUshsJAv;9NbRLSM--9MjU~EZ2eLd)vJ(ks3)Pd3f7M?!Z1C~M0L#nu zzjn{g?b$30OM`saYlG3DE4zW3Tc^C7b}^V{gJW(lGfB4}3*F~-$L0}}d^Xpo)($SW z$56lB$3^Hf^MB%F{@JgQSZotx z+b@4hkan>REF1Ed7;0 z(-{DgdthHPrtCkH#^)$rVWagU{TGq{BZ-*f5WRgHrvsg14NvN86MyW_uy7;|^a0L> zyuI3B_`;{(9x+dcs-0kCZ?SL~>6#$`+|zW}I$Ut%aU9{TYmj%zZ# z!oa@DTC~N$vWDck@PTbKH;OxX%qC<& zscq&;!@E6%YZM^xOodujR|a~#yy(1;WeY1|ZqI)HCL}X3R)U+-e3L7@TZ|o)F#va%9g@EZP!q8jSO1Jr2VsP3M6b;3$=6vnB8?SH^je%7JF)i3uEy5Uy3H*BAnwTVOKPV>*2!N z#;6^vae6M}iyHPOg#vi%V$FmwGp6SKae$PtdZDu)X>;CJVrb2}f^}d5^xZw;|8Ri* zgu_(=4{C=fo8^@cr^)k7uaq}6F8nGIr^;G3F3uC1U)oLOe>wM1-t;Kp(A1OvLZ}e- z!C)0g&Gm2$x97;br;H}OFOx(kPHghxBkZp%h}Vwn{2UCJc9kV#6MeiV8N zprw%D5A(Uz-cNh&Q|lWH68&o!5d%1F4|B*}NIo2ohw*!;+d$3&kxR_L84X51^Ys8(U2LtB}{Fe)iO58h3| z?E6ACM9V0$!qc4xjL#F)dDWQksd;`bV;+C4W;0t@%;`0OgQdfBy{F}>u;y@x(vA9> z`-%eD7Blw#kTzm@KT)7#_X`_%tzR|RSD=0_0>NpJU=t`Dzn%aQi29c5KF8d~dDl&O z5#%P`!Td*g2ZP;)EaQ_KdqY>iWpzmCtD;?-)2->IiY7K|N}O_YvX! zCnBcIADUHM^~IA+?6^-TIeQ=K5pLveuW9;!;~gIH_*cFR3V_;Jm`?J|m&QQc{@*FJ zGb5k~HhV7$`+ZAI$`ir$(!(&kh6v?`?8 zLHdYeoQNg3D4o?ilcZgTSBx4-Y>#b;h?}+dt3Olz~>YrHfTqm|D8VB52H! zMgTq$+tl2r(_s*LGys%O>5^huubV5|*e&Vuo(IErQF&{P>f)~4uzP!7`#(H&eZ-f;gj{4h8t96HZOi#ezx7Ik}GjDl)>2l za7FCR`xJ7fa{tOjn}?!plz8*X?3C)6zdhwuCta)@r?V-GDA+SHb(R`V*w~SqQfc*z z?`TPUcn06?agqGSEA=T*QzS^>s1Wa@hLFMB@z$1WD*QT5>FSo;%+w(eh%-bP2hqIK z5+u)FY0p$%>>gZezqW!oG!f&Vei-uNwnNJTS>3SnL57OPPX>~LjHy1TyZR3{rKeK+ z*(u&tsBQ z8Wd+P91-WWcgzVmM)bF~Ag8LDLkzHDYJ-&of&I*M-lu#qm;9iHATZ|<$IhW zzH4Ol|Dw);C3Z1l4}%4y+rUQ-88uRQyUY%Mhqhe!P0EhrF4sSDeQm*j)EBg)WUL%st%dLcLzGM?hl2G})twtL7a ziz=~KubKBgEJDc{H3}zDK?7Yrk<^BEyB(p`qVAM$FQ9BadArU{%R1$cZZ$Xnl8sGd ziO^6u7dSjZ<;EhtS%EaN#Vs#$M~pg0NmyEv&l?g^;B~aa;^~t?VLIwg=x#%!jNLQG z59J1$A+vTFF)+$kW7O|SG%V2FcnmeZ#hy!epS?c*zW+{5V2#io$DhTEu(02QYfW#g z)RB!ga+X7~;%b0>eSdf5_A+6OU-_sZft|5sCq?oecFjmqhi&x2g)zHG+1I!MPR67koXy_5r z$~0Zi1f!)69-(G1X?7|{ZkQCil^=c#mBMU6mAqPeSdA@5#9hnfYIMeU8HEu^Imxr@ z-9adbfB|qCaj)F$41fDjvg+fqCyJ*mv}rq_p-Y>2G3i)ZMp*@+oc1Gbg$PAJ+dz9~ z2Bv%%17aD262w*4cNNp3py9#)samitt*F^rTLd~ehajIzbHT93Fz5lk4B=OG8l_q! zOh)SWQBO&z^g%rJ$F(Fk^t#IT)(};%pdpWXVXYhJyPoUsxiC&AMeig4_zkcfSZN8J zN!ziB-O&8Z-GVd#i(^2W#(W!>yPGp%7H#ct0wUs*y#)WHd?gLgkj0eY6X2wf%yVQ4 zTSj25_i>il)lTXfm7FIU^0_Y?K)-O*U_sMJX z1qJHQOFt=%`mtzkm|rZ#Yl=IE`#z<2Q|il52EF>>R(QH-f>l!W@vF{o0Z>qM! zU!Yr1%J_t8%Bu!@Smgx7u1PYfyt+zlZ)S2*hxy zYC5%}&k+poOxLCUHF~t<$1QjwL2IuNaV94I%NpT^*SKh!fV{HGD| z+?4Mrv(_P`6^4N;cFzMjAUBU2oL5Q3?|X;7bX@(t7z4t&_YMnAcCk@8rc-Yhaep&~ zrJju_5GA1z3kBd2Qu)1t5w2LHl2txT#CS93GDz@H!5FOpOl1K{|YT; zEQIw;&8KlE_K|5m6`?9ps|0hBgd!W9wecf8qH36A;>?3np1cdk3@vELn&gK-?Xs;A z=bq&fOReR)tU-)06}$ltsCp+-^(g3M@h)ug{K_OK%1oYjn;TVVk3tp79}-o*dQKwA zPY)-wO$zrx{tjV}cwTV!26QLi`%k;e1z;W=0|KnMFUSGccqg=j|2D0X?l4nL@8T`- z$E`Phj_=q%48$u1^&%WL&M2}DLL!LvH?Gmhh*+dJ98ZU;e2U2>K-oLU`cVoDObO3` zeFhcPJ<2t(YufZNk4QBeWn9s;yqs!+TA^bWJv=rqATnh&WJ`NHR}I#J)P37h2zngx zbeeb1*OJmo`niP4EL0bQ9iP~NBJ%tRTc7dcDI@w?dPj<^8msW5;{Zp5!Otw^`5S|O zNPw6CM~x?Xa6&KL3q=-pNaTS5YLV)ty@p72JM=gD_}u8)_&t#2nPA`25@tGXL(*C{ zOfYvJE|$<5fZhIctmec*vmj+X^Q}n_CROnP((HR0s4}eLB}!+YS1ju;;2TO~6T;xLdjRtDJvyK3;N7!A$K% zeli)S%0U05F#UP550XZM0L;WQl8XIQ^Pz4zRQ?fRQk#8F64Dm;lhG>+8SIRja->eA zzXv)`V7t!igN?oxUHtC!W{p;(-SR>Eu^!@zluxC{dUnn<^O8|hp6qUb3`~bGID;h$ z5YZNj3?2Lc>oF5a%0&&p`RO|aJV?WZ+bW<@}{6=FE0Q)DYWgxoxcN$N58>r%&878oBiz_ZBk;i{F>4b(`ZW? z1Sbf8 zSK(2KZ1AEM#d;a`dj~m4nX1oVc!1x|v=rS=cB`>z3T1_tfulRQ8q1@EZXXG-ra zd^|3`Lw!6G$tavc)=}o{gTAa#BpxhC6az#$``_F(PnZow}6zbM)vd^ zCqyIlf*qaqgeiTbKK4B3M}Y)PjP>L2*m7ahJR<*}PMEgm2=@y%P}2?2MnotuEk z7S5l*Nb)(I5Jwtm#*i}|MAW9n&|zwWd;7sQoTi8@j1E1_1H0r2!|^8VF*2C1pWf<+ zp*^c)_`u0poqm0ux&m~D3UE+cu<;IPd<@!HBdzRuI)^RXWXSIu{7g0ieFXxw551Lvj6KA76h|Dvy zjM92yaqnP}aTM$j^DD5-4(8E%#v2?DQDNPOsk69tzsQ_>-;(E|O-T-HK09>cEmiM# zv+B7T;~;-1$!0EzQM0k1B&?e6%rBqMhIYx`l24}J^s%GIT;tL&E=2))v7+>50J6T9 zZyXmWX+Ff`teAowjQZ*dPwo6#y9CvJ(uCvCDePs6eU&lT^Y}c~j9Y~`I+_LrHbC$E zf-aFLBJL~v>|k)$>jizQImQ2A8Udt*tcz?d$7|P+;K3hGJ9YU^yvK3NtxK9#c*;)r zY)s;LvE+BC`V#RC>_9&U;-8LAX0C84Irqn(YnfmP)Q~XD-AnS^w>7J4Hll9f&3Gvo z5K8c>3aH#r)%J<0Nb|`yKZeMxP3pDvNsHM5Mf!>Yg@*pi`?ABJ_wZJLut5CrQ>lG@lNgxr~0&lzFK&3L=?-eiJ6%$-s}fhT=ql zJrnXITSe(p>=LtEv#c*iMd5{~T71TFcrpiSO~~xo=lU2{am#{0u%92ZqoJfuz;p_` zmOws=3wZ{4qt<-O9?xxk)rp3bIMadMAv)BehHpJ!y1)bkj3Uy&O|YZTz4~bp(->B} zUwB4yUQK`Su~%756zoN0iH>_+z;W$jAwfw4C8U+7-z!;}#NHZkRGXHN(;S}eA(c9v z59za%yvxj~J18ylBHKy7r-Z?NFw%pNR%YxN58UCOMWK#n7kLi+7Cv!2B5XnZM; zClW7@Pto`1#$EzkFUTeP4DEXZ!a{N=JF^1xqovoQ!j#^PhndJOD8V@y7>C1urL} zfHcF`BM2!6^)7j7BY`P>&rjF5)<~>(0V*u+AwpT!Bv-TS?9x$k^RGT zI=H%dIt78%sx<7v`;Hka%K*honCB9g=t)2^L;zR=)g0YvYglxQ$)wX^!^Yk0|9bu3 zOZa8@-~qY*3y$Ura!UBV1dRevksb}+AG=l87LbV>B%g#6aNU1OKZNfG;td(*HzJ>= zPg2#uJoD%l*V7*i2D0bsAzl&|t!Mw(n#|0lo|Pf+6BFkaKovlsROnk*L)e6y+GpKW zxoyWyT!Z1G`{AeKK@OYlJ!0>8zH@bYcv&;lsyS1^s;h-&;kLJC8h|539)IVmRBe!eOKXz zfFcgJH{|?+bLyx;6`lkF%5XC|RX62e!5pRuoD`X(`RNQXSmu(we&AM*%E2fTn&nRs3+hF0V@9N$rxH=FDuj1 zCo+q{3^gyF70p-ge)WXzAg_3EM0ErBoqSeg&e5L9#M&U@NY7u_#rmbsC)@C3eqZcW zDz=%$>KBkv{K0-F8S4ua(^tmT{DxBj`N;M*IZ4Bf5ir7T%7kAithtAY&hbv6sUL}` zRgsFR3pvQf-1RcQvM}a|8t2N6*m4}NXjWzG$%KAXx%3ghgHWY|zC{-Mt8to{ z6)Vt@os>cSO|r?=B#Q@p<(Cwi%;P7Cqq9~lMg(A=b)~+;ClZ4lemcTd_5U@j?Ypis zzl{i(>f3fBGqYc!5c*xg)zlI{7np%At()I zuk^M9WSIL$v7VR%`NKnfG7UEDkf;$4UemKPj0u%*8yT#+iRW2uLtRcBv)VIV?dsoN zj|ZomdN~kd)Yp)5km6K1%>%IZ9Y4icC)2xO;^S3eV=)5uB!Rz^b=`Ve8_$V7+&fX` z>irg9k|+K_75J}$U)yNo^^78$Cjy#%cZZYsCK}Ym`o3!CN|bO3DnS`9D(V8UR{Ky0 z9dOS&DrIFxzENgx_|$vS{#AGTW}1s6b$!DPyVU?6^FhpGti~xTsz(|j-^Q((0AVId zNz<81p|&&_lJP7cNWiZVCMaL}u5b%I8!cH+!tpq7LpX(ku{SKetYl?6-XZR7@{-qBXz!WFWq^;2pJ% z{0IrGC=Nr#LDSq2OYY@#<_#dqQxVu2VIC2fDj56K?GnDsj|n|DTuewfu_X`8Y@Q0skNYDKQq01f9o}Z<< zDgmp*ORJdReY_2bq5n6aHB?z3jXGg?*iC`vUJgK>!jRXyN%-F{Xd|n{fC~KB%)De- z@8#-144dNGiO>M+VURE$Bm>U0XA}Q$J@|8Pl|2f@uFjAycB-vuys$VBkyVJ%9RIK0Y}!U?UJBw$OfF?<2sHa2CzN)rr8^#?^+oN_bsEgfH z0J0TVa@S2JR8fg1>OuTAY3J=ZdCijf9CdjX#B>;Zn|jhe>N-aH4m zb>@X)k}M~Vl1&Af!>(#Vpn%^2#o0_iWRJh_{lOmkLK4+cP#57@S;WNL@S9j(ys$co zHeMW9;5mDN%L;P$D9-Drnh_#0C_ly3y}q1QTG{*o_MG4)i)VVI`i-)^Zp(76@L2G$ zv>cHJ9MA&AL)B(&0-8R{WcZDMq7xv9eA{?i=FMCC=R%&#DE}Tom`EdMH`897RiOoF z_4)C69dE@3GK#^Wj>RAe?JDE8CcCA>NdR44fC&&eDBvVR(BXhfk|rp2@MToTU>utn z3@op9Z95}wqbHl<9hWUOt<>9X1p9b3vsU}2%ioadUw-dnR|QZ^jo<*MYY)X^VkxCo zFD@=b=9p+64LJ^AcUUv(t=Dpkw=;x7?yjVktG%#NI$(~OQrgJ2v8V}m+ON-XyNB3& z+^z=RD2+v3&5I`W{4^zWvoPhA&APHb*P_0Vv$F_(Lr`L>I18gNMlhzTk_uTtG&-2C zNvC*}%|67Uk>LaPq$Jl39$BMNDojy;gv)#0ryfCV% zpd_{OD3g_mK!|z{ZzH57GKBetH>`waf%*9L!v<4yw_KxS=1a0Nff0U;*a+1kBawNg zKnd>B+QfcgLQu)yLd(JZ5#0H14G;K}5LpudO7s8hH&m1E)z5@}@|TG36)fn*dDz0@ z%1C2lJ-yZ7rT%V6>PlL#`6v{DXdaTf-g^NT%_b}y_EMwn?Ws}8Nrl7G+&6VTKwXhGj^DLf3byyReXF$q!sHbd2-g6bmA`F~Rb4jm?XJkV?hNah+{#9Y zhJM5IlYLjF6z@Vl=|I4WLrCivg%m-{1AV^EcGRx02fAF&$F5Sx`uO57?!ijs8FlwdYc0Nc!7J@jk{SJd%wc$GHCOeOkX zUJMt!DWEeD`k(c5-<&B3*CQ1WoCo!BSW-=*C`jVjSH8IfP0p%y;aIfXc=1)$)|lrp z2CYU;#r#Mom)1MOf3%_U8~}tF*AMTtOpskEh+D zR&~tX&hRDJ5gXO`uYTP~S1dmcsVz$^Cn;9$EU8H&Vpg74OtUqVz(qq+|2nG8#6a_K zv$DXri($e@9@50Mu;=J?>vEnl_NB$=OKKo{ln2wLB2tRpLaD`9j^~vvUer+49jgoc z3|SQ1qKQMZW{dRk6Rs(B8AOFKRRK!t_GR5a#$ccWV+LE0kK^b06)1N(nJ&R->&zSj z_3>qy04&5)Cho@?B+=vE4ryyWgNt{HYQ(5S~!V=uUq<7BRqKyrU~xOuQZBJN<$kf^;B280S&k?#-ND zZb~COJJR&jAQ^U*THiE9ARa|tF3(8LxJ)u|f^G4%9m;7V3%KL_IPX@hg}d9 z#O8P-s_bj(sf6E_sP z3G`k_TEy0rY#kcZZbIvBllI}g_;Fm+r&1wH{IxGdIXA)|op`?l6DA@!G(rLyupYUG zYiairVeXujhaBf78t?eH#HJ~35k{H$FrgyNYEUJSHS8@%R#7_b_}P`5JYksY4>zPAbtfZT z7_fD-N`EJF2zZ97@^!jX>vSwH2hO`eQmolE*ctE8&Ye9@Vol9EjT&id|Bbj>`n~Ut zeHIE;h4k*y&zF^WE3{DlOQasJ&C~o#)6os1xtr%wq(BZ7%;aJJT}4VZJcJl6jNy>r ztJ(QEU8_SbjRh0n7d2K^>zTN1Nt>!pIh{6xa)Fibfd}uH0nW=a z6Ulb{iX?b0Px12laL|X_2^Jc8J=;yrhq)|MS@cE^Q*Q&p5pD90!#Gtd={N=&j*x48 z*xP6BAvc=hTtym*OCD?^d(e>3BpJG6f;{P*ied8T0BUe+g^-5j7f{QBIBh1f{EJ!u zO;Aiaqe9jRho+TqbqR;5fw z3RW8W2i%|_lZXUgCKZi!;+iv;+N z@5kEI*wVqGs;)~<&S8}_r=XdKhKrNVHZk>QJ{hy4KbcznO=jmNa}i(P~Xr&mCmMHm6G z`l>E8!$`QnwI+ZN7x7LmK^&ACMjI^T48p+P&vk+bBxo7WJ?cIWbJAXN#*NaDDuNA!?JI%pUFi zRi4FIk?xVC!ZQ!Ys-4+&yM?nka>dZAw(QSXTj*_AiaAkInucU3y6yk`hyjye{anty zHyTZ@u3t|oR_*^jdm;S8P_i)OL;2))tb2E!N)TXOMF0=i**GBJ+<5cMDtP(_qt2j3 zj0AYj$23zRti?aZ(630mjgG5vVJDblZFW!iT;&Gb$3wGKV-oge3Nx_OM2df6m>TAGM-1pI6umK(j zc~6XP%xz`5A?js0E%Dp$ci;%qOxV3%-dvHuTK;rp)&o{{U+NWrGlUMpBC24-vc@(% zl7$Z4AG0_;NnXEZ%)DRL_MS+%VntS4ng*H}3A)+k1WV&2r3BqbTDC%vS@(tpo5<*u z+%yQZb(}Tb92>w>|2{Z}zLcTwkH@Azn(MZqh!6{Kr&8^XTsM}`#REuR_B00cDL;!} zHgT2l(JPK)I7_G4D*&`U!#&k!aTJ(&)R{Q1zt(1ai!b>oR(prrjv=5mOuX&-rTPf< zf1bTwpQ4$e?pGa%kJ9GB8O)_HxwT703oB;(_l#*e9Z4Q=qQkV)KqckTZyiK2u?O9r z0LME1HlIGloB6JMDy#+qrsJx9kqo=S_|I^S_f36vThY2(+(Yv2Z_rJlqLnNE)o93_EF9XH-70Lvj|Vuv-4!s+S5oh zK-av~FL&9hwL{c}Fn{kX%E5FeQgSO0H6vb}#1z_W7qTQ%Q3EDO(HHPVkZ$9A8$yD< z0pmT1Nob^8H(!X52T{tilcGb=2SK-r#Lh(PIy4i{Fo=Sp#pEuHvAeB;K$4)_I8X8q z8pO=mElnIj(Z3a}+^1~Ir+pEc8o2Tg3u6`7)hhdt1a=M#LborgDWR(jSX1gZEWJxt z$(H8V>%238VAJ}HsWylvx!|AVP$^q$4s(W-F&^Tnr5Vg<+Xw0+(b&&>VqB}roIbC9 z>IMEwaArIXj&{pCRsx*`6EQw@JWzZco88@X#t!wGKa$^Rmn2(L{IN zxyO$OS3$p9c<}@<>hchQhl1-LN&Lc{2sz*~0%4@_Dt^a&STH9_Z3{5Qq*q-03&8H= zZ&11vUbaC7VY%%1tHio1eO@+gBf--|dCtWdeyGl`JRlq*pwN2C5;+$k&=~j+OK=}q z_5upt=sK+VfEKVh>r&j}ft34$NtK9t7uotpJ^QMzX4OX?XV_Fyo0E*kl5@%K6`gW^ z*ajZ0I?ImH7jeED8+I zmzNLXRjXboRtD52YspVu3iZ@hALVl?9ecezMs`KZj9@b%YGDrzPe@IIiffx*5$xu< zoHBX=!t1Xc;>^dHlfiy#JA)L@PdR|P586$s8OSNC9P;}+j1J0%SP%=&Cbh+ThB!U{ z@f9@g8XwvL$AJC|f!cfg6esgzG;F}RbSJ(_=p;I#nkjP%eHY^kEADAdx`(EF$BPe+ z)0tCyte5&B0(N{V+gUV0oXTDOld7anO@L<5R(md}RfLF>LXYr5{-tkiRFq5Tww(6B zo0Coq>f<~|bspFE=|QI#TUfN%OjA_yGErsQVwH@g&b=p#$_q5YOq2bQAfugH6XCWm zUk9v`5(=$_6L3{;0#c!Cc$o8DrP=0hBd3F=#vaZk=KRd+xQF7WY`i_fGRiv-AiS?7 zgZ%jm7EpyeWMC6UvMoyuikmtVQ3JoaYA1nu0M_hELn*F)-6Wm(=K@RMaXoye5)9(*GWb7;31$;uRkyu3;G$`l}FA>1!bN>x|LM3;+;e5SZP?`ea8i4z2 zWh+YV{08ip%%RRPp^s7sKnA4;zDcZL+J;LMU)gE#pEd*MuXbPBEcxMIz~(lH4@aJs;KOv+t|*4iKzkzeT$uf~V7Gk_3Vrq8OFDfvF0&nA&k zjVH8j#66F4@=29GXEYU4d$5&Ybw6F@w3go=mL!_d9dc*FS`>8j;?5U^WL}ZlrgopB zbBa@4kH{8N%0l{2OMbBaNMLTMbluzpuhu@{SJy^kVz}VmE(y_zZ`&RGtBl@zWneI3 zF6UH4L3GaRnG$;(29ezngln;QsuIdmufJVpDMVz+H>|Y)%;nzwaWn+TvKd3Fj5yB+ zP?1?k7}!TpmGXc0pu7Wz+F=Z=#z+05UX=xHFS?r2ghnI++$O^_gvDY?!OC;l##2&i zN8c{vw`}^Nb*DOJ=iZ!T1@U=Z_36f+he~=$EFdT7^*u5O*PFe^>?g*_@bP5(2jbQV zZR~|3bi54F#bPi~seH4xR{IF97d<3$XZNLEfDD<155MA>eO|I=)%bXuaq~qC$-*Qe z3+e-C5o3Z8ui-uQly#lAsg=#PyFlW+4E_eXPLm#XVLwLcVnohFcK9dTi1}TrnYpZl zmLZt^*>KM`5Mf5aSs0bSgh87?DIEU4Kyu2KZtdZfRk?)t&3N-=5BNV1++qRFa(9e` zAvrCK2#6cHiOO#?ZR0>!Hp$erHbKR1z8N_v)Twmju=t54dUZ3%qhI#=-Tjf|8#36? zWW5Av*LSPl%E;?jP>;v^SG;LZZtsPCGxconuL&T&;H!y=P9+1jOOk0TNrj&y41}tIe$!vZd_^Kc+Y|0J!SUiGglOdW_k13n#BAChZ)CZzMBTgj0a(BGzBxJ zCXjaE4IZSxQ)#rgPdl6+xaSU(d=}Y+=c4KQjbd*G%{BaLPd5b$pHMoY?FRL+Vsxxh zIA3*3j@PvOt@-0MaD=<2u{T9xoJd2Acv{Ru{M+qJObXr;FFEHGhq(h?o9A9csk@y0 z2pQNAsDU&$814mWlEDw@SYa=PMN@~eScDx$rrbFM5n7#|%Il-W(q*^gOUi_n$1cBn z37fLBCmsWz(o3x4n>fWUwcw7ZwIbe{CzMO8Fxl{!vY2ULDpjoCR z4ShaG+cF4yT1KnN5AZhl_=MYa(YasIO6iTtzXB~exDV9Fmb%WajnfDSG(C*pIz_#c zX_$-j%jyV&N$LGU06i4oe32d*Sr_ZEzuT!sRkzjfQT!Rp>|_K7(BDH9*JV~`@5&f!v;4T z5$*(dJ*frCk;-o!#;%+W+Rx$FyWpn5Mw*k&z|>v=BQm&f)1X7b&77koog+y_oit&G z@0E3BJ67}m7@;L~h`@LTU{%CA;dmy93CTaUsb#KUrPy25i24$-UckRj6jWk<({;8c z0l0>Pif3a?y8y9ySv9k)FNRI=(bT>AC1L-I`0sMhFW~#%LkTpMSPWzG59FHA^WFR# zWEM(wf%c;61LA*54Su^97rV7Pp0$v@S(1gR@{u8v*X2Nw9HL&%3F*r4H!w}w-~)Ku zeQ<^JBIJzgQfvFlVqXcKwXzGXjLMH=+E9K=EG8tctTO!VYt`ngfxp@mMKMRNpF{vM z{zU*mklZw|bAh;^)Qp-&tlOQ8B{O}smwz>$Q?BDDZI3i`YBfeGI861IyW-*8mU1_a zoLm}^cg2Hyf{Gmf)5)->56Z&_ZLUmXN2ni}AmwTu&%?AJtD#J%x`)NM!*(QP%mLg! z`J6Ofz+103?%E)48Ge55$uE15f_A?NDRd$A+hLY|^FZ=PJdlu3jH~^z zV0XlHHXude+zU(BN2&ayl+ADw<)XH(mG-<(u?3|;I6#VpBePy^=a>m>-Y&%& zgHhp6$cM2vm>kM$epwXvKg5Kq;aK3U7bzAp8qVu|XJNK2TH@-9T@znvu4cw-wPKYuH11*O%!5a1b1 z*`_|*Jy*vZedT<^U*Fi=WzpQabdBu6vOr^D5oUIWy*W7qrTXCs8Xv76=;}+1cXOt@ zw`S`f`M|pwI-Y+K^yQ5YS;U-)8aYliAkJ{g5)6{=5i+=>o!?q5MpndaNeN6D9D4Pt z&dDpR<2c$|V*^rbV+S1*{5u4{shS(?6t9bh`Ah8vL>BWPTeaLual;Tc&ubA=)_LE2 zHaPm|{bv5Rw2(t|54h|zKe7D2$FzH}&DVk_XFReS97D&*duyyL55uA6T@!^=tjg(Z z6F{c^DR?(I96^g<$WwG1Q4v$|5ea!KF9O_$E;`XeiLj+s!nEpO1=n3_6N1586`$d` zDS}88jqWeFXQovfO8Y%C8En}zm9+1%U_i=mlmmg<9H`z6p7r{<<8LAdoATqRaSQc( zNe&)sRx9GO+~V)@oyT}Xd8@c@taLRgS4O=Ugn@p#J~Psq!4xYHyEHMbD}w`d5>)-y zSn+BLK-@G=sB`dgzkxUQzY~(7=$)W>HJ4%hWDwwAM~6pPgs6vE(;hd=9Z2O8(b|~# z4#KaBc-0}^WYeQ@1G~zc#~QV(*w4|4u_0QPTkrKob&{3$PTp;{BLm`#B%uYF6YxOL zem?ZRxgg9K4@RcENezFuSLJ18Z+^4;sodA2#bU$^@BYmgZ<`#SyAll*+TY#Z&)IB> z5a|YD-ssIJ(g>fd0#kN9^Q;`FN(g$`*s#L0%fCl&1pF`5b@s1%*Zu1?Ai*`<#M<`g z(%YASk2}Tko#IRU7gOynA)i`!W zvwv6Hq&%2_E7cY%%qG!%ow0ppRI^H>gfp)wDm}6}QA@$au{@}C%;f|SxVqWJxz!{c z{V4~|;LG`x>o3Qo-Rb%;tmSAOP(XU5+b)Y29^Ra%jW1xm|JSKxNTG?p&rDTr-(l62)Q-vYq}hx2jlPb@m0wWa)X>JVK}kJqQnC z7>^9`PZp&ZMJnjiqgt%FZ`rAFo*m+Za@EUzZRTFDTfZ~^m?$0Mg}#9py~&XYr_i| z(S=cww4+K-;&m=_JQsN%_3`r`&I2O6vIoCQEjbz7XEV3iS#Of6v&1(bi->4dxEs7v zc>4?dTHn8OY=Z*F==Ff)c*50ef7Ex!hO>l`8Tsag+zvzI-A)3f&~@tshj6}&A>k(e zh!*N7pVu4CpsLUPA((Ve1!y~FQsK$zbKTX03ittngdqt&`x9vg%b`e_^@>VvVdX^l z+Cf|W9g9Cu@cLxeWUZ$ne@kdehe*-T(BFSoOIM;lSDxd3=YF(|oXEf? zZWH7P+kyxu!|Cd3_pFAJE(*kA!#=Kd#H%WOc0XzeKmu2`%aq0SfC<_csj8PMaty?1 zPo@~#@tn@1q&z9)E1ryrFuybqF`&0yhKKVe3Q4Q2ySot9M~OS-%kRv6cP5}fBKIqi zS$l1_ENmdf49&}5)Ky?kQ^xOie*4>|K%B5Gi-@aaT9fSgDacxB==(@-F(Rnr z2?W-h8>eS5g}_k7n0H~=tw=@(A!4~a0aYH9-3k?OMotV_5<|p9AbV{K7pL+H!x7+e z(~SkDr5QtIru`Zbx>~jLO}d^_MVPz=y@(b$ydTB9-LhWQMXl#M%9K*xOFER**kEFd zP4C$OfSIB8x6Y60%0QL?8f5tSpsbsU*e&Y5Cu|WALBT1jNAav}FfVs0G7_pD1I#lm zMX;mnme#GvkU1unq9?ccGwlv)fnuww@iqfQTIS+{X^tctoY6Ho0k)JKgs)19|7I*) z2?O2kXY9TAh8f!+$;RnPG%(Wy5%h_E_OVp21bmfV+-Qv8@QV@abL9*Zov=mT_8xi%c ztpI?0B#9ftWU93n&X$)FUBMp=D5Mnuer;GEv}v^SmxFK0aMTiEKI~qrW zs}M(Sxw=9IJU#2t$hL`&KgZ>Q(cVD3zh+YYXDv&)h1;!XW=$#aN~lZj7ml0t!X8j)ZnVrsuXs929@wk! zA|x6>f=yMu0Fj%6wem6eZ=@@P@??a#7& z0DQrdUvxv_<+}H?a9gMRc0K zcA8^v`c%^o+HpUE*x~u3_Z;Tr#JZ+nEZ@>;;$m3l->fw!0cEwtsKx$8*k9=WaV&1f zF440~)pW2Cy(WI=8FA;UJcFr<@;QcBa$Di0CFjFL5w5xddSf zi$5i<*+kBO&Z`I_@<6HGD%gF=0%rOenjKC{kSE4|gtQU^U7Bvg#y^Zh!tbU@J?!A~ zNUU0j`F8+iBZW(Y!OakC0G}XkR2p^Jf@6@(@UBBeJdwyd<$;RUGiA`;mmgg^f$h0K z0Nv~D`Cxsg#(BZ3wtEs2sP)%g`M^ET-a4fD#F6STop2_;zp>5Tu4I3xfb_P4v??3; zZgMsd7AxQ1fx*Ta<7xtVBBD{-WCI8d5KM~fwoC_0bRaa6F1mcnd7HP%f{qfV6Yf~2 zRzHRBABrs|B5s5FYw+gw-lR#>AoYZZug2nVpaDI{Ky?rZNLlni`^jDhy-dyOx1dE^T^9v$C#gCl=`L$ydIzx2y2%3>Tf~08O>c7L zt&!YjOfxe77R^e3`1U73(2Q`<_`Io3#e)VQ0q2zMLIy*GxC+E=21XXK%aLm^g828i z*|f1Uv-4j~axD1)4-*h;7`$sB#gm9jkoWs`j4|VhA}6m_cNMU>rL~o%TPz1u@Bhv! z`#3ex&2Cdml%)U(hY8Z=reWGBv`Jb-tqdl~%RJz0r+J6j+opb5t8w?g&OJvWHHwO0 zWkww96d+6-CaDoG1W1mRo|Nu63(-bsRC_CEvu|-v5t7c~cAF<15FEc!%h**h06R)( z9Ixk@(k$L~`h+GGMVLt*6>#1z=3duhknf1%y{V=Bbpmyu(nhpVJnMY`wy#I zBOAJZm_SOPX3sXevSmZ?iHI305!60Jgh`n8(b|=f$>w!Q`4{VtIL%6PT|JrDNr#B< zMF;?oFZ1=giRkm2t<4b*CT5BApH7|GKK3wf*UaSBK&$5gr;qLUcNQyi62!EwNmJQ4`p28yPd~(&yKsq{dOyT|#7? z$>qIF+zFO|s@c`1oH|DR3fmgnJ(7xmK)8PUNiO14#E*sL6p%xZY99>(ea@^ZweU^* zY0c~{7H{IS2E|LP_Faps>1=|*w(N*+> zav1)_LPLE%y4nFNPBS!u7`*xa8kc!&Tp=zNY5>tbZDT}hEwe0N1C~Nd>>#^t%+Upq zH~$-JzZ`VUbK5O~6%IpQ2e|t+11g@7#V=9Hp}4f}no%o4`8xJ; zbSXB=PDU-&=ZpJcV5Ln&%7;zqE zP2JpzZIPz>xcK{{$XqtU%Lo7241{~ZR6HF?;?!p;;mBy(+4xE-De!F@S?YfL*8pvw zNZK>v6)I`UTC$x%#Ez_T#!s|Qa5u9B3t?hnQbuVrGp_qsrUIXW;cl8F+N%>w6N>*> z#p8mZo3Q+R=!P_T;2`5yS-L`cE<)34?#~4%dKVC?j9rDN#rDzpKG0u>ca6zS)`$3S zn3sl}`RPd~7Rki&>DKitNK$)<#4i3M=Ifq4m>*gGfrII3=r}{78{!|cI2l_4J%?$s z8epXFX7^8kc`V!!D$E8p)@1NaXtHv3M z|A|nm-PSuaN$B9F`2eNOpCA4tl*~sZE`U6)b6kD=#*2$k<(~!WFF8G;WB}?fCcvTR zxWMeE_D<}7dt@)JRQdXEL|n+6YulQUo`hVK3MON--45yKLP9y2c-(}R-^*nzy3bj!2p~Ug9dmBa? zD?%4nL-l{3iZ=nlbkd-9Nh7 zMG0jcG4S=}h!59>Px;y8acai+Nq>Fs4+1PGj2*jq zAHF4s$($z>Sa@gFrNk@}8M&ddfQr8Wvj2GQ>o5KqljLF7%?vlnaFCloVH^1xC@7M! z3J}PvH)?T{J4TzBsYVHizL zuwCq0@2`ix05v-g=8(}{eEQ&an|eq+-h@*n)D8j^27$8IBc2p7ZBmEI;=2qhUaoNI zxBJ)0N`q}5y?q%o*RcAg_~%7dFLFO?mH*Nv@m>JWhUV6Pr1CwM5YO241tvZtXSGQA zf)Q+bPo$Z0JGkLAWo~Qz&Bd@7i`CAEm>S)`N)}}^ErPFmffBxF@e%|a@~B~`P04|o z%`UEIvor6kEC_f_ekGlOv$~Smj6e485spbtT(WzQ^1FnWILvFsd zlhQM9ItSLJ$^=#r4W-L*ES?MlFUP@%eY)1x+3+Mo*liTQo=vu}=mUvf}bME4YGuu-on6{yrs9-=oj;-LA>#pHP#*M(3}5W5wdu#r zB9*9gDMz_zvb>%ymRuLZyp$2j!E%6~Bt8WTExJHDSduvPxR2)J1$6PnS%)x|?a{Av zc$Kw<@*JU85d54D>{zdF9%E+t$SK43#9c_tqxJ0u7SdR)QWOJB0tZS&7?QMcI{yB~rs!G) z1f+yU%oFHEc&gw|3dkWja@t1qv(p*(_&Sf#kI`N%OghH7JOM^=Ze53fOz3M3_V*uE z7ERO5?kn2N+BQk5t_rmf6cs)_49-3s_>EZIJL|ga5M9E5k*rNOw9I`2L*8HKc%bpF z{UvEhWB}KrL&g9W`P52bHs+_@9>OPK8v{ZyD>P=_9^Q~tiBcpf251v0F!(WgqxIAx z4%S}b1O96NAp2^@jlLF+sh>ToE^BW)&J1PS`-@7sKVYQ_PMd8pCQjiCmg`r*&a1 z^a2>zut6qdNtl(e8%DB`VzYPBS+dbiXT`LD%sa%44co3H>vyOPA0;au;Q%!z1u9O2 z|28N}T1&KLYy@>p!Kb6ez~$?9*f@vHttl)Eg=BEC>WXuGd3FvO&UEoi`R@EVLRXC~ zeSo~{2@gXx%fLCWaGZWRKU~O@(QbWA;;(xrg0b|=j#L3BVjtc~iB%Vccy}}y{;$9m zczEi!^FLw1MWdbo4^KX{;L@hFY9fR@lHQ~@Xxv;*yhQo~Ssea%#<*B;YJAzqE~C>a zGIlP=_vu)s`KzS4WEfAI)p@Uu-yQlHLgQhxdrO6|JK|fid=XZ;!>@AHr%wc$TG?k{ ze!Ku2Pb@NCpeh&wVnv~m#YGs5;Bh&q>HX7ar}7lkG9su?#Gh>Lo1>5_<9?}#wcBE!;1G^BVdoITpn%vru#vlLD#QF{063t$e4YZ}3TH4;3yy(+3CGbr6!6DoOvUszMuaOX_$a zV7BD>Nm(vDFIzbD0;dd%91MYbun>aZ4TSRt2~v+tLwkl>KWkMg)rjvLrCrK<5fm=0 za=r84@8f4Q``)?VL4qA1)``j+wS{^aC0*Uv`(|IFI{zcRs&DH}cSzg|bXWRslWNF` zjH|m@r8!?h?TgeWsDvhvHRM$O3OJX!z8_5aeN+06I=n5hvLHxS@J3mkQw)!lZ09vy zq8m^WB{kI=iB5LJiw95U(=USB?^vG>j8)2$UqYda>$jyqENmtlPR}at%)LG;j6V9j z1s@uth#qY%TNRVP{ESeJRM}6}A?@eftaWw9Nv6L4{U-aVX`qx+X^YE23J`U39zb^DUY%T6W;U49BAY_XJ^_3#Sfy4S-xKK$kx zPw7fd^>q&EWiq;)gPhU;n2}|TwM~DMmnbZ^_=vAd;K3L6Kc<{-!1I{aNDHBydtqzr z=hhK$4I?8&06$WInUxb%>BfsPk%nDELDeHwg>Iqn_2JGbb|_sbp1q6}jB3{ecjkBO z|02m_1ac-9aUvg-p@QfTx3B8HdfJT2p!!Z^c`_Caxx}?Fzp}R7erIWK=*w&!@mLlq z^sP(Iacj{iJSLMF9`xe$%~Z*fm0*SFVQms&Vd{>-r`9YJCjYfmNe;EP^(7MQ9p0>g zPI+5DWBswBiZk}Mno4Ch^B|<4@%@Hr{nLEj^RD5=%XFJtf*tnKWDS;ne%f{s;hY=0 zqJk?PIBYqZ3)jkSsywE>=w8@uS0e{SHoSkGC*OiGA@abaL?+)S^Ko{g)H){{b748c z{k1n-_tkTUq)IPm0P|v-isvGs!q+b%plSwP-1@Z3_lLV`$Z;u>YIo1?Tx_LWOVf_V zd(q1xYPftG=}icZS4vrDTV8KfX!Kxx=^eg{UIcEBVreO7H~+x#Wb^HmU=BETfUL7_ z97iq`Ap&zkX4s|%KNbz_U8DMzYR#+^7UM=lx<@O#V8IWVOmNy`OV#VOLXezF z1B5zvb45Mr1f9DG|B1UF(ptT<4-?rfciL`UtjNe)|MdKu{1HZ3KY0(hUb5}e#5l3*_H~>udCI{B&8GJMjZDM> zJ~OBK9ey7Ym$e|wz>^D``MYxQ#JgRoY`Jg`<5jIX(kJN#1LDrlH`j-GC^v>ZuC`REY3Quos^1LwW|g-=VvfaisnFVUJv9e& zg6pFjFj%7S1)l6wA~YK9H=D7;>WzZvG~0{)<_+OuK*}=iw%X%cTYC^2<2A~MfOlpMSbOmZW(^$vj2xR`<0cw>ywhs?&tjt@S_9Dy_n5her5zom#t0!|E z_m7asHYaCQ$rr=wtoV~Z-b=dhb+~3;1J+2w6%i!JWZYJ|I#<$YY9ndbFo*A6OX$R3 zqPzZ3jBW%(19VGjEyJ`eK=s#4zU;JQjX#+b0t4iC_o;>FzM0VAlOr1My?$qBv^xuO z6-rDvTBcg>?%&+cJuzHIh9{NSl|YHW&cBK?z3ohc?LLX54=6J*^q12|7c_$he-SK# z z8ylO=C$NL4?|!N0%1-ZOs zSZ*WoCqPL>oZpylQ<`^V`vs*q>RA=YVL5 zS9WPlPer(jKYSLBS(@YH4~9;Dp9yX3V8ze8d@j{7;XFoV$~gwv$qQY6s_lO7%mLLf z5=mpq*}Ap;D;%4uD;&q^-L)qDuxlf?z?&oNue*GPPjseR(vH!$B?*WzE?$eACHscq zpZxI@dCE>GxBh4)5UrUz@r2}>w)Xc}aiv#f0@;;TKx!j@8+WmEb^#+}L%off)sg#H zZ4M)>g{^_eU?vyx6Ca@=&^)=*1v9ErOvAwbRUJ$0_{u9Jwn=uoJee$0RsG`D=I^^# zJcqJ^k1Tl!!<@V`qwSDIo;W%LyJ?AQ?(`L91UdsRFaE}IJsp&?BGb+x2952Cwq4XS zM+U_T47>c1a1+e$6vDQi=aOqio+|m8=MT;U&A_ZvK)eV?NcWP%FyUd8Z|<@>^?+1wKWffyBefL$5ikmEt?hm;yVY)=M>2Mu z60eEihw`;63DV?=d}a?t-`LtoNiBOoXJ%H@2d;yHDeImR;tJPgO7S) zlpK*?c^%D#4tzKG)OM9%&=Zjx5Q1mYI~T96AYLxJ7(0DB=F~oZO`ER?nhwM_i<0-+ zm47vdAz<<2`$%85vR%>aoo%AnUN-xoK7=zC*@4e<-{qOV813B1y)u51{D=WQXJ1ue zS!3j6#$a?@j%i*-uGeCR!`cmrUkEY7Jz3czULPu>YpT#KT=qs1%w-3!o1zE zsE|afQqMYpcTyHHl!E%PNZzV&KN49EQXi0ic3j9X!JeNsF$kjB{!W-=c)T9b90aS~ zp#=|WJ`YhQci)qeHm;B#kzh=a7m|jy^sR_W%vAr^ap}2mJZ9s0#hZMvU1L&ot7%4FW}`x=7kWA+h9P` zV639GLsNXCIk$CtD#zuo{~j+%JxFgGzr~`v3gSe3at&NM@`66wjlyWe1mfYLHPJM0 z0VhRv-}N|kQ%HPsfG3t)-FI)A>suCNDk@E`pg^zSnA&^J8Eyq#I+Ph@)|NVWXw>d; zxZ!jRnH-opDyt{aNr5xR`cSCWpy-_%H7pF)!J|hzLCO%NeOCQ0+p6pb$+B1@9xDV` z>)QQ;B&Gx&X@DqfRN!4lFwL7f108O-sRp^uzNlXxLLuGF=IbXRHKghPOcWXXHblr7 zG0I}q7Ke5k5G5QhMpZq2FH~)QBtr!=9+1rBvjJ#Df&dHvb$<#6!A1ZT`?jHE*BJ=% zHiN#VAm%NzI6$-_cT~P$MwFAx%6hc|-4%O_V`0=kG)e^(h6`5k5bY6_dshpvnASnX zVw5M&7LXimgbDtNdE+y%rY9VT1? zLT3s-7Hww#C$bt8v0sp!IHG>Y&$NveGcn=Rw4D(9TTY-TA8$-vc|<%%Fx3z!Ldq#H zfn_0)J2$^dfMeZIat0r$^UvG|3P+()h*^@oGJtW1Ya?ghN|+Rj3M`H?K<#yF(^+qb zQb1phe`E*F(CnCpJ5H*7Dd24;tc(H+X@5%p_A7hF)FM9z21Hdhst2KwUk* z%fPTSW-T8}y(y|t`IDb;yq;w_1^x~=j{^P1>ioYBDJuqivE!JgDm;@E>fgfgFu4GdXZsY)lPKOb4D@y9;d zB;X$m+CAINz^!_gMP=GEX#RLJpe5WX;UK z55obiPcqCqrTQnUB&XwYg{E^>)q9($@J7yaG>F8C7ByV zHKB*@7Sdnr()lW56=VgrHDu&NT9x0ghhI)k_xVgqeR9p?zo=@`jY2~ou5VzZbvwZ3 z@!(x9*jx-}8n_JbZ>(tT_uQ568g;y)7`I)(?Js1-wz?N)dO_xRX`*CztI@R~%b&&6oqgAz5dTWj4GXqNTn`C<5^#)ZdTLV zCF1Ug$D2t92OR*ao{t^;o=F`6E_49?gUYx`^)WVk6hQJUG5Bn0)}IA0!t1$OR2;UZ z^90T>kC|iKcq#{3y$eE$Tb91L4dS^EG(o>N@HMM;D_#o&s990S}yu;WfSX$_c0l zZu4M5;tCRykn-Q|ztw|aZhx?u8|rSthvgPdoh(7Iv}vJhH%KDA8}L(UCoKy7l70-b z-;WhM!^dhXo}EZ@(HxDsy_V^=oeWAXb6#dUH#;c(Fnpp^Us~w2`*!^tu>=&)YsP#H ztzqnh*W-kt)?Olldwl1Yy8c+vS4F(isI%xztsHl6hA*v&CAIfL#7Kd1Af8)qfHOPf zZMxq(D8-y<&mA#!OE*{;Rxl&?Cz@o`M%#g=rcX5Y@H9CTMk*3;Z3l;R*Y3EDRcF(d zs+Uf+=8k}xrY4kz&2V9Nq$n4(KZGz{R zG2YnXo3xT*{~kp^Uv{I5bb%sdjWpd&eRKUsCiD|AAkOv+RBH?j5rt|YYt0J78N62E zC?V^26ri#iDNadUR0${Vq@#HW_yd1d$3skiqn{9j%Gpdd)_06${_(ejs+qCslp*L3 zrs)|gYHu#_S{`{t!bL-7xjsJ`EmbpGL~eOe=|NDsvZg!pjnqM<ddOF()k;?xUd1=)umQIC;HP8Z_qIA-`MosW`ya3+s?hwZuehxZ&SFs5s%=Dyf3q z5`)ogc%p?q=4PqU5=9;sUbOsx*Om+|rswGY7HSvu@wd%g_ZAl$M`8Qlb=9<;!7VkU zs`$`n@CTnuWeQ!|M&>#>$a#$X#DlKA#V5BJqqEhv?+h95sCC)pYp#8WZf4Y>VkVT$ zmFCnTb74;jUPyiJskeSfx(U${Q(-kgAkGsN*|%qM8jgeC+TF9;3dPT?MCip~^0SV) zWU5g*UdB$O&!+5jrNp63$~W+0z+b@H&zfD^Pw~6PFk@q7#>)>SK=Kq8=gr0Uz&+#j zE}l^I8uxrXj-CV(~`z;|m|Oe|UR`X&G+Q8wWLC zj+Yq?WpD1?qf;5++)qc?_2YE181?Kv%~(Iohcb1+TmI3_WRfrJD{3HA zcC>S@tvv8`OF3cukvG(ADNB=Ayqs)P2OWvU{3VvaRtxy10W3Mm1im+c+RUB<%2>1gC)F^%k0p{8;D#4n)X`(#MAk1-RPjo+4o8m1NOiIIAU&bkVIzG*Ro&p6MH0v$IOR1zgDM~ z?qan5jH!T>rXL?J>(jnR_77%*dp2czP*E0Mkwxn1*t_Srym~T4vgGye~cg zP&9eDIS=Inw~I7sT%p+xI7@qz+-_HoByE>C1Co&5G(j^TVn4KzxXO~a5z7cF$p0gL1JzMw$>@H>-!x#g3 zB)V_P4567MH(_&f>33@0T^lw4HzLav7P#3+*_t}<`1Tq_m3B~Y-!MAK6HMbwFV#0* zoB;6Lwm$;Sl#=-085zSST7g(P2GGS~c}=lBIML)_u~t*p9prqESR@&whSegQQLhX; zV{iCD6x@8n^yVh+fB0Qo{Yq&em@a&MS-A1|mrFzt%Xc(4=>{KDJhEuMMKK1Ya`z@|cV;#se% z0_07kaz73Rg%LEp^-kX29J@G#E;on>3eEmPuj#vE{z2?&uA=nJ@`%ADMzjl&@M;XO zG#MiQ*@8Qb84VYm=F{O`X2hNa288>63OYRW`53Jrqd7ys@BFrPQui(?#4U73%E)XG zqLY|bM(a{En>^x=z_jPYdQwfsaLmZZVykql!)4 zcAW{QjBt&HK2;3T8ZIc6W4gq91C-15upv0%65{7?`i08*?2GWiU={q-L@*QlGT0@$ zE9qg4Vcq7Xbv1mc?W=at>-T^HN4#DVgZah|yr@U1HIlJoiLxqvK6LwU*5$+MsyOS$^J%-5Uf^8p9k2?(E2e z@~J)eD__UUCqT8}unx{8k=G)t(rvnH=^Rg;=?9qU(l&~c6U)9OEn_rg6d5FuM{R9B zl-^-Gp)LPfseHN|8%a*8*O;zKaYAazYXiR-4JG%1_K56ucQNt^`0iy>kF+qm;^E&D z5c2!7%4II^3Z~p7;Z+@12p>?Y9XTatn-{7uizQk&e;qTsz+Uj2H%J!XP{BTFghh^Q zgeR|uYr20v4Cu^08E?%c+KYQG3-ENbVYD#2OftCkeKJEA&5?DOq%eYjV-AV>hC@h6 z7pr_ftj_ZWMdA9tu9kf4<)d77&pedPHm@xDOh-tQ2sp^S6UO^A3 z6j=3Ecf0jIb@pCVi(XfM9$epmZ9a7UwrE*kynw^#@nKVec^{<&SDK#_yH2tI?KGXa z+F>~`Qch&Gm~73Gyk0=gJxbz31iEW>{jxnW*;ojX|D&%;L*Y#TG&$id%I1UJ4K8|@ z*_iN`lWHq&F(sIbX@pXjIHaG#V1-Hxr1NaiPsyUIyY`H`=QXKwr+lgX0n{GL1$OlCvbT-&~rU){#YEg`m-U zvr6|KP}7z-_1JGv-@KWB)m{i-xzuUeUnB}TRzA|tD<4V9AI&X$JJeNQv~sN2d23RM zCJLSLka}xNnX?6qE}aw7n>}J9yY-2=DuoX3xEZfn zgx?fR4efz>@dhH;Q1!aXcbjt0x`C*p5Z_7Gm*2kDggMtsDpo5}M2*qE1rIohEiJ8A z-VyrZ(2!-$v>NUxHT6+kVgU)PTJ^`=E{Y+z4E%jD}Po*kjpCjRj*#` z1V-k*MgfuY7B* zanr+ui<()6!`K~EJFZ%xI}YUP#>=uo(q(w~_!tYMs<5i;b4x<>e+BEH_vmDW^+K4Q ziy=5AZkpkqN$LNyk$6XRFLG47W$oGZEVH1b3KD;z;p25*O;{5J1MfOC6Ra{)IEY}| zozo90H?RIoQaRgm5IY}Y#f4!S3_20l%)bUH2QjO@dpZ!q|F&A5aT(dj(kcw)X6=(4 z#Wa8sR&!`nfgO9uMy}aAUy)NvlBeDNSQXs`qH)S!B1d_JMpPbIJ|N5qV- zqoGTXwVEoK73I}oSL~qfx+Sg@Fm$LSl@0Q1%^-^Zf~q+rL4@!*)C9&Q8s}61bP1gL zMLUklWX(~P8uuZ86dma{nH&tJqZ*M27jqUAGj9wq=@5o0S7Z{N-dc#BG-E%!K3I=S zN7GPkZ=r4AaH%OtZI!_eO0R-lL(lc@-FV2R1C}yfZfEP3#}bh+bbhuZ8T7-!@w;5_(l=`4l8JTi?Pnf1*$pAC*P7TbWZ)&m4eF1wpMuQuzCQ zaDi(}npB)ed+#$eI8PUu=R8+&jNXEOo}nZa9?xx)-HXRW1r$Qz1k9`S5T>^e$+ zC>HV2>-GXV(X|rXH+~n+P>O}sZk@QHpIv!JR{fT({Wfndi6buG_H}G5m2@r^GUji=dO*tx`D%*F>L8*5k5G>Kpp=RTH8xvS3Ww{(w1(CZz zBb@dmm^yyt%|8e%$CQ&f0(sFC`zHLD6t~hx{Ubo!twrI~4sbzE9SQ8@%oJLpkT;9a zyMi+%-ESAyd8q1SLj)diozXM23<#s8>Qif8HlbsOcM;YTHWFU+Sk~Z7_HP!yO!p{U z{x}WV*V;Z=aT5phM*q`Og9Z!jF4_t`m+p!BmV6I7Gt~|`;v9XwgLDIbx|M@K1#lR5 z$cxxdDevM&nZ5vftIg%#x7sI*9F5@^o&gu+ zy{_nI(oq0~g@0F{w4^+-o9!Zr76faC%FeKHko%MvHYD0) z!$A$7VRG(5GVw_JI+mXNK0`aAVCXx|9Z-3aq%?w(d`*P|NhHt%M@q-sKAEJb8E_{m z+-w&y%I>A+w16sBwae_Vg;ST>s6k?}29J5M4!cV9;6&BCi5y<>9)vmHB}niC$_yeR zgXxpxEajL2ew_A770DNAQ}-V{w-?I?%qr*n&6!#!2(yVKOY#6u%JY1-wWj{vC17j@ zew57$3JPyhO!R8G%2Lu!*~gUqFEGD6CLAyhN~$F$7{!Sw{aA5awh~N)c7C1-ymLaa z9%qVEuB_553XLv#<1iEeb&r>E{tx9vw8-vjz%IS8o$t5dXzrov0Le~id>ejNuu_#epXl&eak;P2@>V&3Ig>> z$TbRHRCIJEy}rzw$K1o>>OnxnEtB~?EKA|3JnH2#IANZN!SWXqO*lxAKas6E zdxL05o78CQS}KXMw@`A%taGIAKUMc(wy?e6X$IV@LPV6$2K7Po+*rOJvik8Ey1nBb zXkuY2u%Rq)@1?wz{HPyYOf^%?k9MJL*re5Jbub9{o8>#6;Xz;zVRD2*zaUVC7}G3h z4c-)hes8sy{F;tS$lbj;{o|MzC6SrXh|JxPWQE~C8{$MB`&%<#hIbK_a?5%1B?fgJ zj8+VH!@k4>GC5RL2mk%-qpP6>EJc~|Ip)7n`Sz;Y7|YJ0r1x>T=A%)a#FlHCBN-tf z%XLM&8n=!H@Z?nwSo{n7ixbJ<-`0z}cgQMQv(wSZjN7Ebuf7QxGED*&S3H&67{e=M!+S3>x*5y>jM1 zYxr)<4(rTn&{L2ztmkAoP+aB8jFOp))-W!;+(-p52ufGh)N!;b1e8OR&94bgt0a3u zZ6@U<4pb#(6?cw#nK*x}7q3qM& zU~;F`e~m8X;DDz0!Kdv&K966JjAm-d*loE44rxeF_&EK(E}8ED@0VPA<( zp{1i*WTBS*jIap7pi4VFt#&w-$Dahbp%qmFr6I9#mW7tYplG{6qR^diV467~<8?>U zP)TpxRCAh8&J|X>10()!49FT~6^`{mimE3s{Pp^tkeDPzQ9ipN|I{LAs4vx^gq=iS z0oJ(X5a4%Zw|bv@!z1C7@K3!KwW=x~?;}vq{l~o}bzJr&%Uu0)iWS&5=V)A}Xu(+Uihk+D`oSen@?cjMu#$!GYK@EzOw9*X2s8Q&);Q&X9;zHMC#TqOwRRAY7v7Wk zNe5nUGLI-G`EqdE3C57<&wOjxeI)I!j`4(k)6n@Vlx41ZO%$-) zA3J3$$H-=XQNNq8v18$!t~)qMgGM-PXu9UiQtQeD2lkWs=>S_T4{K1Z!)FM%^Nnl42gTHz?LMtF8;h(h2U*7 zj%TTU4`5ET)<_WOFz_8s21RfV%3XaVc{=WUk@AXtGJ26wt9KfV2l;rqGFlPB3~~|BYhU z?3oiiYtBwE%D2%4+?l~RS@hLAx_gB`CtO+wzX%&0mtBs0f{7du*hnvu-_)2@q>953 zFpWSNjNPUUPX$A}c=OqY>Z++5$K#7&*fumXu~I8LRT0(xlF}SAMd5QX@^+lZ@hJQIdNs!#uyHuoi4qk7-|Jsc%aG(e zPC6}Il>jd-gGKLTX{wMz2xyZMg@QPN$jT_1Y63JVn41iMI~CqR`>sBax(MGpBw%8M z2`I41DMVwBWd;-R<0YN<_b4w6lo#BVe*bLne17-bpd~a;&Fg5bVCx+Qm6fyS327n@lx}SC7&gzCIL8`Bwm$G7E;n+gh-Sm-T4fYNf}1Iltk2w~Z<4Mbq-V zs{Sa|e~_>d0WYx-Gz-NO>uWOJ4k6`qDYV_hY1AujN~9+nZkAr8F2k&h-jO;`#$Fh?5m^^g?=w8mPzq# z@i$F!3r;^D+-gCS$n5ml59NfQvhQh)w&B#0F#8bD0fPSJgxdg`B)>GMI)u>HSq;cY zDjx-v;97)U&tG@^O-ClDTGG z`pt$88wGu&K37}uA$jfr8207l$VRGkFuY>O(o^^8rb6#FE(jc2*iRE)q|(@|ViQTf z96-PS@qMvc)RF5K>HIhL<1PB$4sagdE{d(cN0|>8gA)4e_|B>x{VFExM;M$nFAn*5 zOB~caxi5xluRzT@4~si^@5c38bChARIDJ4A!qwt0?|)T}=;mR-j%@E)GV0=Eem4L% zK*_(gUC(0=?c)OASk}F|4d+A7I=aYW+p8|8#AHT5hogkUMg(z~v2lZE(XY5*7{S$W znOzZ(_-{pQ8MX$HO$qCS0K?_}=@OzwZSPt~%GF71^oxwWnfOO%lN$}FkEJFbFLdw7 zp5OMR73{w*^#Rko^0<*hbO&Al@w8P!B_Z2#P&r&)`FVEB8Nl#egTi(%Xm0-Q1crvj z5biNrynt76#lKF&tp6==ItDbye{!vUIv$m+k8=33b_OTJyH-H0-cN2ZI!e3PVya~| zNKHeCpr<`ROooz)I6+y&11)w@l7%(Qt-+=RkE@9av${7i?ZQXfoY@32YuB(xQ71I5 zdBqP#Jbhn|qPBCm2EAeH35mIR6bL*EH!6En&d2aKW~f`*qbCvXx8aZfzQxc`oRaDV zPqrsW(E0N$FHC>dthz+<4~|`VCxHEBq}SsTk|f~mIlDi?FMCgfmYy z2mfAb5J0VSLy$a>#chb{*xxp|-@A2bQ!7Z#Ybh?09;Af+43*ffsqy)-B zZVK_i1hOYdW5nOf|UPKH)8;|dffAne05etaYyN%`+~PBu1|p_LRS{_}M0aHutkFwc~4 zt4I`r_`#tizI{7^y4y$i~7Rin&@-`vor^m}AS$cA!f92Zr^`CUB zeV7MjoYeo7dATnAXcwe(90%$c^_n+6qLqm(PID2*|6ZkO{{}1P^_@7%{tDMzVctcq{ECuFs?Hr@U}zF?|=uBCqZ zQ_O+e7PsC>#J-r6S(2CAqXZYqpEuM3`b66+;XMQ)?OnZZ&S=fjd3n%Wyq`a(*L@_s zlM>O5ywhb{voiW_u1NCrYP;q;05atmmT?aJa1UFd3JYEIHyO>pBkJFW&PTT*W>0Z? zZ;ylNN{sAZJz&gQTHB@(0|Wa6d}O_B1q9`&UZFkBUh$iP1iml-{{TUXYs_PQMmZfs zH%6mEHdliM1Ss;mPFm(S)SD?3BFt}M3A>V+tLKg*fpmd-Gds+0V(6|I+7(w?a$9ac zPpAG2u0PgZ@y+z%mrMyJ{5?!d?eh8adM2>l){SDgY>QihUh0LFY7V$;3mwfC4B+o> zrda%#>)W{@*Y;~BQcMeK2O(s-|L(tG@md#?P6a10$hBk`9qg{sgL-PvWfQh^q+GDp zhtP-G%xaYe=)e%}ZwILaj0TNkHtxq90b#T+9Z+6l!lUMY;lsq zIzQ-Ib&nfCY|u8%k3{oxy>Mbm)H$moGWh%nYpvpU3p>Y`sC{h8v^ThNZySA5Gm0q%n^Rs zGVzIV7KV;vmD}3FTTksZb9%5MWb#jv`q4k#30H=caT(#8Z_nXfw2J6dQs3p3CS zT{Q6a=0o#~sJ_V0w(rA@1FB$c*S}zQgz+Zq7H&V?GQsp7Tf&JN-sb4ahs&gXUBLpO z+LUyxgI*o9Ujw2Tc24h*(J;bEdjmyn!lwrr|G{pI z%hDF&HL`p&zpN7JcTf#OcrL6>#pad5d`|a`FayEn>+D@MPIaDF#8*j8Foq@MOV>$` zXtvRcezYCM7#E&D{%7H^n~a-DQRcR+uqJd|p1J5dr?8qdZar&$n#g@*FisG4_bJ2H zY{=J6Rkm)05-%x+S~`-)i?n5uD)sVT7lXW-)UUk-RawB+LsN^$lu6!EE-+D{$ zKH$3TM+$7i7Bi*bWr_b~1#c?x2!Uy~48urk3!?~s!_FP1fftr{K&ji_NC+00Ac9C( zu!iTiFRMK?C(G){5Edc0t6<|C(|=v2geOq>$2G>e4OcS@kH1a*#VXq>r)P}%B9;sh zTGWlFbu;&vUa3pv=u>OT2Ys7TGz~)@eltUn?l$Rc)M;m0UKF{F3#2=u-vzj0=bP*` zuv}`JvP8)08MA|{U8G%QYf~)%a(#HS;v9TUmW~uF7+g3tx&=qJgF-HxW$vBXL_c+f z+Dl0ZqQ%}`V9np@H$Q-4)7N)%6yicbmXF6P={2_vsFl7VmW|@y``0*z;H)G^nId#! zr3juCp-~v9 zyDC`Q5nb5H(`%XT;OEV*^`fBwS_TL;upma~%S82Pk?W6#ES!*8H!0b-@tMs?LZf}5( zkohcgY)=*_)ek07Qgq12c#_1|<+rY_+UMCa|DDc2B0wBuG}y2mv+9(alsVXUMplx- zT1{*&t_Xm1)Gtyq<%WN`&I-6i^4kk|i(|yL?2^6K=P?U{f|LWFE{AjX zzeoY`b4tx|aPm^X#p-IEn;w6gwbtkki|4C{1R-#;*XVxD7O&E9DSUU-5r>%X*wfPJ zyBs?jX9IfHChN*%G~|j(XXQ6#@jMw(YWrM69fp=p$(kvm+$|NGL~Nx76A%2&Wp!^Q zRx+aIEp?Xjw$QiabOy>~w>kw9)~bJ)eZY*$1W*`-!NT%F;_(mACSc~BJTfnB$2+9c zOKZ8ov2D%%8W`EL{Zq4nU>8Wg7;)&!vyQlzML?^WYc#+viGmd+y6BKD*p(UummhCs zS}B(4^hVU+vO0R2LOO~$LfYF3Lnz!GI69HQy)wvq<@Oo{)PS`VGC<8a$+8_EG!zFCCgUQh%}YtywczGD zwR3Yj6y&ya;z}B?4E4J0H4U>F(ejRN=E#sCn1^a^l~!5uuOR^b2n-nSxYW{!q9F$; z#gYm#nU?Olde7=&MUv9&qCd;dUS^MnK=9viL=#w)=U1h^d>-ObqyI2c^v#}grV~B_ zL5M-I-o>b!`NC_h(&KPD7{B#GFIi5bB?q&c)i=Qm9c2^yk!Vo~15}X&B(b z9MO+e&n`nJ$vV&b-Nx;XG^u%pwqt`(`6F`Oa?#~_M#Y!oomC&sV+&Y4IWY(HAtp!X z-J|l1R7fuuGC6V?T`vJNp%R>sQZ3M0-hllc16880tosfqnWj|P*jz_&g6%!w@A(T1 z4h%&K6R?tg8@(E>#gNYdwznI3B|A~kT@tK#I{!y@uwfoVC)bf$t5DTjBumC>`4kc{ zCjg?)?C52&(e~Vl&&8t!h)MzF`^lm}DsvXIF~gSW2|H^_z}*)uqr^{1o0kYCSpZIp zy$eDqoG^K+pWn`|l?{_#HoN@otQK(&anF-2TArZSm$9 z>fz8Ug5|Dtv8c*`VFaPPGYoCc7NxreQ)6c~gmJlrsr&lgOaN4KRt*f?P)Og1_L+782Ja#i4$3x;X9 zAqB;n;m#_U%hy;C!GhN?^n<`Q9Ze1`u(yoOt;4PhO2u}q^Z3;=AA|jNqV=WKN&wCAH7^Lm5T2!s>?L&nE`V;rO+tYrxLP9qqRz`siZSbg@H1M zzE~g33Kn0>(k|$NYo1qN1pNOuhT))wlA?)z(+vS-pbJ-*GYf-XVu6%k zNxtIKD2(XZGCxbAJcJahN)jUWO!5y2ZT*XZt&d$|*PHVg4R z^IAt2Ahj@HI%;Eqq`rN3+350(#}tBW;cB5}!3(8^^_U0c2j9CE&~%r6FQ+f&Yp2m_ z%z2UYP?9`yCGb=u9Q5tB+294hCzzxP!bQ$B#HM_PH&0xGU+6)!(}5ZP#}Di}DrT@6 zT)D$|FSc;QJYwrQ3&;n|zETyc>U^3`X6erHAd1(l$U{FhtWxw>uRrJcC1yW#864vb zumVxV45kw$`}|Soqx_>|t=jMQH73M_NkAf}?Uft`(uyIrpC7k;>4=8C8A-V8TijX~ z_i<8J3*am(=BrzH5H#vw>a^`KzjVKKc-~gouZz=f#SF0fZ_r_AhZ0LQ^YO!b2n7%; zmpMDN;Mafc2m1kHQU_CNN9ycGv0)XgJP|olS>Cc@RTp9Ad}FY{B@Hr$xL`)q#-q#n zkhD1)ByKs+!-U12(4e~^59qg?6byy0c*V%S+rzH)z~hV^(SVRCl8cQQOl&U55(2H~ z??Vx7CIsi@4niX1aL;n7+TfIG?@tH&Kf(>)PrDf_qPC+I9-YQ8z8x$6th{XJ5lkPA zql?j0|Gha30Nw#94pi@T6mTvW%keD;JAQjKtvxzY;lbQ~tl$fzBEQ9^ruRvLCu3WC zYy)R8oW{0|$&M-X>#t*>|1-s0$AvHPFGJqA>roRP2Z7ahZ`vq)P_(9}-weA8>E{Z7%tDAQ?2B`X#R4$CECQ{fwsPfFW&qfqLK*jV*8xZn{jA;lo{ z5IK&~$-DRMvS02;O@@Xc!Oq}24sY@aM~LK4MzL0f3G}6n z${eD7^&RL)4BVYqT9H->ijM?O(ei`x5cAuxIJqKF*e;MKG1ypNNCvV6uxLo&y!r4a ze#yz^n5}vX40l2_B(mNlX3&c4^RAkA(*y@Hw8lDOBQ(7x{pwOs@W8r?rSBR-nnZSo z2%?kOEQpA@-1(qHWT?vz%n1@yL5yte^qM9E4M-J_M}_OFMG(8){NtCRyqkA11Cvt8 zxmhvmdZ~;YpX-=!XFK?1)ttxpUmzp@fS~>d%1U`h7g_k5E8IxdZ2BRLC7H&5;+@6)WHEEi~KI|&Ck{eGi_RJtRTP+`C={2-|S29COrJfXg|^){ClNnkw- zEBsQ~1GB4YTm?Fw`wMy`jsE87FLRS=`j;)Q96N~j$Qi7uq^VY{OhTx7N-lO^YlwLF zwcf}%h!-)T_>A)({}^P~Q+M+W>|o0ykI^ZC>5qCc1k-nW8#1Y^Ob(FakxrD2Mw?(8 z2masI-fTUm24^r_em2J~EeYt?CZNxHrv ze$3>qRR6)^qaA_dKk>2P$bYdhtT7BtPo>Pk_PBD`bo3nOp6WZdYmt8=Pszn^pVJL4 zpYJ;TV9VctcZ7Twt(BI{9wO7R4WI9n2?J6(6xkdnxocw+`!tO$S#yRouDA8l4mPBqUb$n( zuuCGmj-gOB6OE9TkR|LcXjEUNeib~I+5)k`}7P8bGs4W-{DbaAts&T1*a z87XS1>0Bu7POk7NN`MuRy3{s$#cryIY;!)|k-wEl**zi+WQ^}T8ZAAt!7}>7yC( z^=s;jPjC37&V6ccja=M2GeSozb2cW2q6Nc7FS3wV)V%6z3|4P+!5d9)Wpa8$vg=5w zupiBzDOf{@AKdo~F-S@hDEYgRpd_idDtZT1NQ$~^llGO5IgjDujK84SfCX(`ASdAW za*JM}+qX&eF=XE-ASOSOA@EAx9FfZ05nOY{sC7wo^(VJG3D?SIO9z1-SXZTKm_NBT z5Tneli$PUwpE0Pf9(rj3ighvy`ligHix9a4Xw+4Z6MV#P9ze(!p^EmH`OJyHXR48K zPDA(>S^ks<-0zL9LaUkWKw)vsW1E+hlIRLzpVbz4pu=9H9U;XOzb_yw7!*r+p9dd} za>-u^kni;$^{j^SbtN7LF7&d`xOAN^5Q>sCre%Ah%k~50i;m;A{!dba(xs z4v-UO0Vcbep5d<$L##8j0xQvp3DY+gDtTNa7UqZyKv<_mw+uHNi^7~9}(<92lYpQX(T zpm#~aum8F!@aZxa$iNl&9(?$EM!2eIYdjo9$1uYGgmp*3Hmt$BD!j(L=#+3wYGf$^ zah=a3c0@g}oz_xo(^te}{xUFEPNkZ+Q!D@dVgm z&h2Rirmw)THoPswdDW_96LqNd2!kLlSR`D~uJ5<4y~6ub#@Q}#Sa3#hbf&SjW=s$O z?3a;z8UN)I|I{g8@4i$6rrUt+-|HyFj;4Dd^5cC^hF!FPfNi~s9B~b*G2#WBFS)D~ zZx(ij$hfba2F-dOqn0;Gg;S=KX=GkNK@X(a*6W@L;U!kk)nSM@F`>5IPok<9D;)-g zpCEiIwkcBOEgWjdDh%9WKu=Zx3=NI|BOE4Eu}_f`P+x^BZrh_m z(ZI&&NF1tXmdNC>37}Ihs=_KH@t<5T>qL8?U&4_%O%tk|rEpgRh8=`%Z3oNc@93f+ z=-|{uj4=yYq3n@DDh@Xgi@}n2gwDz86Cvll)Ozkj<5?Oay6R~?*5tRmN9EY zN*>?X&{3yeQ&4|!OyxRhUEt&|xro#{H5IE!TNdcJAP#f0`Sp?l6kN}Z6zH>l^hVgL z#Z*w8(b(dX1)+;=N^W_1u_6)?Fs#GW@!2Bx&^^Pu7LJwYaiX>j>ojd`TL~r<-Sl&Guo=-Z!o@SL7q2g@zya@+_2+q)EpwC<1qO`-?~f{^_#$JRT-w$K z_5$%0*N#6gzoLYt`3~4M((@7{ELB>%)(seOD?Zr0mah;J%mvY+WY7R;w&9mn{jLRK z%!|Uxk>}(yscl}V!u*pvNmfN=f>(;WDJ#2-EjXYnptO`WV6D_7Q=Z_?Dh^PT76|9t z+P9It`KDwNUzBv9E79}pPN}6*WwEpLq}odciuIkeSkL4;sSWQRzOfiRVTnQkWbq&R z%7j7ylh{(;^h&aYQ2+3rQc!H!^sDk?cjzjSwX~Vw(z@OFqp{881>*kdt$U-$(di~v zikpt~Wm_T0Z(*Ny3&-9|7(48A84K#$xiqqfg2%*o7fVkF}`Fvw@a*F`3&gUV2%gj4P+}xl4_cm=)9VJ0{VP| zxF)2h1QBiMJaw?qoRt$%hr_62lR&js0TF^|a9fFTZ~To{NPHY5w&Y5~^8mgPlCbOs zxn3{X)w$#vg}(V#(-Y_sfKhonPxXP0GF}A?JEQCMaPeFH_f8l_ZVL_Sr)jOdsFy?6 z$w7KKm~6()b02SelQKzCAi%uetgNK6ud_JKNE&Vu_4(q)XFIxnwNRy zbVl;5-sg=!h$S0|9;pZG1LG} zqa}r-Mb(DsGv03TUj)){58qG#1UJe#Lprb5DWUkEVlwHj*FP)l?4ka`gKc)l z$@LMMqrht}QKG_5HTcfv0xk&HR?7w2?^-Hv|1}>=f(EN?0951-;~ zmAU!qE8DC+G|IoK^XglwD&Z5OLTXK7IfsoewtI23Crv_3HvUeZ-8$9rClSdjMV^QGtXVO$!S z(Rf>Sw(-}I?nd_ED@mQ?Ct;a>cafp~kRQL=^USq(HP9=dMHvJouVD~UX>(sjsE5v7 zl18JGXtU5WhN>2-|FgR%Au>+4R-Cxuo|3*Sg%=P+!;IidAWT})X;OnyepEGU7Df8?I=KrH{>KTEhE zVqBYw8Y%C;LE-@)R(V`CME zZt}kvS=EGQsR{lfnRes27QvcRiSOLKtSQ(6v9%6_(GGxJSZSA!IsWBG7jqH;0tPI= z?P8KhG{QLIjX1k(8X7o(`IL;Z*1pAkQ#mwe-Q_e228wIklvw9OWxgwBEjC{@7ZSHy zUP)f0f!IP)O2&hPL*hXj$vIcDMMa~s$8I@fet#pCiOqKn^uW;>B#%e-<jfX`bln=INc zZ75J!5T%MB@UUfnN~q$MV68rq73@q;vV>tewVZklz%|2KOOQ`seAh7q41k{Dp5j7; zmw-y1)7dy?>|w03P2q%CI#r3vaYE`4y%yc`JnZKS?i?Zr(EiQA#S)=Np@6C zB^uP!(!X3~&)HmZ)ZS5)?LHkT%HHU-5dk&-bic1NJvNJ@Jjl$3zWGsO-w#@FTp(w= z9^1!>w3w=(9-G7Us)TMpZwx3ggd^2KLYNK0vaSqyn8uPT-k6obL8}R9j`_BkkC1}@ z>lL-NEek(r`-a8qxWkrojj#U$fO)cNlSBPOxh=eHV@eJ=7^Nw zepkU#EALLD=#WBoFt=AX+>oW!^bT zY{tRjKRh00J>KLgH`q@ZONapEGY_P!s_ zpftzHmQf}*1^oDs)A09Le4}CvfWx3nT6F1Nzk2mvHF9O;?Mvl2L>g`2f1TODGXHoL z+pgbdi*WOT#foK6*u%<^7G_|ou%zgHft}gyMe;6+&KQA`0WUo%p6;%{nd%g~DSVQQ zo5InnzPleP{Rw%|MPn?l<{4alDrdwyemFqh=J>_E+u*7{rxzXpLq9-TLkwWdMu^s7 zS6ZS#4&)0g4ZO&6%{mrxVkl2LOcu85UEB=TYEqZQ&GzI^yNKjSYiY2fJOrJ(#N#pP z^V=X81cyS#j&ayLcl&~!RU}|e6pj?Hd4x9`a>3@QL&q^`+^c$N?gzHT?iE0Jsjxup z&4FSwa{79PSvT&F-KI5nOT>ecMW^ln+HpM2h40sZE!I` zZ$4qKgc*cRoelOMY8)zR(4wZYz$CS?E48SvC$?uLLNbR{hSh)%R^TX@Mlj#~)*0&w zcwMbZee|5`>yX}?h9*`-1@wa%`R;mqq1HPa=`F~Z-GYteBIB8r>FcPs(AwqQ_1zy3 zy5)BecRTP8S^8N2p9!c!()@{tf}vA1^%L06a=NQ7!R-4T8W1$fu0KfQcthaQ0fs z*J&vD{pWm3s)VzQz&)b9bNp+I5WPi!Ad2T~D~fR4k`8Us!=gv7$DAb&5wZDnYP9+& zEe%O^_Kj(1P?a_8T5N!6sxSqoS}W}ROzd4xyM_6Z=|~Ck_4Svt*Ewl>78`90lsm`sOP)y%X=`B zt?xp{j0%h{8x(xuT0t+F#C;Qg0AV0>;a^2?o>yOh);Br532Su^mi}Q?FW#G%V*AO` z+fV!wsvT`Dc_Ev4K>eFv`kXKj4{dtEultyPVV6cDiMc34Mtan~5Mi4HVKf{-{xdv0 z^SUnl--pj|bC7aT771&N0KppA<@;&}#N)sMYPw$)lefi`@mGM$HE%=}5Ih}$83}CtsyjMCnDkk{V5CwfZ1QGY3E-OQ= zc5B=WT~7doPUR%-7b@M$DiDxZ!$S==q9c|h5o5h=t+~~@yZ7~9bT5Sj&105tXNVF0 zcOqN$z9tAw?tO7wRj%2dO1eOp<*?bc2lFP!P~>O3ABBVXuG`+eS`aCGew6R)$s^1q ztb9k;@cGOo#%k5`qApqTj1|^5HrBFNO8w3ARCx z;T~8M-c)G&&!Ti|A+yS@?+mM~lOZ?c402uZ$Ye&b1=Yy`CCm z7B&Y7$^xn;fSSK_i2P2stKv+#p@HDu4nlBd>B64P3N4NvWy?+~Vr;uw&t8+vQ%B}X zy`T}@j8UBP!Y1t7Uc0sR{8e2C~qQt{%Oo;D+m%51`AZD;`S+SDs-Q3>cKU@xpWF(DA?y_^OryWga_YK!*YbHWc zf)W}|A{ZwNKL8kNGz4P1M0wR3SdFlicUv8;MdeHF?iF>KrXB zWt*(u`lIkr|8c*SRQgo3xQ~v``(hCAYFUcexjMK!L$CDS0T0H@bHGXCNf0uYCD*jG+ z!iqP$ub~m>eQajdc%bY z_|VgAXna$y>1lyDgP6_?aa2Bnd3UHGdIj3NiSIcjS#q6|qT!3gSN^=Goc7Va<@JWO zRLWk?kB9a44%Te&Y7g8;vdYwj6q%V8D0f~R&@*^5c{`(ha^jX!=x^Z?mf5}Bz7$(( ziouBVFGh5lK-INsBBk_qu`#CdmwZ541MApFG%j?=~-Wl~70M(A;gx|gsI7*ARx=bsdxP#wFo)|1{z7!G7q4FJyGnz)1 zEE{y!DsBiWl5BwA`uqm8DqDb>gyy{izfRDn99Mv!T}U2s4%?Hn!jz}g+ttaE^f+-g z9F3s$YH{3Z!w2-S#TPUWh`*1t07`1Dq8=72@8ge;0@I?A+>COJq?7dUz)0uJlNqd~ z%4i#ol}m=a5i3_SEBkPNz5_cJ;U@P`j7G`&v$CA3ME3E&@0eygUHZ^LX~2X_PM$hR zsX#8T{|0nxa9eiXb^hV0XMI3=*u;{@^a3`VVB8> z6kX6#SeH;Q2yrou^tE9;3*Z4a*>)w3C=7`DGS8ccgP^1ZgpwZG%>xX|%_;@k^Fl}X zE+u`s<0q|esw#YfB2kwzIRd!|>Ekh1dW9~tL=fYwYDB~5>n28ba37S#Oq8!Ky~ehF z>_I(ae*z+vm+t{+r8ey3qX7UGeoBw(7k- zTA@LJ^8{g732|Fv0@O3?bfZ6llSa3O*po&4G6kL=R8}b{e%~)~l-+aX(7r7mp7EwH zJk~zm==_5ZT>fdxBv;D**tI6%OC~n%{d9PLl^ceHd@!h z7>wfcwLO}57~Ea23T8fSSgb&KR-LpiK1vfg|3?ZdK}vX}D3O-ws(?Gz#uM*0w@a;9 zCYhH--$gRQg)Fhs#NviYcxrgu0WHfzK_QVAu?=w;9aWCJ@4kgUKNieUx|9^c2LAw^ z%x99Pzmlt+VS%JKU82>*Uxp*<(*TIA|Ej2B5Gs_Kvt_!^yzdG7>UCwvpEoJAEg)iU zUMK)36{+MFKiBC=QX;e=Do;B{^7DXGMIV!y*!35H1fT8mJ*uQQk<)Q+dK|`1gr-`U zG{LhQWteQg>)~>$5<;XW2|8}sozmSiF>T*q-{`MlMZ)QfE7Pxmxjv8I+(bwr7E>K0Vf%IuUDDs!lB`6?$#iZUhJWEmEb36aQ6uG0;9_tJb9^* zDkOhYe)gpo@K7%K6{IMM9vvhkd~4~-xtzi5igo8@P11!sL%=x8`C8+fM?V9;Z3P4e zb{qPowhh)913H4oSz0V6Kq`0K-o9ujBheE<^h%g_uidRgVqYY;m*)euiZm8b#R0`K z>Ie>Jg$$C~g;b~xw>W_K)*Lz^M5_@JT%mI{?!I19 z8zL5s5E;&!*Pw=wi$C|Q_#85ptp%j$^OoSlq+Lge3n_i-A1LTlrLYhDt-NYo+Xy*R9>&R9nq|4o`B)RU<=00_2vA zo*`RLbuI>|I5B(or9~G?8tilys@L;Eqg$aZS*&ngjb%61wC3xzZ*jfC)?eEkxJnrj zhtA96FR{!5S!Ofe*^RBP!WOwH{qj-dluAF19I!|CWkq7? z%@oOZ!R4RKC0?VOWU*)ilo(EOe!5TsX2~Thusr09r(LG>s4*>|k@g&V>iP+D5|1>_ z5jZasU{CpjwklxR?e8}L>jc2b!RNL2gd)(feh4yFY)kXwJQVKF1lje%gc0w(S`c5( zQC{@?_His~Bl6!ji3$G2WmXu1w0&rPRZa)#%eMD@u3@h74QBBHCAIKDg{umQTN@|Q7_AJGP9vPw5ZwQ>EeN0FPYADvO&Rc*h^9m=}%rLFe!-@0y(j?sae z@kjCyhB(CtjQ>bJh6OjUXX-EClXgQ+O*_zRDRd_&nJYo=0fq%$dOma{*nB#4OG5qE zr_>Z=MRmNKas> z&A(qU%0m{HS5CXTK77V-_bmh>;oWYC>XP@by!84eG_;C~*ly2wp7a$oP zvUan)5X0yL3~{Z;(Pvb){CF;lq@=`&!+o=LlgDdVb2!;$_muNI#3#HRX$I2yFP<$E z_mNqw5TG!1L>YzS5Izd0;#vY%p{Z*=xt|KTZ*1BLqstXZvxg2W zdizZ3QtD#aTWC2*UkOQtb>=4sogOOWm3mbhmg*Aatuv17;Ch|Z`%eYCTIhFy!hTMc zJnNF{n(_#eVZZeDB0_fG_iI+f^oF-M0S43ihBNGrVRHBA=HTEek7O#&GcFl zp)yLcX2f)kX>5>EIx|k9Qe@jTqeWjEB+Zk2D+2tUFQP!mpOV#DRp)U&eX69GJ2u*6 z?^IHv6T{ooePUK9^%}vzoyY*gH^T3R{Ac>tfw8`Q zn~oR)N}4v$xKGE$fLpVVm;1+IeWUfrE8;o{MwMZ8HN0e(syrcIz*MzzPnUS+RDAsXWS^n>wqfe5XW=vJ-c1`9SlD;<}PLPx~gO=t@`9Z5h72*Hl`S zO)l!2%1R~;Ykl|&nc2yIOrrVe{L!;ZuZe){hv4@DR2-GE zAUZoAF=vqH+)obJKR<(B77+p}LA9!p2Wovs3mtuht0WgU8-j5R)NZPdUm6*Jl(AQ zd-S^8jz>u(^>M)e+QS1WV zliCtibMDs`1L3%mo%yB!u61gK@BBcnzNpua2zAo^bzohppwZ&%+Xz28Cn zm5?QeHKX#O-u$@NKCn4g*gsrxT=VMEt1h9Qx9`x>?&-8Hlc~;52#NgY#MQn>pc&YL zn0Zvc(?KpFu6CKSznV0~E!lUo1bBdS*`RcK{q^EFJ76#&>phKIT_)yc!3lRNOReBf zYL!{ij=OE%%E7bNd5cd<)kDtMjlI#IapUo zQH_z-p~eQzqU|$@n}W_#K~hm(!10u|C^D{VuO1Q~#+H*SO}!Wi%~=`C@k0)Jx@;AI z4I0Tzi7x@#x?xX;YvhIeEJVI2u5?-uNgeP9LPtMWSw5Fyrw|oC{4!b-*T;aCrBMnK zMw7L3-}Qkr^f?3@jg1laAioA*?-kPD_=QMCq4`};>k;2Ny5tN`i#OFoL+>!ruau$# za0@>_hi{-E@ur)!!0KDAT;{f>qm0xlVbx{GY4rXeh~_!~^#v zNG{A)^NZfCz_FsPWiHjvf3NoSEO|wX@j>SLuO+?}?u$>E7#`2|mViAGa|Unz+OjMR zNQs>fW^l-r&J#wLj(FS^0M-Z$Wzvo^Vze#q80og|cvrhW19VP_tl}&VLxF5)p_`h$ zF}(19i^0sW)XM~))ofomaSzDZ<;71uL!5LGt%9m8G!~!1C5Ks)7i7(klLMS{o5C;( zu?x!*BwAvfEx?A<)$D{LA^l2W2O(WO#$@zPq3;pEy1p@ zaGe;wHupXrXFcDXM&j4P2FSm&nLo!Nd6I#m`CHjL(i{0oOZFlq1bkdW!04-OI>aef z;C1GY&k~#n*N6w--$wCb?r%Py!i;{Ac#>Q}RYu)H$ej@#jKTH6OTbFZH_inDm?gVN9sxoQ+*_|$ zbP;F{b}@Kt^aoAqeevlz>?l1>xJv zd_EbaSoh+-NIZY}v<%=0SC|+up}a^dZDvCTG}t1#XSw<({hc5Ok#?uG1oRSPn!4t5 zQH+cADJ}5Mqf}`3z#6ns{f24W`OT$ce3#m+UD*y8@BmmuT?&pirKgAWoyxJa+NJ?; zsz#Lo<`1@VE0yqCbHHc$MF-?2wVO&O0uzxWEp6B2IE{r(TIOFqPyzgw=S6xH)^Hn# zK&mJRFVQL1bkiQZUe3&srp+;IQ$?c|2Vi#|$I7>khZG_?A8Uu(KaftkG&7-!eh zsgdu&H+?~y{6G%R0mJWkKhMcKV5jAUBu5YlfRtQFkKr|>0M?+LOW>uR4PrDwz)pe6 zqwTs}CCN4iT~5Nonos61Os;#m8WzDOr@DtN$7MR&i}-E+z7mZwKrm2o-bYUcb_^VC zY_-#UUiG9m(bQQ$X!vRL#9ehGNQ`dxLk$)8^X;M@HvYTL@5AgB%o}Jn-~GZ-H)R!n z=%-%D6}XtYsoWUr&YO3(^tpp%TR-5(1RM+Z^B%vFbP3Z5yHJK*Cd)_P_}@F<1{H#K zBS84fm=Pd{=dGgaR90cQo_T`9DyS5=|E0)Up)iZPTh^$wlIRkr$+ig)&^}}ASohX- zbwF=l6<4JaEJB^i&SYKo9`MUJT!XDn!S32Kmw79C0@*+{V8ujr(15#({r0IvD`SKw z__H(cM^>kLwE$N@sK054W5I>8hH&YbEP>3v2Ke?I94wmovX}fU@R)y!vdEJ)Ti6b1 z{De%fB8NX0BNXj(*-;QKWA{0p3DxDQB#K}H&JQ99gev80H1EJr2~cX(lF$x~!|$Oi zd4v87HD}`?R=YYd5sJ($l#-oCmuzzkV%r3_lY#mL_?y`yAbAE7F%wBq@G9zAO8~vFez=vljltKe5B`&-4zLey)u$p zx~C0-eso(vjGDj-mluO4ZWU}j&j;`oopGTtB+Uike4z1J(PRvRx?`OHD#n4yxB_oN zy-T}%i+EG(u)zJ1z2(!MArX5tjLTt#7a(DmP%L-Viq_3&Hatq>%cdH*<1#{E7FPpL zdp-bexLYA=Yi|>0S>ee;p!UL}=zD~pwzxgi^k?OxB&GK};{4H>jpwT1LGrvCHGsZ| z;&7=b_GJ!GDls`M=N2mTL*B$zH1_Lr$*~cme33WB@;7Y9G8nkCQVEwLm3i{?6!%O9 zg_OAVQKwlhhqkJ-`mSbz%z`}11705`{gKS)yfClc`)>(8kx(6NPz~^aMdM(_)(ncGSN4eLpp8 zAxg$tt#vNYGqO!mZk?RhbscZ5#JQJoP1G(00Ik3~o?3yOwJbcn4Y9~c;vxm76lB!ScbT~P-e8FIj zex3vS8)-&l;J2!_|2s}LUz_v7L4dOzee=V$d+6KNE#MKXeV3ai>)C(2 z?O}x%zD-1K#UY!d`51nZ{(JpV$Yp#zFqDIlph`~Wet>y)Z{CeIw{=u8m-UmBC%%_A z{1TklVzQX$T3cCY8)Y~E4++CT1^?;ut^Eh$M2T?$HYfe-BfkQd?TcyvW96EPPRC1Z z9g4HY!XOvRNxKDEi4>RU)en%A)r=r83Sh-&i*%P@3GACw(29Tf4yxH|HEp@~w4J+O z2Xts)5bjQ(Q*Dk^to-jj6{WFC-VJU#XWxAmYIR`#=4SmIAGi>o@UBEtepvPt5&fsX~j@=VFuk1M`!GgnN7=n zC>xKdDl126{Q3_ve22MJL75=|(yx^E=iKe)m+bnM?1dSzX&^VmlrKw; zS4iA=31REEGz@PkEBIu^Nh6}hLkDBZgU(-Cs9t0up#gr}aJnX>Y@g(kfuc3l~M-9Wr7{NQPTzGdykLDil7r;!cQFIn1VNMz4$GPqi3Oe@^b>)~cWQNcb^r zoIq?XEr>y?C#5+fTCFdh_YRcX&ob;@aQYlzJn(%ejS=#tmY2Eb15lno^}fcN zl^;`oAwG@~_Wwr}c-k0EsKGy!q3#SH+pZ)`22_Zg_ryD?p{2UW)o6H?14!x??Ka=o z$kRzr)ohCs{|C%adCI(;bFnGX90Oa1!0W))r#(8I#HGVwkL5cOWM8$xKmQsh9r6;b z|0osgK!G5BEk^O;%&{BE3RdYSNurY=TABJP8kC19>m5)8bz}N9yA)o+Ha31Wt zg=g~5adP`k!{V1dJyMk6j8wZ|31;9-xKKZ9TjDALGtMRa7hBFHp__!YQvR!5q=(r< z@b<---N+ZYv_})@Rn3z+es9KnSQ5Y9o$QrZQ)kMx`KvJzYIpue3 z7~g)1n?DXhyM?C=Id_F6%RXrVZ`AFaD)_7DO(Dzt+-vY1NIcd5vxLLO?;ETpG{?E= zZE|U9I?J7s;aA=5(q#P7?Y}YaUw!JzadS_@Bpu-8W6}tD#q?6?{Wn6kWFnpr7b2aQdjR&;b1-zRg=9Cn*oMHs}x!w)1$0tV|i^%tJwB zwL$_7X?Cc-g4iJXt;V4lk&x=vp%yfTj~8ct?w#hGmZ;hc=)oaICI&~V9>JDeM1O{~ zWl(AmCah3&5{Rd^WL%C0MW0s8%b6u-@dh(wyyh9@fVBfD^yQErp(0?7g$?~r4@eG`4Br1uQ(`0*AqCA||OtN3TZcqLIa zVA5b@|Gs&PiMFoG3ObZGf7=I4m)+QzEwJW(uV*Jqhugb^#i>cd3f5BVIc_6#D-ghO zC%!aRPBwCdL!pziS8hs;Kx*TK>uH$P_g(cd8n*zk0YKH{Kawrt9=j#J-s0#*4@SMs z=yxMW8QWYx6Ji{+r~T5fdUV;DJI?)IXH*2L{sMCCRkRX_azB9y?ocF5tWR$cTz1p8 z6zv2svk(Wta+BWXfXg zM|fG^)M)ESOrL-=TQXsUXd1Dqm%AG2V=#D6K9!%?;y4A|EE^5F)av7f$jqTA#Emtb zb|HY+SS*gsfPFCz!ron&b#I4KGs-k&;(97R;_9q-GU5<4Vx$*pwW&wr8Tmf}N$`&c z+5@25h(rkr{ten928+o3r=N-{v%xtyPg}90n9Yt1B@<3kIv|8T_s@lNq!a0=q_eF~ z%$F~`L3S8kZ!ky+!oNC6uXJZ3ir$|bdZ6&+)NG!k;xWc;8?~R6Bb^6!?+AFm)oP@@ zgV<^j!AS*bb`0Mi5L)iTXoFeW!J9?1jGJ=N4me&JK{^IB+PR!Nu>`T<(OU(-LYRRx znv_&q7w38u98t&Z7BK%I#n0FHpX3^m7MTJGasD-;U#%*nwZ6bR$!qbKP4wsc< zml(&g}rL*OJorPHJ$=Jb42)Aeg45IMFIzeiEsB;it!+vo+a_@TG5!Lg; zvB2NPi8iN0zZ2#gAh6iW9XBe_=CKzBXG{`4Xh|Rm1Kfa|9fnC?QT5&yR4t}XrQW?n z>^zEtYiseN5}9v~S1{|_KaZKaQ-G%52apBLC9c8ltm-?U&V{05p?tq2Iq^=8h*PSo zdp33saM#B$ghIl^`v69%^uKrbX<*iZ*6hi-wF*n@+M?sV5w2(}bhHk?S}C%Lo@8?` z)bE-p=k-n8eYbXwHp%oh5sKU82Q5LBG9j`BGPHYTKx4S&*6U;5$?iq~Cf`qCcU+Fh+Ak^tO; z+C#9XK}Qz&w=Dt|GQYO)v*jmF&pEy%6;7#u+T66v)!st8i>0JSX(Op@Ht9QSdJg8X zwyi5UnV3AkIF^8Q;;C?&07GY!%NSHinpo(r=dip+Vmh&h1-{yfJTmVC1ckh5kh2FZAm5Sguay^`iNM5^Nj_}3(Xu??Z~;;=FMutsvb-i~ z51#3%46r;wlfQzel8)hTrAJ*Mh=_R`t(tu7YjGl5Z*SZ8-`!Lg+-EaoG;`Z-|voxZ{bq%hQ-Wy;;M_X$&IpX0VB131gE-b1T5q=jm( z%+&9<=yp2h`QC`FdO)*qPt;lA=%trP;$+pS!nQzfBTkhnQFn}$`AlRI=VaM|t7rgp z)un&kDW8l>rWHT zvN0Y@F<2?mq}51Mau(;h(?RD(CgifALH9tWhZ8PwiTYq=s7F8#Ir~5`{hEh3I|tro zNk%Y2XydrYJ z3{yNr?Mx<`7_H0BjpFnSDVlv!iW{jb#el^~GE6qn>!t2hc-^#{iu&uMq3HDFSP>Z} zl?w?r?=i}h-U89>;3bQq19qTk#w-S80Kc9ARFes-*=G^vX~cSq&f-TcdJEamoO1jvM^pmo^n zC`KPRqFH|~)F|mqeq1~dTG68@t$A!&H&xIxWOze9xgvG`H0hJ(;`@g* z-p9Xn=aJ0_NRl;iohpu1xhF>NtgKSh>*-{CqVICa*lBjw%P(~?8syPk`nMtd3zM2P zNL!S*i`;!G$BP79NT<@I+lG#y+1FuB6+`bzWZ8?XOQ-|Jd2fG7wE42(TvVgh?`JS% zbhIw>Fv_4oRU`6>Z7>%6&1Jt~il%0~)=6>Dx_v~?NA6b?$riFMDrM(Fi2f8a@rXsO zW$KzsQhNzIh=WAFHDAJAiiiz72F0Y)G{XG;>kZS~pW@PK1$IzGo^4%4s&Ktbo{Kik zu`?4BhdhS>bVX2R7G3$!prYFm&gcO?%ok)Srny=$-Z{P8L_c+P@bDGU z`AunM-INnOH%S}b6M7V5Njl&J?k|F2c_IOagfSJb;1u3&ik&C|@Y^1;qdVRTy7^qA zN(=ps1qqYGBE*P=Wp%|1Bebp1z|oDS8Lu!5s@yg|-(vYLp;+E79m0&n^vjs(?lPw~AO8I8x$RG@L9D!TRfKSsZ8N zctTNLMc+1_Vc`Fb=TCm^F)!;y@E24XrXN)k0FTFc#%i%hQv!N5eCNXBCpj<@2b;1c zS@b3=ppmqAnb=A+$sFnsxa8EE0(x?+rc279fCNfk`R3<{SDO@MaC|dj=a3 zImLVz579Q-oxcDLK*~)0o-`;>o_1P;j{oR~D_x-tIKj z!R?#Mu2z9l*t=2%R}1H8$&?dRy`^Y&QulymiX=kz)ejpi{Q9D9d}z#c&RBJAp~-Y< zuahSzXWOA8q*8w@>_~3bs3^ctUQ{_lv<6&u&LF;Mc}^Bu|B%EDW9_S!!<=h|GT${~ zY=o!!wd$DdNrivRAMZAR$}bd*muZl4RN`N+ZE(N-Xe@ttYWy{>TgAG#`nQ%O6PN+j z0Kf$Qy+sh%!!+XXj{<{v*uk9u*_PezrdGM?m7tj@VB5nzcEEe$OV1eWD5MpF8E@*J z*T1FY8awi+#G$#Q<-)43kiaoxObS^o^c;3sIrV78AQmtZ>gqBl*qs@hVi zTP3Cn=A$h|t<2|g^Lh6x41x|7e5ZuKZg_)NYp4AZ0u}Ll4?<6^CT8))+a z{kZ*-KXYeR@-rr! zV|Az0_dH;N1_>s6&kOZUd$^JQEE%M_O@T70TeTu%zbz6B@nJS9duEPL=@$zDC?3tg zI@C(u5_avp*7b1?2@H$T&4d#m3qs9b)I!unIPkmBaDny;?71(s`bYTpfmrLta;2HX zgmFx{bGBqPee_!#dPX6Tfriq5G()OkgYD5OeBo||OBjA~#k|=G(D4QDK7v^o{srBj zK26g8xz>bsEDeS47uw3-kxh$``X;1ps+p1~k2<1s~rL)i?z2 zB1LXmC@P6kPXQn+r(0)%nbLmw#|Yb!alyL&Z9EN+|5~ZfAwoF{Q`P?l6InMOeg*Hq z*zfrnzTrWydy?ZLA)DkP^@$hx=?wY^&7Zsg;DN&*#q-7vPFb-tR!~{^Vg_yuMo4YB zYD9AxFrW)vJ>ixBr38a3n2d@CCg3@lt#e!YCOUnq2R)}c#hJ}!uerQ$f<_gfi#~hy zJk8)W-WY6z>>7`UyDkz@lnxD+H76|GB3xowF1k&Od#uAV-*8w+D)WBX$MrxPpUADO zZ&a)>!D9ChiOpq&K#Wx9x9``ZV}H0@MT~o>6L&8BkR+K>_@i`9(4V>n?V5;w=sBo+&mlYo`8`3;(4_kX9gsk0U~c}p6AlPgpC>&)@R32 z`e7I?D|70nGGfa$Reut5;zoy1__ED?0xTp#bAHhxDt)JDrryvCQ;A&)(bVewmCT5l zxYeJOZLOG_TSgjN`AD~#ZDc1ZezMTrAR=-FvZ~eCc#XI+6Cp7QD7*>lAZ&gL{L(%D zhh)%mhBq1&cLt2bgm0>T9gXnB-1bbJ( zA0j4B|MN(_d68uL$@PKvGSv|g<-m?r(Ml}J<>0wOiq#DKxl}y`l_^@so{2ApOh5Yi z10}8D9`jIe?Ytcr_gDp;r61S1b}xv+y=y;O5*ZVS;lvMkF5jsb6gL4-1ulygR_W>U z&D6C*fY|l@Um29;?yGXl2;hMFpyHA}S1p|e$LZ*E!Vj}f-oL+ZjBbTDk0=e!jT{}5=bgs9YR$=9cw7E?sXPnTa z`8*^!4}ZBih5O%TnweQE6GQxNeKYv5XyO7Vc*k)E5XE-5C*)-c+YcU%C0$eT{&*6A zG*iecwr%dwUDp{t>+dqi0G^JR7~YIii5MUcD32s8q-j5*+)M9~n4|}= zl$jVSyJFl?auJ5##UhrM3z~b^Y8}ovZr=^Y3to<1y)=FQ1(vDvkUaL1;i-Nu(#CoG zp}PCN2L_ONw0m!~cU3N*tRzS~)*<87M_IbT6mR2Ju~-#vF;ayjDSmYf+WmdI3(A*M zXL%Mo)`~R$rkV}L!XPmReAK_J3K4Uoq;Y(CT<6A!Ibv%svnkin@F%UxuObhD6?PNV zxJuR_Z*lmH9R#JfMc^`(eOmjI@w1N{c)J&dq&UTawj%Z20z$}OcBQJ&Ns9}!O#AMs zLi6^F$Im?jiRvpx^fZrGlZXi<*hNRU!i`0*j(Ob&R!0v|1DO%JS%>VL zbr$-CTZB=;A&~@gs|Wva+khm*sXIV-o{G9`&T_AfDBRkzMSy%YMl|87rP_x~7F$y< zpg4hAHt92M&piNll|^1f&(q_XKkNAG(`WJDI%o%9I4-d-Y`}AsTO$EYE4MXZ32*N# zM3G^<$bYd{;zejbL<*F3G3JOBCBs*9)x6qU)bhikojd*j7z48FCfmoG+MkAJ66@65 z^_UgzjT99o`s#Tf!huE2&bO+J_NeUszwYk?d(O}c;vBK;vp{CNvfq>p$NS|&6Al(Gg+no6?rF}6#xGIQnZ2{7u8 zmE)*t5~Dlf)NIWGWwI=?id?|xhIMY#{Ur;kL9!Xj&!rB~P^YG0ADneu-fnhKD!c`3 z%T6@D4-Y8=D#YxKF+Ly6^rAVoq!sK*KWS3Gm96S9V-@^lfS9+@O=A;kV{Vs_7n6i; zy;S4p@Z&4HDFNs&87Emqo~ju8!?7nCuRS#-ot`@GN^-@H@n@vH)a8I${MEmI=n#VJ zEd21qLWFOtjFgII(eUIr=RVKqq6l-oW$zZFZL0F>y=@%$q*05ZeH3oqO2~na5=#WE z3|>g++$=q{ze0PD5mV^PT+#1HOTAqWPc0S?880R z2pw|_TO;3RmVV|yq(uyB0)c%T;Cq`aMHp#bCJ-qIwtCDV5~Gy*sy}5_INoVU*l@(5 z=BKsWo|M7PghjTi8@gUs!Ftd)uLdy2-*iH9xbVozd(YHy z&9*8E;YjNCe0Kdq2whLF_>G1QlD7UvW%UGGa343#;_U~F`h&}k`+O^rbY!&G+$FP3 z5JjPK)M=&4!tcJVDqaVwUmNGXFP;>eWZPO$`33~}mcpk5KXBymR_d5Fxf|c*FksGp zQ{d4k*=6_-Y>{v^s)J_<1?VNKQw@s8iH>IQ32t?~5}E-0Ip7 z>QR{HR)iPah9aw%(n{ZlZ1iKvs16nvLEi8xIcc@G3YJF~fRq?C^D~b(aqQ-azH(<| zuV3l8!QbB=K?YA%c9jOYH2YQhIu+Tj)QC7z$4m1o%Z_JfHm~Q5zcu9%$Y$j~9}rh0 z;-EW^V$pBq_4uPb$%Nk-D|9Gs`TM@?Q60O^JC{X73= z+_a8xncfe8$H5P<4Q&+WFB`C#xd-w_sv=ok75!psK47(x9yh0`4_S>r(`nJ2?PT_{ zQ3v3&N;$D}L;#g{`%e*yp6YC)4ll`lFgc~b?p3A1%=YsbzJbx7>m8w2tHEwXt^W?p zP!MKfZe_Cp1q{*I%+7>%!$y$Uh~(C9>VI6C{MvnrteivO#4AQmln~sWxu&xK<+&oi z%Sl>V`PDBAR@6Ke$}VZu!(uUOMjd0HzMbNoQwW6}FhN@exZ*J0DGqzBGS#5EEGmg{ z{PZ24&2!1ko}8vO2m5WF72(JhC1>)sXDG1bhOx}px|i)wHq4iZ+0}j$9!x(y8C4Rc zaPfU`Z|Ynj-YG+FB?W)vsYbt|vFbyj#`hv|DC$snX^wlXF&y!m$PH^W4JkOoLn8q2 zh=7*JQP6j%X$a@Dd1A^BeLYJzwB!4R{KX7^gs|7^ z&UJflHG!{FYtSJ>P->!3Wfw;Q7GGx~e02dv9uR1GaPWBundWqCLKobB+lMrKt7;K; z*e;OoBUZ_@N9Tfa;J%}sbf7}YU)CquYSg`=?s~ekiK@B5?tx6Za-c5h4W#!7!J~BA zM=l%5hb~$Ihwvx_Wc(peEucIpVgnm#r!H|sVZ$XzPq1R8ZH5;D&|qND_XNLouZ(v* zMM5RNQ|iZ{`hDAOOBvMwfspv86%KyaYRfyHWvjL2f7BQW0FRAC!N+h$SHj^Jzik(W z4g@3zd*$PH{z5%E~vi1aTI8gyc>FZ2(^Tw%jFgz*4B=93=gMpEi{}G<0!@(4P#T zaWF$P7+VK8>*?Av)KQU?zKz*Hp|j%v|B9~dcu5nRvcn5sWE@a81UZ*F@=)|bh2ck% zeBnyGd-8IFRZO(BVzcSiQl?Rhs(=Sk zCy=l8I2d^1I+YXyb$EUelv2DbblVVqtbk${o@7Kh9TnE()s!J!}yCS8vPhVae7Lq{H zP-Np{mp`qH&}(IyM;!{FX+W(LGb`?j3p_w*LxaZf#(}6aM*ZP!c(5uZ7blfe9om0J z8FQY0ZBQVyi5dR2n$La7^W4-Jkgj71wyneIe-TxTzF!X+Wi3w)?C7EjH+T^|GIC*p_hXn1KZo3^V91fF<5Gc-+ z@ER}#Cl2(u0r0Uh>ZV`L8&)x zyFo6HD=lbQK%OFkbO=TCZC&wOtKkG|drL-ry+2k3a9133D`oAhp`p}|7iUwkurqy` zVqS=d5low>AB(} zd+nHp6RfnZA9JdrjJ(P$x&b}3+vK5K<-wF7O^3z}1q8C%{&zp8kGKiPnmsCF&NX}pH=18j#Z&KtOCXi_xjqJEq-{V6_5khAd(V~ z)sbU88mutqKy_s1H+okK8uw){GXXbqa9y~yEn;(97#*yF>r_@wYxASomRnq$SDK>` z%}_L&qsKtDE_xx2du83wc7AF%B}Yi8BjBbv?fgLc8bF0n`+qQe)Afq}sF)5}lfQ5n zRHuqR-)l2OQ6X^4#wNAil3ZB+?gMVMH}z#Nz6DRXLxZ$MlEnx1uI20z)m+2$O;tw?7aMCv$T4&G$B!`4jk;grBlF z=*-Qt1A8wa19Q8&%M}y};!}bm76aa1`jsi)UA+$-a@uwXt~9UDl47C_JZ)qvgKr3} z;;-?8?ITKL$Eeh%PPxMi`UA@>^y8|S5%^+$D053zsOq|=@z2<7_%VWRZ;oZ`%@KM| z%2N^IqbO{wL`+HyDtU=M59?0m`liuT5fCi31)naX8-ZVG%bGUIq}iuMv0=FvL+hEh z65RjIOiViIr6S&ne|7%NObI@Ukj#tb@Ns-yL%c|^p((uEWIM={=7HiMU_m?-G9<}uDt-{wdK{QF z!$7b!s$}owVGCq5MGP!bKD^QrRdW#%@)FEnVmTn8cwj z$}%dCeY1)Ur8!f9aK74D3yo-7j0YJydp7O%vw?~ikM8#rs6Wll6u0f4e)?BHADSmi zC2X$SdKxuWSFT{=pEFhx<**!@(s%{uu1BRTEq%)i0<^|Ql@j|*m!TKsCWf^nK{Wna z1>UWebN~+PW9Sx(Hqctc4`<;62t4C%7dDYHxqt4LS%XkPlo!&s&%z*udVPg^4}2AE zB5u2nuHsG-<%sM2q7OQ6&j1nuWPw?hSGPlkKW3jL?4l+=2s_(!AB;cJ`AI;{xW;vn3g_ONG+bS`Wv%KrOU|HtHbo&iU3H4f)TN`oK$Z3fejUHN}R|EJY!(2?a7ilZ51APl_eQ6jKb8Hl@` zr|@jkg7ng8c^B=}8*jXe0eL3A^mh)0b*n>t1Ts=@V*Q?ayZ^3TSI2{tuh|pw!JP|L zNRf>DB1as)UYzL(fEtNpHt9tJjXyoR$x;9rX*82}ZIa+xOg8&O2l0qiU z9Gt;VSLT4E%*b%;J$Fb`@REECGId14;Wj`uXuP51VIWDt53w#a6`WJiu`w~S@>NKb)TV$;?=T2#fTn_H^a~^rZ~&sfy)Afk z@L2v*fw__T2&_LtJw$42#emJ$A*&UQ$Dq4P&DWfK#jG7DQpj2=;s1ywSBjn_&&KKy zA_hF2gV9Lm5SVm@HJM!94NaKM+LB;UzpW z$GEXA7Ky}CO!XKKxpp+MtiO}b=ev3x=ttcg*7{Z3e-cpb_AgwdI|-Z{V%c+c@Z818 z4>5`YEGm(}13V|FXR3FJ zfV{=#gh>5j`fw-19yP75sS2YuPXyt*d|KdIas*ssW&sP{o!k;dPHqc?$DVc^r66kn zJ6{It!E^?;+s&48of>Hh%Cdj ziq1HX`@>v*XRf#~MgREs6bmbdM!n!Qj-Yf;H$J-HYYK$l_r_=>9j5}x?Q9zuM6GjL z6070#yx?BVh;DeqVZ|urULE#qL)QxW27>Hb%qBu)I<7XoVd339QiOwroo^3A6v{&v z3Ue4)4bgBJCLcn+1|281Xh+2ZA+&cPG8)luH47vWT^h_(gRvIsX-uZ`%Rpicm53eV zxY^4mjL&(|L)~TV7`Iua@HtV37Jkm(ep=Jl2cf;R7-RHQbn-o!ZIdfFY_nL}GUuJc zj`zbMS8U<1toGgGF zc7dHMe!hz@zQfHR+XcAwk!$3JMv?gRJ|(t}(eRv(`EyW`b|+AfrlU^G^%$97`!u%Ix7?A{S5{K9~Vp(QQMOKS7(IESDBy3%dJdn57abTG!W zELbVe=}Q??Sys1hGsMC9^M_ZEzNNT?eMKrmK9r;B#6Y;8E8R~%qOfk2q5l(LjU+-B zuDO9j*S$}Ml4C2e{RCgwO{Xvr7n*-42d%xanwJeqFFFDtn zbUsu3eD!@?eO#$%J`~`U$AU;cFwSSsUlRB06459tP0GxegYjOe;_Bnl**!WzoZ3{p zw$ac*>0GBmQ#vPm^-WhBi}RHhHsv~pIw-*K2qXkjebH$+3Si4pmMwbQyG7n^JXfmB zVMt|JOio1z_~Kh}3X;!dkT~Z0o@-Qu-nVVp8wGUSMIoOs5?E%ua1xzk(ARm{x8%nZ-8>GG8_q4(vJRjWyT#7X2lb1 z(61{mx1qOD+n3Fg(xI10!P?MYWZq)c)M5ZEw_C2|u0T@hyY0u49Jl&g|7$D4!KPlO`_R4x)J!$CHNRM1jT70;&unCgM84-TUC`+A z?~%wgD?mweO9Bv1%oD46bj&lomZu?p4K;8`>CS6@TKQtBPbWDQ4Oz3N1h(NO`x|Py zi6Y(~?l{^pfNxGa&Zy<)PL}L18kejF^h(TT68;;Kry>Mgig8%(K?E>v4Vf3xeN?Yz zr!?E6Xh7O4^skx^Q}khsw9sz429dH2;UPmdc#(?wp$i=c5~0;b5&Vi4u*pm|sP;~O z9fP~_hWu~SNA19C4ANG|){kbw7R^0uwLj;wSBB9nAF48XFUyacLm#mek{niFw=zAmxl60S0ZJ*E&MPv% zh+1rkk+|qcQtHRkZJlWXRZr)9?#t7w zl&`NpAL1RguYpz>>8>~^Utxm&$cca*wQE8JAZR$k7qw;x2WlxhzKUQP9MxI!sAEBa zdi%eePWEzj7iiTnA0ewvM1osH=N08^;h_j0thZ+l@EgfiYrKp|111L$U5rh=$l=zG zmlX40-{x5kb~qKlh2iE+4zRPj!19~AJ1=WOiVC zS{jQjtmDB4{yg%4yWI^cYQ#+K+o{x7?(oTX%M9BNY0^az+XpIXr>r7l5ERl2^Ws@y za~4!0B3gS9sBl+3cP$R`I$XDm`7>s+Df&>d7P1h^Hq4y2%N_P?f1(2-%>iL+OgAZ< zyuUFxTnd3)2RLK)6p5%ItkLQ4PBbYd6n?j>H^M+hhw!9`A6*pS_8E`mj+%siya=^f znJS3G+jjtdj;A5Me`mD~n03IzW1JkjX8DOr8dHvPW%26q8E5x4YeMa|ccgTZvDm5w zA^`^_;KLxnQ_l@axSx3ocFB{wLY62^Fy|+2!Qdo5Og8&U!?5l0kxvMc;BH=qTevNf znjywTLEgYr!u_HEgNx@1R{x#20o;B@hS1(a%nQ9&uDm#}N6wT)R~wAh)@K_C=1Fq9 zewNr1Kuhp^Q|G7)(Mh)S)=VcNU8e!v86hTnGV&px5X4fw;mVUu9fEvkublEGIs=F% zT3pv%Wl0m?tAeUn{mxB-VKT>i3zMT_|29f!^Y}c!M($DZ7jQSk5c(7XBko3;QOK&I z1x@Caw4^=WmC$L$ibpJ@O?%0zVf6@*ham4QhOz?r2tS>Apv=yH*}Xs2g&E%$klkF- z(`W)@(1q4(a)c4-&+f#+V8*)!^oNj3S?gp8-pb@ltn%<@ZxAS$pm{K=xvSYd#;d4; z_uiV;g(XBR6!V&xx9S^0GJH_vFj>^aw-Ax#@kaP^zuYMl|}z^yR%Nfo1aSwTK% zgIx1;mPO-04Cr-XHPSE_xubWD6t+4`vJv+Jj{dL{)oW!_Cu&Y2Ml>{i)b(09lp^f2 zB-${fIv%*5PyX@W6IaBkid9bh%@0VnpqLd~L5WC_r~O~&9Wr$IZNDoh@Z#)1U(d7^ z5a3{SgtaeD0bjuOD(yYsfYwRjg5s>whppXwAAX%r-#cxIJ&7lEFX{9vP7_dCjbzX= zZ(h8t+$!bQmZe(WTV%`fMwduZJZ>#Ck0X^Dv&kG^dRyJB(=Yi#Gzfp8Qs74P%|mW9 zZs4AZhXKb-K?A+pcrw6$b1SeEDF&)hWA3~c$3H@Ag5$-|p1-$RzOJ56Uf_hA^sBZY zfep+DMJ8Aha#dM?*^=H}wRm<6&>JH_jl_`Yi}~(C?8Ty`RN~;?<~UDh$@YC(`x|sh z@MmNIXp{3Yf&A~xaB52P;)ju{vRCKN<2IR zaff@Fj1~77_j7Wx0JCsSX9x?ntT|N=TAvg3@lnVx!4VoBdvdIl+NvNe*vzwPHbj7fyn|?S?VT#wW-heFBeK&yWrAK{q z$!*PfEUq2|CZRNqIc*@Js|%WI&m%t4tGQRHJ-vR`*w)B;N2%lVEG}MECSzd|b>bxC zu8t?{NIx1Te|2r%Rn-*&=#v5~+OV8I?*%fd;bjzu?cQ6g)_G7iSf$ghlycdCbI*8c zoB;v>qhZMb%XwV;ZN;Y^eVvUCvGQdL>~ZKi=pm0?tTM#mJ}~r&H{yge2#symu5ns7 zi<`!&zC9&kRd-8%WogU`g{5o&Ej^C@95#c`=RZU3AdUnQ@Tm9 zp=Wr{>LWsBjSVqftMO|>UYLehoGH>kIy=;?abNBE zA8NEf7K0H`IgG>gZvSvR2%)k4AYqYk0As|Vbx7Sd&^#dcI{w^jO!mW8n$mT)H>-)o zvw>Bn>hzP_!O`x~Gzeh~ZdMIVrkd3+QD*I>LVqhr<@Y}Akr@6}HPr;%RZbHni&a)? z_4jVpr>_WCaRJKJ(=v+2m8>n^1XIts-pfYS%eMN8jV|39X@oqP9XD~Uph;&8Sd<@NukrH&BsZLLu1_ZMRE{_W6)(*#BnEwMA=53eNc<Y68-Ps=wO-2%2hz6@!gU>x(Q$b)GCG8?^2KZDL7B9hSoCY2>i~(T zLcCoclZZYH(!*j}jB_bnN%yk+40N>5WiqSYX3?!{gJpp@aA$CucnHV(fzyrSW?R!d z&B0?QWbUg!-N$#1`%V87xWtMWZwz2ZZJq)KHTaY#H*=29B!oQGBh`XX2WbbocDzw-B1}x7WwWz(*lILFZ{x#Pjc^MpGv@Wp&RY{w%|SW*L`=$ zZTTxi1!x9GX3w_0AyU^NWMN0taz`04Mj}G3SMEg*P<=f>>8=qTeXx0zMY(v@ zCg>E)44nO4vQ+?UV9$Y2D~t=9@8KZD}9?pWFdVD zXqE{euW1HJwqFqj^$@6xHfM)*K6i#1VuyShH}Jid3Sg9J4DksdgCKV1jc zpIr!i;UQC*AKr)b%egEwws^Bu6>PlyH~ARM-xEQis59%q4l*;^AgP|(WJ-C5{ki%Z zLxrJ~N(4p#IW!>6h^7QEbVrbvC9z85X%d;(hq$%diVimj7zHS?z3V(=dkXuM$*fbi zPwdlsuvA99n2JG-th*XcgsD){WcGX==ud*q&HU9lGaeaz0Gcl*)gEYup@;obQSs`|pysAoS zNvuKEa+q_Riig~on4^*lbq+VbwH0RSF{4vV$;wm7)4_T^}s$ z3mJaX;rHhrjD~%aw6ki;s>!Pi?Y59ug1$#Gz}eXp2kdmLM=0-GeXNbItb4aX%lIY$k@IM zfqjzgb*1OtDenv~O~^z}zXT8{%~Q#6oufDsaE1R@Rl+c6hj;!WBh8QF(nd6(cL3qi zWeSf}hQKNyp4$7!w206)My+fp0(M)%6NOST9OF*(y_DT6sdSOJF?Q^69pr`{B8x`( zGIew}g)Rk|4pP0DjPvo|M=|6FNz4;7yD&zf$32V0S9uA8Ut-ZZO`mK_!zct{f7oa% z+fIZW)r`+qgQsfYdRxm%UW~S)VwrAJ@@#sHZoP# z1AdM05sjpy_NA$O-hqWT6;;qACV&ItEX|a!meA6^MaT?IR_dat1E&)Xor~u-G3Dgc zsFg~u_#GMMYVTx!3>ie|eEJ_aft@7E!;Gr5)Bb$cs)Z1ZBEPV84%Dadgw3Wut;+|d zR5rwnAYWyy@6A{@p2}}KEY+>YlJ)B&xrOD63m}ofVyrrKa{bCg2>{ubY?symRBjwoLH*}kp$i~w0L}2!ax`^bh*glfB^hM> zW0ehfF&;YJaj6ZMwzQD-1F!aa3&rA9iR`))AO6Gb+aavJFVLsYjt#h$q4Y(3?A@sK zWNsw!LU3rlC0B6>r{y^8Zl8Xv&W$HZNb>>fXi8L@;~oKqr{q9go-dwLjr-N#gCQv; z4~T(EBCI%$pV!X9VD)_=ObmBIg_FKshg=CM!#t*aZydh%;#7S_xVSEx*X2Kt2f&p_ z&cOdC&az7E2c^J%_4pT|u?gGh>P%!25IB_lDxRp3G^bWbJZOjqhva}Dwy zJzE1TJ^hhpXs%@F6G+KF#9w=t3^Wh2kAP7kSG*vFv^1zu_Cuh4V*)?KYcy=k;9?Cs z$ZEcv^Jdt@&kGKD;~VI16-+$oLqOyZ!_;FQa7W=T-jE@+(i z%Qa}KQEOG9RP?{WOe2*l-20WpphwYJsAq5EG2}cZy5G&j|5>xg4<9q`%PAI02va~b z4kkTBAT8kfZ^qP#N5Y}O%}L}Ctw{(wa6IT%z>+QQbNg>C<88$AjJ3(zG9_k}r>tNV|p|XsN2wI3;M(W%T{JlSizL6J{W z4t=BMmm`HR^w4>>c)qb|dZh89$4ZBPy&3A(TQiF6l>fL_`fYxPgHM+9kD-J29LD~q z@4UlggV3vmXn`NGWCXnpb7BwJF@X___Z`N{J9|g2Ap8F20EK}?;tu~A;v1Ot`PR9}eu&tzN#=(*7 ziWz8WW-A?^VAOl60Fog72&HvBqYiNx9!6ryLYOSaq`rAQ9*fHtv)?UXJQ%in+DyLm z1xRYK1;9NOWL2(8L5Sz*AeLy~R9hs-^v?JC0JU41PEIdi<5LnmUowN>T@CZ~PHN>7 z59iZr7m}fq^WSv7#!p)_zJ{aH3|#4{f94^}{f0bJxxA7s?^yJ@a1LHkn}B-5n~&)0 z*)*Y0`)QZ4gI=G5v|Etpau*|aNPX8KsKa{hKin>M5|N#!=VrPoSykbo;u8+i!*YMy z%XIL~LGFXvXQ2_bCpgE>RTlg{j0)BzoDqKe85_4c?X$^!^$7XS7R<`+vUp@#dUttt!?x3E`pB7v{F!m{?evk{8Z{VBzuy*=M~obfID z^w!*GZsuls8VfA{rO5%3Nbk_xh^lvb^l!riLZ2bZyB63E!(Qft!yvY6&OM~Q0&w1> zIY+?xN;A1ZE`5h%IUz>C?v|;Nw57`$`Fqs$;({|vyFsLnpkPQTWczd|9%nFA@8SEN5kKhufoOHH`pF2~~ z;{*y!hax@mc){jA;vHZhmd}-WSD{3Rv9&=BmhY+QrPW%ADa{bV@?i`1` zB5fjL4IMC(7eh%aWNtQ;9R%)RVK$%m8b(T7o3Z>AlfG);+Sp3vt}Dw@#IGvmu1us@ zTwQD=NUc`8>B2!;!gOH4R8XAuy4=Oeb)lnzcQ!XdJs}Ksrcm&6^yufU|22Pkwam*A zysHHQE3x@HS)*8z&vG}F2o?8yCa02~X|q;Kj7UC@k?ddGFQ!{@uQ{)t#g|XiHj(oS z-bYN0v-3+m`k^E0Zvw*UoA&BC1V@q)O*=$8&qsv~>oJSLj-x826deyENff10Y;n$0l=ISYMtX)-mjd z>0I<-{K^VENGbo>seh;_$*wwN zdnfYmzw5_3pT7n7yh70O4?3Mn82SLlsVm3*)_5LWnE$qE zAM>xTZeQu#Jhr@0!KG8KkmA`A0^vrQl|DPg7RC?Mfd6?U%we2!&%v86HkubzOhvhy z2(SavV4pkXuB-SL)f$%eI?w7LAK5?i?_NN7WeMDT(du7>y46ORNh1~l_ql7Sw(PgnVN$9& zz=Ke88?uavfrII+9V=V=NWu;(htabS?%di+g`SBxD0V3jqo2F15oTQv;t z^FyE6nvaDrAr;cm7! z+PVV``G^pDK#5U#sk+Vn9CB@p)uihgI`gQRRToE*I^F6D^J`L9{?}nyZ z!cHYW;O@tHaTa@6PiBN@R~JJgT^vRQ?E7GmJ5ID5i=CWma4xEe7>@fxxnN-Qyq`@p zM=-*R%%ZW_LbTp0j4?Tw_s<2SE6PW{WrU#nu)0OX4qw|t-~>gqzy88=e@$^3Knu@7 zcR4GfcP!GEM7daMW9$ReyYLt3w+FqG#^gAbm-ox9cwlr%hKdS7i(dUcx*Bqzf1L;c0_`ysOcy|!DniD6t_NqgXmeQqoL=V#qQ*f$V$OTk0pm!w0T zPxLYD`3_@U$@b13{D78>pt<1e@am!QHZ!9Mg=e2>Zv(V zC(-Y)-d|qd_M%)0FJDnN%A6NoT0;&Yn#g693b{SUFJ+p?#Y$+HMnu5`$i7O-Q{irg zw|aj8Xm|bCSN4o6GV#G^Gg&8BOZ0%En_U zQb7pkx2=pg4Gc?_D^@{T-gtb#H-M=fN+PJz<$I=zf~Tl4om~V$wVN@o7Yzs!Y$P^R<_mAc z+22f9{+K26ox&le_0cpw;v3vh{kfqorlQA?XPN4N(r78I!?ZUH$F^D# zVc%So`RGJjihK7tBor2NfuRTc0$AOFTtf4OD8@C)!9;xexj#Y$Qg(SX)R_be_7j0W zakg->Oi=h{GqXa+s=?2Th{gb-uno6)g<#%JcQ4zg?x?jYRjsz-8FnP^C#QT#b!AFy z{qO03L18li_4Q^H>fX7cqtqe&E}pywILCHRQnrRue~O3)RV6tqg>s=g-)z{rMNwjj{H4eO)s_^Q+x?E)&%>skvp^e`9? zPOc0xsA)yhQka6c{JSWJLurYV(a8Jo-(~c$uK$GLM#kP_^2V?~CeROwk{F1i*B3{` zHKif{)%ULV;!u>rWF1XWcSL}CEG~*M67miL@!n!GMhPIyq#duRniR0S%QWP~RIMi{ zJeKy*Qd7;>k#8iybSpY)xCcjxe?Iflon~-L*3|5OEl`}5%hu>cB3P7kOhn<(t&>1y zPpSF0Te9iB@OH*$>cbt|{ zOjFaLKdI6B%3EqjI}WN4bcjBU>1{aHKU)XcN(bQE2?BEQUu7$5lX;rC$F0&7_8Ysx zUq%x`N41=h+4Y+&`BnT-95PJgI5e;4Q7gSa!E|-&W>Ol~DB^Qb4yyTCh1!4W49VEOc3)ULZ3(bZ#W|L!3?98p(!R6}Vh0 z*sO7aAxJ_-AAc~L;rz&sx}F+Gp^;N$XrRfoWY=qDyM)S}QkUiYuIsPwWyegPqxKqY zUZ|#t{YfxyCe2F4o6Z~q<(#$4-v`1#iCfB&D4dbbTdoZbE@DzTDaps?NK2l}nHqTU zk@Fnd8-0h0k)qz$kl2zefo$x`3p;v*ZdcY{KwqNThMI!*rFm8fNLvHmTzFp;2z;0^ zQh_ewHI>h+(<<;Kg_oz;T=bIz8Wa z-zKdPdt7-mbk9S6nxSW$t$4Tcwa|U(_TzGqVxEm|G;oq}IZ()JKR+PNWP8`HkZXYM8zC2GOQH2 zY`wyT(4TkumyJs00`b!yx@3(H{2wE519aW_*T41QU>~nLz|%%|0Pf=sW(w4MU4nx^ zE@&p8xdU{_ND{M4Nace7IIcs7yKKNQTV);|-U5nQQz)y$k~+QqnO5}l*-Zl7=IpVj zJo?wVDb}LqtUDQLh0jqpdRDQdXy?6~&t1gQ?3; zHVL+$z?*v19JHMG(5MMgBr30I0FhMIf>Y|H{%PNL)F%$W653wEkW#C*4uO~n>K}2IcOCPN zGVK&HT<@Ja(p;e4m;2>$%giO9-WNiOGOE839v#es?xcA1^$~>|b+1eh(;jFrgOZr3 z0T2?6CU^&l0hjzMil8k)l~@>$L1@=0yT67V7zH<;_0*?(g^zrw%c4SdMV+EK;$u~! z%C;_(Sqph5S;d3XpD>WNox_*8`1#S!nFHJ`Qo4)vy+pMu@EPpHs1XP?etJD%;%RO# z1kE261q{=JeHOiR2*s6)IurUhO$fkhn5YY9pm+^b4G{g~vycFU!`c-$U-WZeB}t^8 z7K_j}C0xn{4*i?Sb2Ecaeb$oQv6fjKdq)WB^QL(hVa~>Jo5hbBZWuB44R%^*+2013~%NU zvt23kVnHq<2}eEHrvUcADRL>ZcyBcck5@4x3_NX0$zB2Yx12f%aRs%v71@>5&@jcd z{=JA?nwu&83m4s2S~8?DY8KNk-0oq71qSozR%WMs@RIhx>yH)u0Xb&8EA-mF!I%zl z@Rvp{mqGHd=4!%qH75M+hnBCiAChP9F@kbCCwIvF#Ptt)Re3XDjaZ zxeuSdSNpSVM2GUwA9xF&AaEDfu+m1&0m&IB*eQJ=+h?SlB8(;~7V9#GNp}%xB=j#8 zk;wqcuQ~v&om_)1Z|H%l>V+v8N5d?QcA$tDXlIazU5Bh+OKe%f{^0bsz|T8h_1Mi9Weh;kB!ducO1T4?&snCrYbe+{?~s%b_Ou7|w5~up zI|s=zVE>Ral~Ygbm90Uu^?wmNjxAAT2fMp$NtN<9^LVCH+$xaY3wUHFk>ldk$x#*3 z(fd6+J$TUYdpOT=b6z(O_v`_~*2m_JgK|9Vx@wE+y_h|53S`1WgR=V`#QBSLw>slr zQWSD!uuybc%C%mB=wAN+HI-chF3J{EgPAaV)7062xOIPL4F zdwdC^cbMZ@n$5kLwJZ=vPZ06IaWkq&HP1o54$=MgZ4X>}WDRfsvNaJ-zR9MRUdAB> z&7!)BXg`lo2Fzx(c|iC+k4}(m2&4DzuU*`odlb-TX4!mpnasABI#I{U-W*rSM)rtd z?9Hrk^HqAwqqDAj!zTF5wj}fkwuCHr^X%#jO0{$2)f@&ya>Qowm+VpnCS?tec@kfW z!3TuFHGz`31>7p~qu%ZF7VbI<^|?|QX!*A%g>ecwB^;9tOngrO0_1JKX63%2q<}E2 zOgm;Ox}H!%w;>oFF_2ysQM03QiD5y?kxES!ypPhlnN9H#`fx%@@3mj9)*K8 z-Nsn-ij8puro^%YhN0d$eMfXJrqeO1Grq@B)kVN z=Ky9weSZJO8^Fj*l8L@0SWMWlF&$;=1g7M@B3m!6;gPg+6Q&A8ay6mW6U#CXExO*A zA4uey?EYXnS0T`xaoU_4_V zY&5GhM7Cu&ypmEv%^PD5?o#L6Ak3A|I(S_jfQ{0qGtzpD_%cKcIxbSy+)q-jcd>e}9)f~D%cuiRxYoij7T4Ts zo@J(|T_YVw0_FG(Ivc!Z3X8tKx4>@6`v%Z&!x~(@Pv`s{p1rN?O7{WS+0Dx1I54nr zK$bhx?OihL0l$R!eQs$EYiRnMkEJjAZo^P^wm$TsHGL#AGu`n5J-GEV~oQdQz8pC?001fIcn{kmiQ0 zK;Wv)>CI$M01%ta%JXja7>118g4Za)ja{B_r7vOMqQ8@B_E9e^Zk28X(12jd4v%;) zTeT(>yGi6?FQ_2$T2nuBnZ7X&4ARCoM9g?79U8ZrC2oW`tTp=yli%9hA>SHw@S-0g zPXaqaYqcBUY`c1qaOo2nX`U1^$2b$Ooj4V9>n-#jVgRpp;Cah7xyFV?=ZK|59dkv4 zSUP%K4=iD{4DEsrX7=5J_TXoG5Da^RNN}&X<5Exe`xVODXC~$U*SAIXHmp7&8~1IL zpyKH&>soeyv==d@7>j&GWhK%zc48GkF?2N7q>K~@{^1c_a|J&iE^BK63NZd` z#lTO?MwSowMDOAGvz5}M7kdfg9+#&&4Bp63s5f{lR=5EU#-%&?~sgSZmK+uejxM@Z%N8f$5yXTua~Px4S( z{mHeB!Pw0gJ@=t|9Cwsq)eC})giYG*mYp|Qiw9N)PnzEt*H~+4xpL}|8v-Hp$$&0w zSyx}_rNeJRR=f{%k`yLtH(!CUBm#8O3Sgrp~acEYmoNei<1)H+fx+&;e` zcAi=?^YzhYy$BZX0h!ojZagwRE#E^()(dujyt!G5&K#11*%o&Ak{VdD>0`7+18^=| z8_uCoaqtno1X0kN>JrWVq`57g4lUE9NB|5;q~U0rK@&1)Hh{FY3s`@!G+7H2W3x9p zx4HDmqFOl(^7cyL{1N%EjdgvZ7l-&a{&wZ%!`?L)KW>gYQE?27xu)MRnL4V#EIfUE zOwR65h>%o_qRa53*28j}rm(TPa|Qpj^1y^MC`~y*P_q=!Qb*WCqN93m`9gPE!`U_N z@_{&+HsggShBB?2G|XG%hR0L=e=>gYtfdU5x8Rp<{{e;)8-Xw%sBzMUW!y&|G$fpi zW1Sz5=!V?z?*l07CwKK@fO&mSlIt4fdwx;U>Mx|Q$ z+Fi`4SDs7;^@;`tfO(%BIxaXhdlGiV4}*Ae?P#!u(q{g;Y~Tb4%93SdgcEZA-HnyQ z@Q=K3Jh`sJ=V!AJZ)m?Hagf{jp)wqHf)Z@iQrfHRgOEPOv??T?#Y6F>#VyAP&OfM@ zD>)4TD%CP`GH^K@L*o}S#MH{{{ab=uRm4JwKLq|cl@inY z%;;09_B$+DV;G35i(r_u!n8DMR<`wNF~b72mf}&e2EzO1E$Hzaclq)RclUR~>=cys zH4n|?^xcppkoT%(f<(4|$~w1_q!`YOwzrnDk8ScihXbD5u&*iCftn_VbeZsVtg`++ z!9s(i$e5e5M3?jhGB`FH*08Qj+@$CB&eI7$1wL(BH_Ees)9kY4gdW_$SU(sYoj~0K zBfL&3dc@+cL2-fPWvUCe_d#xQB8_k7WEns^JYy%^9mvbviv*$ow6IuZdIKjy*=Ns}t7N zVjp1Yk%AC_5qcr%UypBbQ?D~wNAFe9pJFT85{B#U<60$Q#`OTgz!+Cg#@b*0(?85F z-vZ*EWG}z$zO6jV+m;kjGas{DPt7#v#Jf#dlV{+0D`1xNyB>TOQW@xspEC5jb!K5? z5>G&qpYM%v!#7A;*&T-vrK47%@9<>m8!B zTi?xdgvd6|Tb2~%9kS8w%fSNXxxN}1GR|r(aix_1v~QU-pyNf-_*aPO>#WpbRM2!nNJ<_;Clm&_XVYo(_R% zCi}Kag=-ONWLwZzy4i9{U1B{n8n$;M^rtUH>r!Z(c3O6s$s?aVjq=ObY!oWkkDmt^oM?QUC7yCYUw~3ccaU3SC7{ z0~^Z57siL+`kYl>^xy~pZO6%sEjmjoX;>YZ>eqe|8ajRv`qsR)*_l=){iae{Jnu`E zDOn5?JQvwQxoy1@_VImb-SiQ?FUGu0=#U=?9Frw){NJM~LaCg2A z4@UPgPG7Pknj~sjwhBiDKl4S70S%ezM%TVlIj@5hFK6r_ms9E{!lS9hib}fSLAvz8*+9E0d$!ya1(UuH|FO{qEJ<`% z&n%_0)YRPCzpsny>`saj7r}oML_4_45}4{bu2$}=uME3}jif3;(w_6gyby-b#gjickW7@-EA^2wrW8?S$v?k^)aP;sg%hae~z9 z8X&7Y{2A{ON9Rq*5?VOl%9xh9)9}Ez$uW&_3&qdvG(PZTnRk&myOrkUB1o*D+j;>@ z{wGjV_u6aeb;H;DzwE0XKBDsmreBZsiG}S#&8|oQ-pzi&6 zH26SwYv9KaUARY3;Cb;+Dh2*K>72J_GxSseZ5wDfR>y>!d4;?tm+rFrTbZ2eH^P?% zAD9ZH(`f|{@5cZ7rbev2%vPDtduL%D$Mwt_EY6-t28Sl+ZS~J3TG>L(FTtx8P=B)J zWW>FKz>U`jcU0GGVhdy*n-M0f{pQ;}~2xz;KyYMS6&-g&lvr!@Q~~3V6-6 zbEL|w1G;{ZZ}mD5TlLQ8D0n!Wsye`0KBF>AYvEtz80jn}l>$U1aX3nA`J()6S^=V=EE`GEkowyB>PogH7F4ORt9Z>D1hYZSv z3i?Fa-A;Do&FZVsc|GXfTWj2eGuJOiO(-^}r-=c^XNLdge-*DEOa&$?iFhE7WJWkW z39~tUoT0yHB_2Gm8YBADCeW^1TDswU^d!^px!3Ja$$M9UC0#v*)u8uJN(me+bhvHo+C6-}W zdH|d$vg@>gvABkchWly#;ay<65eVe}LO=kmy2m~Eiz0Y%O+9mVDdxCM&>o1VDQ*UY z)fKcMhmuU1;(2DcE1uP0i@8HClgKXXBd?oFaqv;Btuot8#v;4oq6}F7 z8tQnt{2L?`dou3Sb$A<~pyIlQ)Lp&Gm^QmiaHl3+2j z?!@`HC(`5xjGIBxjU&)zMWllfrX5b_;~#OERqffe)S*Gl+gdbE;A-3$?hF<$Qe zzDs)5jZjxH+1veII+zg#GQdLf>k%?y@sk{dUxa)jrIU63u+8Zahc9`|b?VsOEL$$} z2d^ltCzILG(_cMPbjf>DcG;_Fu&3*;PS0pl#2XeJn;-#5iB~s<#tu!9?}`Cr?~H{s z=09ZTnmS(>54fgFQq47QiwG$Xpxjkf>0g>u&4DB zPk@hV=vJVA&Hakn8hqGwWJHZiE0~sHQ9C@i@(c88EpDCzyo>UT>y21o2%j2evVrxb==Z!5pltPK8bGlI)_xOa2E(x?*fe*6tdUkmbvO8|N??6u zPz36!%l(aMs5rXys*QI^{I-THyJn&1^nDcud>G`6AJm4EkKt1i%pa@;*0907A23dK zRlXq*Ykb4z8um|m(G{W!sNc(S0LNc7OuuWxOpW>8l>rTKCrb*KPeqqmfdE7Leq80) z9O4}4?3|+@iC{m=jeZZy3R^m^Zh=Aa=`Uf zFSV%jHT|lKH>v&Q=_t7Vw{_Pkw?SoWs_$)09ztK9_@~=(OLOe|CI{cU_~Xh6R+A*( zmo-_pg%$(E>{zTR;nA?LPUxoks)mVfMihq+2yXm$r2>^H@kmO;K7wdcC55G}TKk98@{fn``6;JU1H%}qY6uW>6TdWYn$5~MXBdUtVoK~+cv+qt zE@GaP`VcSb>lDPlDf2KN_frmX+7%w!5(wv96Vrmwdvh{g@t&&$ZQ^A#64m$ zq&>yYM0Lysr3KfvUT0chfw2kx7<&zwk-5oH2RtrfQ!$B`Lilvfh4wXK3oBrNy{b$^ ztu?EGvyP`t#MiexAbF?(fLVua5)~Iaw7B);I8a0XMV&xzO-{Xr+@~k=ke#rNab?S( zn;=*Tgm6PTB&KtMIb*+LM~BJoH}PMOfCdy}lVj{ZD0F zIL1BQxlFJCick^lR%_pA2`8(HcT;O@bk{O5?(TwsbDeO19ZMqjyk0|h-GEX_BwG5p z2)&0clAoUr!3w&;(ha6q` z9~Ofs=ogqAOu0{^9k~LEmiDh}5FfinlMJi;f~o)&-~)@e@$7@nTV5BdBwuBnZMO28I({pM%z*0z>oQ= zUmxf6i%AhC2jyCI7rWW>5}t1nNdC6G80sv2x~_n)4fUkif(4Udv(>c-d>B-Svjb9I zi3i27u)xrk(nrjSpT6G#Bb2MTW1vI<6N8UTr%dm1HYd5q3Mn4&Qypftw<|$0P)ewE zfvb7Q-GHBc`28`(R-I%IpxORBtj;#1X!YuBgf;2coRcwwU!V2(kMnR9B5+}$P4~W# z+LFkby3V6GlLSe_i49vyY|)aLX{HaVl0{Nuv$nb(5<@X=q1GP%yq`0v+85~ z(Q0km7BDQ;$t=39bS#zIxFDjS)g}7~7Qy%8lbfhdAQFtgW$b%N--MnWR4Z#y#18DA z9b)8w0pOsBoGlyj1am?$2FWCkxh2a<_F)wZrpK&yFWR+mRJ`zV{zm?FSv5Gfx)Rp(uUhQ~b_6&aS{50D zGh&GQ!R?Tzx--#Sd#?1xG-m!cR{Q~qZvjC!830=Jh_4GejIZLmR$7cL+$E|IYJb)+t!jmhIwKiV-{EjX8OI36XvY`^))ZZi2b)Bwr|1cc*TjjopaC9`#V7dUQ<-Fdr< zAw*7YGN}p965(FoAg7i9Jr>2h28Q*3Ip`8|zhcmgxrfMsr`)U#Ud zy8~?f<=T!uKdjx+p0a_>U;b1WF<1Da4+SU&9Pe6j0sXj1jogaaPd+I}V&m8S4(aXg zlBQTNMY^1Cp}DFb0>a`5+nc91f$82KJ8x->O!U1$*B-UjFd*x0y|0IFATu1BDd#s1 zip!APkkt)Jb|F|ZGPX#?9ZiEJjSB`jeURZ|9zmXmLu?l)NzxDc-HIv#QU)s-1t^47 zc;dJ0jiT7%^)veJ3oU_3j26Z9)^Y9?lCBXa^_^x+f%=Ml`xHuTHXSu_S)PLncIfWg zXHfwjx2a^faf#2ZM#P%=+Xm43MKJA=zD`EYN*$~V;Ic;=R_=*58wzx|0e=g%DKjR9 zlF^~UHw?_09kLKfXB9Q#0}M@XFj9q0V@v*vH#F~I-x(ZFBXib%k84Bpx;VP(dZr4# zN?rKgcGs|pab=}4m~MuVyff8(RCn%>^Q`dsr%*N5J2nq4Y+j%5%7X`sZk@7caudw< zwZyr+1(5C2uU^SK^j@>@yaQmir6#qoZIWB;U7AYAMS|O-anBN;*`v3sZplj=gQR45 z6UB@l!=#rNVuonFKYz1T`?8@W|BI|0DND*;Y)l33$m}XuC`{PvuNpX}s?@$4W^>JV zrrc#k_|**=lrwBn4Li-Z<>g=a+wlS-oo65*b<)YPfD9FwHi)L zX+=6)vT%J{-#1?&#%gBELA4y3EAuWl`$J%i(74?!a1mnH5RFXky76QTQU_N74r&R+ zxqU@nDz@+>-Uxv!mrihtUP(R$bA7{zdbub|8JrCp_S)jF!ZD8RsaYdB`>*)JNs9Fe z%J<7_A3TB}+m2oo`>zB_98bP^RP584HK0HjsC4!?za;H4S21I58rtTHmSs)kHwJ&U zj*MTJT?*<$?Ea3N-3`x^pk|joVY&{jv#*--n0qMZiRPh5kX;ZQei}v_TCUE%s9|w_ z_>5=NH~gqkg(NohqfO|Ip~z2@kQChSu)-JDBfrK+%BPNW_{=JXp9%Z(W)qq5dxW~G z>6mEIj+?&KXi=55^jfJ|Fr-WL2!x9^^J2;!VRwq%kCQ!J20vdjJ?wc1gwkfX$7yC( zLKb{ZGkt39TB8ty{Lm#Jl#?c?xYxbZS#klLc5?i8g2+&mPy(D=>!_o|a-5OrFNW&S zYAJ3gSSSJ+-A)4ew{jwj6g@XB-@X2{zP!v+ZYq+B8aBcbxm-aA6&3+MWMJ$q0Np?a zj(isX-K!v!G0p*lvH9-+IV8xZp^o&|J;%!z;?}!gp0QAndlUGdbcA&`QY>QBRhST9`PgPJDJr)HMG^3{b(k> z0%n>?j7;+&uBzz?b>9eGvSL(3T2(7r^sh%RBx&neeEimMt5EVZxANH#uml(3kah}k zEW|gClK$p>9{vm%EB>m_9N(ejvsfBQSvZ#GsIhP_ye;CVN<*yrxF`dili4aLE%{*c zobxPW+=-Wb`a*K8sq7@=s5vEq;fb(o;iK3WeicyZy?iqnJo%>0{UB}4a;Rn>Wnes2 zQf)9`qQ@au$+E!iv`*+^z5=XrmSOH z7{2Cqn_|3eZ_)}@fbT-6JpeEmRzE!W-WH;q@SE9T+a1fow&Fmr#ITgq%8=m+d3!zi zZ)8IaY8NztqjM1;EUl$j-B7x+EmPdihRPx)yzdeFqMUUd(rLjUclaU_0i>xGjcm|mlo$R88Wi{gSyw3~SLtwiiK05NvM>Og z-C-(N`WMFS`BSDcsfr9m=RFDR6kr}0KxPqE;pQJz6p>X*%JM^sEv(E}3d<#RG* z%1%Q-w{zw1KD=m)lwU_n%Mn_629UHcQtZ30&1*@?Bf(NC0S z!1~gO#9keNM(!SVUnd6=eF)}Az{^t5g*OC0)qg59&K8qt89OqKQROlXv#5U++vOja z1E!)#Nj}7F-ZvD3@r@zO(?hN9$+)q0S_>J*3ix2^C?a#e%MymU3K1=|&LRG_r`(&{ zmJ{;;EtT2oyYdlL9<`VG=BD)^P|5$yO@ zOu-c#`R0w|{70I^6|s0M)1gOU!!br3P&{|x;ZW_{+wYW5AgE5U-;rBF*eu1NIxwXf zxd2@@09KM0O)zJj`3&`yhv&=k)azE6GkVIoNT33AN2u`l!IPMkJi@IaU&THa?NE4w zmiBZk)o<7LcLC~3k$2h1G1Q!uzU$~B5LuAj98`3{GOy8e%9ordE9579^Ah|_f4=58 z%l+v`KGS27u0}}AJYf9~&Z3W%;;ib%?R+975$kHUpr@MUl<~ws*dB_@K~G7fRt9b- zEvNqK=QZMnC-_7C*puyq)*gy%WUguLUK!hkDXSlE-20PR+Op1lJXK8as+>9I&e%5Q z_h4kmGcS}K6BJ793qU+e0RUl37pnVwcho%90#c({Iu72uvK6Mj05fm9JOXph^I>RU z|H((h+O$vOw<1#$Z_grCGMv6C@4a1Fw9YsLdqU^qPs_*B_IOFag1 z@$j`LK=!e~AQ{#eG9JwlPF|`VBxU^ZvTWo2 zd=0EBiX*mvWR|A4Gq90(612!GBwci^64u?8cKduH2Q2UZ${5X)Nzdc_LAL0>du}VJ zYTq1$-~xzM3U>X`Ar5c{`pI=Y;6WMDYWX)i&3OO9!(QQXqRY#fMw{>tN01SMqJ0P7 zz661)(sVd=X3wSLa7-s*_LPsrqEb4dV<_rTcs=6c)_@j8%H zr|F*lNt?$jdmRU_NXZE3eZ04A9G=Mxa}D@LB%m=+C z%UpP3o>ys-Q0}sMoOrTm;hg}!lDto9DqWmn{^4tLAKEKXgH(_aNTV>x0^uA1)hYxZ zhdw5{AM+*PCGPHBO7&4pfcY0>y%@=pW1(jY?xSn^V)_4YztI6xC{cr|ylly7~}6jjq_dmPVxVEgBA)LPN5zoy}t zckg_ME_G~yRrg7#aW}#yH_3pxJ&x&DT$e03mwq+ut zdK^{(1bo3O{T(pU@7{-Oo50|gYh!YEu!P)%Up6QzlLYT{JoMvpxhhoARVv^)*Z8mv z43xGH-2Yj3O;zco`%)8~=PD)<^=)70G!48Dfb>BV_zbd8*a^>L(Wt&7skr7%=iAh) z{^->AtAtv6N5sv_l38aDZWej-<6hWo^I!3N6eT)&X!;}1(Xd~{G=LB>_Q$1Rj>c69oknSLA(R>dO>nYE zT)ynh#qmwpgMj#0#!i|9F+9N+`gj)FV zb_guqCqQr&K2?FoS59W!N|NonAl2_;a3CdfvGayb87CAPv;1Zm{!;^du6XqwWH-d> zLk*Sqvi%zq4-%4ez|pL={J!!N#RUp~hbZY?=ooFddvkt={0+T(%)0zwD62XXA_E;ps?$6uPkM7SYP^uw9LN1V^m|}Yztl}DqY(7JPXb$xNgS6YT`oqRy_#?SCyK5>2qt4 zG>A29T&D=CCH8>_ANQ6i4`dCQj?6}dDNt)b&LLA;BJ70$sL9J!tqyL`T7*s^AwNy$ z3AVcd6Uug&GL%{{j9Q-H5Ke@mA6NnWe0CT81{`Df7>xE}Fu4i*7eKXYZ^aM86e{~m zaOCkRe-c-_MKH?bFi$uZ@A*L4$DFB7ZCam88 zQjy2R?}&xN;1sN^Z#3>*V&C*SEX8WZt0`N_bm^G_g(Ex0F45pIQ2yPLi5w(xx3Nuz zQoA0ia@4FoBTGD@5%C9h+YNyy&WU}VDl;Lw;ZlrzL8P*xp->2RVW`~#o=PwaUK0Pr zII)7Veaye-28$U%*umAmxnCKpfWW}+(^t0$u1w)I_8M3WgOYYrKsm&bKlgBTj9|{1 zdSU@(0!rpMu74Vur5Zexx(e1|*qTnxA~>dA81%XI3rXM9f`?>Rb$FJ4vKmlkJJHS? z)FJp^_=N1N6{RhinjW37L|mR+1pxDzLmo=_#ocgPNy>F52Y`z-{K`4IzuW0XA*Rbj zV0RSNhH-lWuU$8}d2)Rzblntc&C|un_DE*NLUle&>w(z+SEVk1R)I^a(>!`y4&2J`oZ0&yA8q3< z;PQJKFsHr2;NIEGjjoe^n%vX4O!+CehVyRX7DF|^dbJug8CKuk5H&bxbYOuM!v%4N zFMN3+Q5GY9&9WGlf$R-QsVxQ32VolPc+JFsFLL_vi7#9(?g|eCmGNA6O&@{y-?gkF zF)mVmM(*z>qmgtLj zaB5sf1J;H@o;`qwn}2m+jEvLnLl=X*BYxvvT3=U%((OuL#0L$CSTj1I7>@oLrP z$3r6|O;-Rj2tFg#uDGA`$Pi)#1RjypidrNXM}HRRP$!+oQFay>OGxBIzubKZ0l9ImTo-_GGVZ;~GO zrCdf!kOe~7apH29g`=S-5W#v+(Dj7krHbZd;!Q@Jg&TH>IsjE->3|~4IGWsrI<^=7 zo;G?VH#`b_v%B-elzZjDh6l4MiT1NdjcN~hNv41ZImy|UK%OhOYM#27HJ6qXpdixA zehId}1aiRjtGGlB7ZtR>TR?fNvx8!LeI5uG`BCxME{B+U+mPweAf>VJuk+cGTUEWO zQa6&|CCdHU=*#P#nA@##Wy-P3b8^Ae@=-ZpJhblimCGiE=_7q|z)2(~-c-@_*2Nz1OM&Fp7WK6}Y4_6n^!KY37XM&wrWsRz1? z7a;f`eo{al`(rjX0H+Taf`!*jr0qE-S%YgGiwNvHyS@FX{$_SA;19w-PftO>V*dSv zc?`>Xk>T{Xz7vIX`@dSwGfJ^hw#54@#;tYED^nC(?;h;MQ5{E4DKxpnM;h6rdU>FgM}2qx z3@ige*{C%v11avrZ2P;NgQX@qU8vELW}m@-_`#u2S84=eo#*)&*TyG8>WH}@b?~!9 z(N*B*7s5A;a`&aV3;|NZcA?8++C>Nc7Dq4Bm|w4cmYP*@Y`1O*HPXY{Y`Jfpphs+Q zwc+I$#pv^NP{a(-{>kD7t1BJqCRH6x%yJ#|HlgLQvZ-^WNC=p5h%KanvAGiRb zSXW17nuQ0i8Y~t!9LeBaZPY31Xl9&j?m)q}4gqKL?r?d)xKW8$L(WtB-PMCgn37X` zyL?EWs%6W14i)=*OgXp=uxP?9hoI@kNY&n7U@1K zWwtob7awjAJAz=T%kF}ykG)!Rlw72n+@<8OGV#+2ltK!RY}n0J2sHkJ#9TLCsYW{T znyQ1xy}V^>yu3VQ%eF(4L>8G-Y@5`LrRwyTzChqFPwC++zGj`)rpa960l+1oShNwyHATpfDnQ0ta}NxTht7J4%_|$2T)k#Y6`FCH;UT-A zh%vUv3{X#Trsb|S2^ixh#xdItQv?#Fq zV4|$8-$*=Y2Yp1ztO5T`2=Vzc`kD1~#ua)CNB-vno@tN=4b3ys zK|Hq2V~#U^C2D+&1Moi8V*zDG%hqEg=N&OUzm(ly90S!nd7k$8!p2=}GPmX@wkT9f z&`ToJ*?YG)Z)-V6yRA^g2hG-*GOEu8L~_M8Vb>Z&~2q}!Zk+QD21 zqcpAZp2d^v1@J|>)||q7M=zrs`S4vCbjCYa_*Z*rkbhJ+d&BFAcrMbtV<~w5`!zPh zuS$~jj`gdDAt-gWkysFOeW?|5L`@BxU=!PYt?kOOht(_ZQ!=k!Rf&U%|JQ>0Sasai z@U-o~FL1{0v*~=$fXxCxvNO_|Gs0*BRoNR(iO?`CPz9#Nh+zcLLsBMP2eg{qgkk`CV0vY#>=7DhrXCSYsFB5F;>ga2?c0G9!9 z942QI`7F7kThiV7L0;@4`pi~NAjq^Z_@`4UKC(~dDSVuI0hce<4e{^z3^8~TTfYfo zBF}SbjY(N|LMurYCm|(*OYW0neOZg8G-KV98|Foac&Evm2&5wd2L7Obnm2|iyH0$` zl1a5wllScw0mh-Bq(&q!O=owB@Mlwz{a!u)OVx6SlB;9j;^`d?7Q)$zmR;100r=qUNW7XNDJlauZO10zo~*uTf4fAk`ZR(z>zz z-6TPlboc*H_@j7u_{yt8snJUIPKZk3JO7Z^~Kl7Yz_#ulrK zcxBt&Y3eGBh=oo<-YZLVt?L-SQ=s7Mc;1WF$0Sq( zuaKYYv+{gj6Rm_fQ~vRyynUDjiwh z#)1*<4|ba0_Gn)XefJ&UcZzjDn_GXJ@H?qB7VFX2yhR3t3NK`$`S72>Nm<-?Xi-Y^ zYHsNp8ZFe&@{C78liAklIbZrO8sENX*Tkp+_TjvlbbM1F`uh5*OmPXaiI}Y}Z(xiS zmMYvsZirg$kZ#?`DU*#s?L&4Es~8cb0-Zzfu|k7Zar%s4!tKU$_)$OH8o`^yZXiE^ zDlt_0YROXU1D^pa3=afKwx%VFYVB4bZj;%lU#M$60?d?Vt(K0U;dlJ}B9ZlS=ZEfG zhJW6fQhm!>cAExPv#)yk1mqdcZQpZPhxd_hO>29p}N8PI_FTC;T$=sdQKEN#Ld+MH16 zfs`ry|DWA=JO|SO6%T@)Y5kj2x$>`P(`Ihty-&vOw9JRet@7>Ky5R6!S)T&=-XhA~ zOjZZ^TS_TE2-mt94%z435P4joh)sqDURm%pH?}?NPm8uTrW}ZEsn02?YrtR~b%jvI z=c9?i>8S99LJ1*2CsCR8W|u{fR4c$0N|g{S(~^u1lxdy<`wmRDv!~l`U$6@j)H^*h z_s%{&kK#oiyPrGbEV_TM{QzxZ7fFVqP=$t0uV(eBA*F zMl0tX&_mRGRqMNZ9dHliYrE@BF)0TFCqQN?1??oBQ;aFk650`_4Lod-eZAFs3+n~& zHcmD*Bo2joy$pyYt%tQ%vrqV2Zsm%v?mX2m0brbw<7z-#$i_3-_>xl`C0gUh5yADi zL0lbg-BEkY<`gl32s%kztBA2dAR|p*BHDKRNrfD`yTGJTl()qBb&eJ^QhPnpT^50_ z#s;yekUn-f78TdKqid?W69Tb9pU+mZjsjilR2ikSTy?&RGMcAQm0A=FeDWxpL5%Wi zQp!6JL<~#Doo`>)1@5^l7qQ5s9f4Uh1K<}*9?W6TJEZRwcG&)~6+D6P7D9;P@uZ$6 zESwiFvk{y!>u{OMIx<>^-Knt2?^WyzEk^=a_`fA3ou?HW_`zcDnz>bf!} zue?Z~xMsJJQHz>aAR;S`V3~}ez-6KT3pW;UdV(tiPDpsJi)%37n9uyXN;H8#r`)BUH3jVK+uH18@_nkOq?O8hY0DYXLH z3p0PHiwfokFM!=UtuXlR9cG=rJ06ou=s`by={J=6DX>0vdBB@3ADH-P-ouG*9`fJbV!+$s z{wj!2=rMPs!bNfEU^%m@;|d6Wi1!7Zu4(^8E(S{3jd^=ryh)hiooH|pM57yWC&r&6tkRu6_p~4u23kEumrrUeNECdd z@qPeuejGU$JF$x{YXO<=13LtuhiV)|qA~c@qSlsg!eH-MYidzWV6EOKTw4mDzT9#A z=?n^3b3R>xbc5=3i`PrVw9pDLA6fpL`AIc?9tWw6T54OC&L%&L79=viK`F!Uh6cW#Ixt5mQVF!- zLJ0Vzn%<;x56}YMEa0Oeo`H3Y^`A&5V~&{eQg=3h-|!&UJ`|?54$<{FyZ&RED#n8!~p7eXiz$Q8Ol5n;R=pP8+K%>=#o%d@zwc{ z>{P=rpC=DclGBfCz?qxisIG>Y&A#Zlg@%2+Q8p9r%nwa@X_;F``z_q#Y=yrET-w0$ zJEf~3v;OizuO03ycO8Jp=JaP&$av(}U>rVvd^XA3`BBTpCtz1XN&n1pB*p`C&DBS9># z1_v-Bi0xia*3&Y0(&Ik#UqB|TL8X+5fTw>Iwe-psGs?+bMY3Hxd+Z@Ae;W;{QGDY+ z-wY&!!fHB|jGIHoPJi5JBe~yL;+DsgiyiuSgTLk$X^8KGNyeUD7*-n4!L@KBm zl$*9cz~`GX{VXUYcdfTzA}(L8nFrZYo%)bC)>n!gpkk7+7oMI+vpDZuKde(62w>l$ zNRe`uy4vMCO|mLcR8SbVcKYFz6^Bqw(_4MdNLVs4DMZDa>> z2m_?GamHJB+FFp+2CUMg=pwKC@l(Q-92n76m{*~tob6LlTx z^0L2RK+Mk(-JB{>nYKSu-Rl^jUuy9F2(Sd7EJ>%w!%33Ju^SRLa?P5na=tJoh&)Lk zq%YcqZdY1ml&zzCJJk5z+x{C$0@_JkR4zx#%usE-<#v8|A+9--5k#iXICDH!Vg{r> zSam6{Rx7dpRhxG9QUP692d#I!>nGvXLVjm;dH!w=h5fi3; zgg52d*}cz5-liE0da1z+-bP9(H|Rkt2CkAG!B(|pw)3maNEM{p8`+6}omlSG?4N$c zxz7Z;vl_6r8shVfHAIws5`iB*FqdB;l*tWv2`Nt?O8iHXkhac1qH=YLau^lHc`cio z7xF##UcnQBnb5Jl$z>ts%>Y@`?UG^5>touQ$=oLrr8pK|Kx;a+h0|;sM3t485 zBOq82nI+Muk}dK$TvT`x=t1UA1r0%vg*DtpCODQy4=wWf2vz$~ zvP#l!!0N^|tLfMU0p^sP65AJn1_4l(^gx1^6OJ5VqHE1dGUMe6!4|#1>M_cwY#|Qr zP?xj=7JRGD)SQY>?y?wv9z~0g`F9ISJ-#ZnWdItPSgCkX-DCnV)tD6>tk~v&2ob{o za^K$BartMx54wDleQWgz+NH4eGO;_+iN&oi;!{p~tHz>Tq}nRU+!8NAF|!(Ox~7M- zAr1I(@SDcwznL)HSNxgy!N@w55?{fBjquI*r1ohr=7PD$onY68xEp?g+ZP7X|oan z;Iy^bG{PX;AOuQUa0@pFA_=dNL?4ShIb%N~M3(9}*qBxkUQ`6=^JahZv$rKhny z>|)LUwT7Sx_MogC?1kWqj2?V+{wghAA9B&`j8;gt(6k~UYjb`eKZAM#?UGcfB6TkQ zL^WwI7u@3z!hc$`3&-g^X}a1ckHUEqgo{sS8JU?Yl|`HpG{Bpf(eO$a84KK&(kb4_FNs$`OUjb-T}C{bnXNP=Rk#{N7RI-lpiSyO-OUGmujb)Q*K0e_4+ zV*FD5esGQTo8wasGj3p^63U{(stnIMMC*%%#uOD_Zn%2k?hH-Fgjchnq5wPS5=q2o zpS(MEPik+;c6VESJN~m&23fdxP?#1n*m7J1^uK$UguyVwTtg`D$gqd!9i`9jQOIi& zIFoQcWKoMCXWODV)i|l9j8={#!8mTlW;!?e#Sd4QII)vaq`Zra0$JI|gzu@;YU-m> zr+rOW+j^uo_pQZulk41e5|x6%sm_dqe&pyJ_z+=zU;U zOq}$^@4}42;=b=DW(b+7OUD>te&lDdLOPKFC*2#UD)XihFp9s_c~LZ3QExTu4#Sh> zu5=}9XyBQQu*MmsF}{f*JRyfP-7Qy=45j*c(sPS0g>WY`9h#pwRtz9Z75xlXsq9aw zid4@U-74j`6gD;MA zm?OO&v9?O1L?@4x43{v8&N>^qbU<_4I@%_WExddJucdN4NUM+hyYp=(SB7dny}xZvw}Lr0!Ywybx9zxhs$(}zFFFDX~Rh_UvMbY9j+3h!*#gTRAVhx z@L|#KAY_*@W9swZjVMRH!DJ@x+5L;ryI*+GssC7mgzLdxxeWQij8X0fPS2e!!MDCT zf|OH~Sark1-t{@9Fl3yEi@fZKYWMMuK#TtWlfp|wm7k9)JmKwu>VN7R2o!I3m>dyn zzswhbcHKl@>81#%YJYzWPhHb{X_GCh`j9WPUUm(auNB~m#NHp-|(<%H3T@xV;DYPi=s;P*(LB1@pXx3Q2;vAXOeUS zz0_d3DLB@UMb7B1=v&QWX$^A+P8ehpHl*3^&ha|IBA!|U?Rr>*w3+9I(mc~}k~k?- z;~Ju#HiYuLwpYLVKn2h!cCPRn&km@j3=CS+)ZX=2UIu(F5zj)bO_EByj!xgkt+V#+zc=V)*yI@5K+>p$jxhVh(AOH};8*K^lm1*<~TwtqNS z`V|Vlq!8<8UPV^5x1#Jq;AE3>nTP%;uJA2vXX#u2a`~U{5<&LRc$VkkEVMjB8TRkk zcv7~#vHo8#Nr^PgII61+^uVH>F_^cBUBSS{KDkG0I7us9Q=U8aGAzC^RuSN+WUZ0{ zIo!HBH%>1e&%jEZL^E_m7&G^jc!@d@p4qsLW+!(eCfCu_&wt{JIKJM*8mo;Cr|EN} zx;Y5V{6RS->%_9%H`ln$r5asq6-2s4H)Pq+0K(yI_%GpXn9@N#!9P}#=}~Y}qQ=bn z)qp9}2JD|@iIz(0@M^F;yD;npD&w1^P`Zsgpc^;j-NBxYg_%LGUi;4sZI9Kh#6&{* zz|IjCm3tWNsmesedvXoirq#j4hUQuPQMeT_ORl5LmPTvAc-~j_yI6y?K@QHb@vZ_B zUnsz_wG?H;<<$B*+^t^QpN@jS-b^q5W9>20L1bxc0v!h zNb+-LGUy#7M~oM)G(}1gSMcP*UqG(YhdDE7$A+lHj5|Q*_z;aSIe+|a&&Ta)LNPw& zjtOc;dg*?C1-HX35epuNP-QQE3q{p74uzxe-1(AjxTq}5&5t8gzNXcpwU?2CqH1xw z0fr1bHd8f5Agb32#d{}pt&+Z?=3|St-QKL$C|uYR`4Im!PiKktF^0qwi`LB@be0OP zU`x#dcwZpiAvM1ZeUh8~-i$;XEXrOMxHwkxC;t4d=kA=+F!#x;dX8+i=d_G&KIDvt z9plCMlgJYX_pW!)BW8)uM{9Uyc}TgPq%o_1>2;kQPg(*=+ID7wZpfa-Gh>`BprfE$ zwhgHfyDx88f>$f__Ka)Vs1Z4V+svscs#9?WnXA3@NK2;e@65QRcjIhy_G&vRaFGBI zd5)z{1ZWSSNt%&)`l~I^XAO;os+Cpw;A*0{!kF~ADl7_o%=!0fP^?Ww8H)sq+6c+- zf1|U88@*GWL}hiX)I-m&?DKRtM1)@=g|W)~hw@LeS}7olEKCIIil%~QrrVz}5udF! z-Y=3nS_3&Q|DK0W! zU4!!t{N(K5_FJ=s`SagX1m$DkBhvSw1GaJq-;y~+0m@QjW-4OWi_ei_@d2t~s_q@;Y#(EG3l@nIB&2Hh}d0d=sH70z3GbY{# zhF?TZ+yP?~I?5!6E2@ar9KU!kvb_DqYZKMaDB|bk_a#}LX&ELK@DhQMomv>k;!6W!<_V$*_1O-iGUU7GmMF2fs?2am?ABMUWcXirfP~c|&6p@kR6&2SnNH@) zuOj$wWjU8rL7*gn;?JpL9{Rpyc5otEk&6xfF+Cb&op8U5_HCu;T;?nR z^@lo-)*0R17eV)Zmrv&@d)RFv{)m3*zgw^02q)o7ii0i{>Qge8weKb>XK05Mr2?z{8uD0aU|3pUIyv{{> zobI8lMrcvP5O99$YUQJ9@dhaK{2R;T$4%)_7vd}Lx%M}Sm&>tOgi>Nkr=?w&DpC`y z&gkUa##Sh1!CFJo_l8 z*ek$XIfFcdH75{*3_8VE@-h=&Wb=<+q+Y)e$LuK&bYGMpZnX zvbrSA;{IC~MX@SNnZHUw(J(mt8Osd*Z4jqCEt;G+M;!&2OQLep^ZJ}s???z|*uW_m zsr#Jk;Qk3xE7I^$PT@C@mN0UL>@m0+E^;|@#2YBhLwmQUWZ5!s#jGaRYcprhGm*{9DatBrGaC?u{)2kHAqGper_+2QqO0A-dznnw)pI_`Y}(pDTx|+j z?*iJ*Gr19@s*X?+v=xUiZy&O=8B6SkQY7?(LRUT{dg zuvuy!-AQ4p$~I67({>|+a)PFsD=N5H+wmSmW}1_1+D4jqOVtz$-_)D)U~bgJ@V!i3 z%T>2$UvD;ku1r(=F8I{8MENb${ z+o^4*KP=h9(l(u8;qRCxR^RV=$p03|7*IcA$L~}-1P7QxeG?)KLiDD@qo?3cjaMxqU1`Xy}4@XAw6W_3wSk4rB&CTRZW zcKbp(5y`$aPo2(J$53TUI+XX3!V;|Bk7XzPI6HlF=J^f2(e z4?f+1Km2s0mTNKXyh9)=hSdk~ntGWw@W4d9N*B1N-Moo{=oDIm5L047m%z7ol&^4j zkPg8lYyQ3nt)mCUr0PTV)E@&tSgcES+^p|@Ios34py{;fbUJX9&%1#v1kV|I;INXi zO{aK5NJ){j(e9Rg_rh#h2K}D5-vQB4+-g@?1FQW^5_T-KAKDVu7ffYDFPW*8m*MMB zC`@AmsQGR(NQk?HV@*5w>4k*x2w9y zgLD23W&p&_sFhdbO+_W1*ez2OUXFgfUQ|tIwcl(QyAwZTk;)Oqx&u(yJTtR{{RMF! z7+F4lc4F&0Ynrzv3v8$~L74HfBy71GQS}>xdohnu723T?L8_B`xF_I14#i^wHbc89 zXL+1Q7rqyxNrB5!8<|8fHAig=@`LzJGR!c*n&CS0LTt&9xwWgG7P}Rv#i`)U%}Lck zvY;3K(&y}$mvZgH^_#|@MKTn`Cy5nl^K5iqP(3t?fzC)@OGRCJ7+4VIwRiv$kdQ*Y z{K?MYT4?Pd*}t;(L>3n$O=hDeWaFnvjx~mPw+~l8MZlhK{s>T{Op+M_6vu7?0B%|B zXH=lw;EFEGAZM17_B$=fzX-bE$(==`=_DloymhtHT_7)Tc9fxhXKqMw*!{uJ@HwY; zU;U65_^hW$PweXnO}fHEtzGx%-`+3wM8M-|*jv!2yAQ_n5ttgFAJ6G9yX#=%w7nr- zaX=8l^Qubvq~}{+PUm_Rp&hjxO5sSVI530e&NQQ2HSt8WF5Hz($j5hX^KUV*EwABB zNMi^c-#pG)_5cY^`dwEU&BXX<@JJX2zb63MuU}*5^z^X4tt>gSm;R5sY+WrH9JKx@ zc;UMY)3|0e`fSqJJAWb#l`2!uy+ppt<{g(BQ5-<0$&Ax_Ok9{)@8^z#dl4?=X$<#SJ<+>IP2RRyncN=lD#+dyu)$I5;Qu@zBM z_v!nmlc1y*Z>XUUArXRaep z!Xl@C7T#UeqlHMtG2G#6L|MTZ+Q{%Pf`NK%8b?Y6Vk^K>S2mo(@~p7~n;R9fJVMaS zevF$#s|3=^d|#Kjq3p$!8hsc%1!f)TW~d zSG%DS*^Ziz{SN($&If6ody&+~UjmCtY#Y(n@>#y2!bPyL@c-`9odCb>i-F=(F52-` zpua*Ek%wpeZneg3dUcY;{S8p&uN4*|yz+wYR;yLB;(ajQJ9?R;s%1hz{5?9znujSq zRYc+BiXkbVuzBJ$ec#i}l^i3|@Lfe^IXK0q`A9sj?}+*;EqxVVduSo&*4X!K?aHsV zJ$5SSBF#d6m~!8@-Qtl?6_v}8y zx=7tAVc<5la;1My#gRO`($^aZ06V@bh2b#ZTk{YW+yBSTLXCDW4-5nvi3scTrpBcQ zF8PZz=+AV{T$i6H;Dm*D670b;Rxhv6^Tig+Y)Ara3FWf)fRI%6QU8x>!FUgS6Maa!1?(ICwOZ7%a zprBjB*qia?hEm#TMA8>(l*paBn)9#(bK~)k-!PSo-R!i|uE@jb(bGJ9O%_0)E(YU; z!Ik$@n)hc^R3wprtK4dl4zG2sJ8)YzkSHOIcqD00+sK$RkuIh{1~pBs3xio85vraf^&a)4HP6QpghBolk{W z+jX@=cM0v6)BynU`C$orPZ%JV=W3_}z=S)n1J5qYqZu4`Va~$8r>FX78r>LxFMJjC0dtdznx95CxEb37)MSX# zV6h{y)3gDlxP}}+j0_0rpOiQf`+(oHU({sY+%dm5GdM!Vr-NQRlZH{zngNV~n&rf9 zVk6S4D6E(RUv&!uH~V$nx}OQm=4(W>V3!0#u4i5gRBIvtpj*0E7GQpWvwg}8SatXC zRDx2pzL59sJ7saJBZVuXR;0nj#ar!UqK*^tQKmD5ooNVs(O1W-KM6VP`l=tYA%STH zBu>wZPp>)K3<}`KLcA)50xJAl{=~*Vl-IiDU(tSI zr_8wz?r>BfmVCQ*XMLaS%&|b-zt@?ke5rRUEVD9AYZkl365P>u(=KM-aT zG=EyMHn^c`$GGYmGIG-Z9B8YQ52B2JE=EBzF;N9Lb$M7glR5jPeq)PC%F?|?-w)SP zLv*xYy^X&104-31HAK-a$3+3=(F%;ABX_ecF>j1BjW|1cm}&yf?-#k3A^G|I&L0 zD%(N<*2rPLkmha1r3A{@X?6s62Q{qShb5TBi}N;+zNx zss>WKE|~y1K*qn!%Gos_4GoTGX8C6D^-PP5cl5>a5oW=ix`%ID<@WIsB@AS~ycPWF z&K}#{&~>Y#JSVqk!4$r?itr?jH<~Exnw}gwA1nHa8Ne>?cdjdWY%p0XOY9|q7TArV zcJ|mw3#k9NfJ=V3HFl=`QoDSpC#ls1GjLkrAoE9wSy&YuHCYQg>AIX5SuQl}ZBjkm zYlpP?j2mdsam95C8UI_&X6Yow0qGR`2f#kdEz?L*oS_^ok+0uv2&dzu7P95!_AypW z^*!!XP@bbEM|QFlnQBaF+*57#f_|SqxTqy5FqXVqqKS8`wN0}2wz3WXKaOY-TweMu zEitIxKLrkiGdn%*gZidlZAn z>O0ldfM<73hcLhcG&|b5^py2dyQ{0Aq@O_j1EFCQ6un`{ZrFp_pA#IaKsL)c*^#um zve7coD89lB;X;=L1s{yu_pfQ=Bho{TWz|TSw#ZNe6g#d{M-HC=ph7b4lRdUQ`49>? zSz;JG?3|EWea6i5fgwprnMG6j65a8hBueE^hHz|2dWN2bH6r#X|BTdU1hkkU{Gb^O zW_@Fn&0I9J#CnS|^l0pM1A%JiRj?d|7<&TFbn*DK7KIT!S{lrLpA8P@0Dmow%R0%5 z;}BGYZA^`R$3>-9t6M7l8t&(G?aRZ39vAA`2yq#&VzQWf4**RKU#8grPxpn(TDFX# z?pm!~CflDrPwX(n(OT~3PAw$*@x}JMqLq4JNNO3w0WPe2E&tl(pr)yFMO)IUAPF$oi^q;Aw{~Gig34{ zd2>NHB>uTuS}Fq7MSRWLC`DIerFk6jW)(TOYqnOuhO-biDu*aAFjoJl7fb9+G>$Wk z_U1sUW=2v!Vg_8NQ?$>Yw4`PxC@rXc6;gF836(GBtfTJKUKwgPXT3>EOB+0lDGmER zQNU$o`G=C#{4ZTQj|6z4tlB)l6Jt(F^T!!&g6PphsmxVsCg}TB^RXVk1Kk8(8T65a zxv98f$Y#k|Bf~0i$(4P`I@G~ocd(pV>KegTmLgT->24^Dw8d=T7jzIhL=a6q%XNlp zK0YG{=p-`hq@IN>OHD0{lPIrtnS`bp2*q;;mN%2pyaeQnJD&BzU6@A2$3V?xALmZ!`UUZpD1JFMXY=Y&M#4Ip9Z%t&{HWBZs2fC@&37GKM0 z#)Tv4|18%YZR(ApOIk{F4gR6EddxFPyQq+5lvwmkV z^bMvDPwC~;p28&8lOY?>>N=;PV7PRELj zjmz_tZgt>|=8F)y31--a(1w=2hx{~g(>sKLjUKPpdpDU~bTgJ8gVTh~r#j%zD$4z2 z?1`!O=B0!yYbz0G7W}?A5MK$L6-tn5B@-^DO3j*)USNCs8ld4E0w1&iiQ;l`p`xGN z*-B1ji^w!Uo|(gy*8Qk?x}<{}$7vro?oyyI!cn2^CBuz7g=_w-u^9)Z)<108EzQl& zTgz9a_zKFao5#m~b~SkTm#PG^606XATWU#uts6{4$s$5?=I+|fjV8W_WPgj&M_dB$ z%9-4*WkN!!uieG9C2&_{G;h4^a0b;|N^H*0(&EO?Uf;uTttiNvIF^i;%y)i=-zPlZ z6IQOcC?Hd_Tg+F_frx0ii-)oM8Ha)96Kc@ZWfvxGXO3?a68vH&Hsj%3PYKhw!kP#}s(k?JtnpZi1=mBs zUiYXQAxz1)-JNVuJE}k=M*kxu)#x= z-GF3Axo@@}yNkAdtv6w@QoNPT@%(H;_XxGw&Ic#ff0)-c2;^jhmj@9au0hBly=3w)O^On- zDv%Hu?}tnvikK^}lQf6)<>G|MDR{(9inV=DuFQN=G;1S|5a{1>c_odDu+{&O>i*gX z>M~@OLYQcnP_cWyQDUT-o2&RF8a!qa#E>$7#DNafF7?%*Rdm|9bZLCYZ67b8(8Ifh zVBpvV0V-cZfgpYC&VWlE%XQWUvEJSO5#hK}?Qo01HetI3eP0YUXj$EPRvHgBSF}*8 zg&)Vyqqy|OEuL@l>}md<9=x5)QSbZ>lhl`h=Fhex7ZB2h2#gMUQ_A)J zl`ElQ;ZkvMeZy2NgaD>47Ib>}GlD7Um2X~+&;>>pG*xIk)V%NbTnQS?^K>bIUG)hm z8*rHREEqbi<}s4=VbA}7)nK<%AEY=zp+qE4LRDkp=z=ddKan}5VG0yHSfp)y5zG-)~G3CO7v)v zHV?{5LTq4&NB}*ao8s@~nz6}A?b?(RNk6vH?zfn|=QpyWtB>pxPo)@}hrnhd^P?>x zn<%pBPN>r?R;aycRuV(G;09aGCu|847u>(#2{Vlkd_*GT^<+fv_vWrcgkVg#uNkm8 zPCko)%?wPC={&=}pBQ)MwsvuXIF$SAZ}+|mzS{6<562sVhiTxaDi@XI8xJ6{XFO)* zZpGp}m5`6%fDM6Y^DiC1U1M;bLyyQ54&CUD06n=@Vx$BOhK>JlEh5K%s}kX945^+)7K+!)voaojO;Qg;w)OmO0-)l0kGbKQ*E?V+9*q$wDqmc zq7GS(lo{Dt0i2eORJS@H=-H@ z#VRE4F6TJPir|9l5OPaaCK3)1AL1S4cC7asAI*{4bnB-9`|GtU?{!;Xmx-?qdeE5t)pZ_sI#X`%g=7N z$#U6fuEnBSm3;t?V~>{5g*e^$UG~00nN{zC(L$$}e(h%wMPOfot z_Q`hJj3rw?`aw+|H>u)GxiE4}jhXFgeaw3P)2-*|7C8bK^#N)x@`2`@T)S(yAQlg~ z$7zu;7Sc%EKsQ~82g|HLyIA&j%6ZI4!yPi_iWm5YjSiffW-Z5wljj`1CNv#Kv3Hd5 zy$l@uqP75m9By;Qw{u0dg-An3pHam>dTQCDwMHU&%FYus^}SSW;nLOd;=I{P>}~Zq z{)ylFV2XqApY{Vdd#=OMV;R_d3DUe?7p^Z#%hbj8>^kW2&GE2#-O`tvv=Ghk4^IU@? zmc$0Knsja!eUuEZvH$KI`K|?|!gu%PX7$G01!9L2;=aoIh~D}zxfXXO0By|>*H6<^ z0w!WiV?}C%!%r8W=LX=76{VxaMm#R(NDgxg_xXP+@HaT?4ExP43X`ds3~UH(^ClCw zxDHJthz~OgZm&e`h2k>NwgGaE))#?Lh#kM@{bHVWMS)Lx0DBn|GR;KvE5x;-C)~>S z08XApH6|9=&hW*HRb+W+G^xeHu$tL*^*&OCRDD5~RzS{8G;c#o5g11ibK(&V9zrjW z)@)7Z+K`IWjqC*I5M^K-@e&gDQ`P*0iF@V8qGE?KGAb`=X=e#P+nTazqrct=Labtx zBL8Y%S-x@!t<$T;Id%RD&m2)f;12N^zL5$xy^6V1?Z)yzK}F=M;E7>8=UQQ1o1bQ5)23C zIf{`Exf-ZAY7Ey?scVsS#CrWg>~1aXKWXUg19+WIr*C5##h!{9`UXN#W`51=Z^FQ_ zDV_c+;41UE0^?WJ%jzJ&Ut#S0kq_w`@KJCh5GMIH_C4g$N>M+^OcEh@u1gx=r`r|S z?(mnS;TVUG$XVguk+kjZfSMe2AMBhWXySX`_gvlBiDbhrn?x8{h!62vL(NAP%z@A@ z?0!nB3gOVg&~pE)_#6!+kS_LIp5T^j&Dhl5B-hwTko*DVZglZKmturIh=W!KT3{fw z_lVIMW6xw8=QXfYoq$N7F5UOrr9#w9LO3jbg3*mh=3h1yZ*M6YYj(sG!nB>!KnruG z>m!5G*&vCyWz03a{B1I{L`IP5M4DjenI?g1q-({e#_yq5MbLE}DN>I!h?yjMWEvTE zR0=l^{FeFyrJpf{Ct4Rp)bpvQ3vj+gea2SJ`($S1WO{8M9z|L+3zh96B@&f5t@p))OvO&*p0%rkn%7j z_wc1wfr05^*j!qMw4mobH>sF(#SLjU00N;Nq0*eu0cVxVlKR$0S#hLshU&QiE+z_*^*GB`u@a_`naN(tffk!OsJQG2hqPmc?;o*3wp+xa=AO@+lhunRG)(djk+oi^htdcQ>^L{qk2$$n}^XT7PjGDArl}6 zw3d>jxDv#xm%`X)hHmW@TTx~Hj_$S6H9euWq$DH`M#lQKE=eaQ2VO-9gLofzpYW+a z0;iCSm3@~E#qxXq_Zox&vHu3QvP0!v;6ckgG_huX%MS<3J$3d!xx5p)jJKn9O6NX3 zWE0S!+_iX;JLl+&d8t#dlo|%_SCy;ZZs5@OQ{sMwP3D_=vXK>q^mRu+N2{xymXdN& zi<)E)8=`Fk2!aCqR^5aYT+(I5?*lf7MrO%@)gnJ+!x?g1_?koT@33eCCy?bOfGAn? zv=TBu^_6psg27{N;52ZqPH3Etzt{!OC3J*vrSpHE3Vu?wBmdhQN@M?vdl8Kz<<`X* zja12eLWX-J{aBP+cm@}Vb}QZqQGvMqw2FMPk0w%^s##b2RkAj8WHVc5&A-VxAPy;BMG7@nkSeeG2M|#c=xd*8>2vaO-Wz?} zw&R|w3oKRK-AE94v8}i1rp3i8fSDXP>*O*;=XIsfs>@QVo|Z|2t_Y4Yq^o$X>XB@J z+K`IxISe!W(tLg}-6VLWX+p+@oHy1Jv5h+RYq{XQ6kyr*E6`W3#;e&ivAGhh!oWGT zrYx-f+^KrA3#d!0Zpl|^pK1NchTfIHHDiT{=0bxr<9VpSgSytk9xjCdsXT#a2vdmFfV1^bsn|)hu7a{$HIVC;qT9 z=mz$w9%x-Hl@W$nzJ!QupY`7~0(0Kdohc);55aFjzco5>^sjtFA?|vLvKAIyuSQ;X zg{VN_CVS#;u@SUF30O7_MhLCaT0<8wjc|05;%51rN3(aa=q~0&7HHUHRH#N3@4W zP|L2-aIMym#8*>yhy_yBMucUh$1dFDEL09>H7)bEed`9 zk^3UyKbW`<-AjxIZQFr9FtY4k%WAT}@I8FOb9~Z^1T{V=jqxcY;lVBgMm*cFRLvzt2PPC^(HvZ=zvv@Xx|_0!dN(o@m%} z)$2s0zJlWEFJkKrhWO*4_+@Iq{hLsMA%a*XSb2^7#r~43_%T%2S;K=Xp)d^w0JeR? zRb-3>^?CXG@Vsp5w{K>|ccWz|fFZXApQ8G|w+O^{1l=`=+4&h&ciV$aINn(THLKFm zm(0a#ztc>QB?39f%bdS>+S{^xdbiMbxl7^sL=^>)jO!5!1D2P(oX?|fLj#waIM%b2)3NFjUB`{&z00=*?_WzeB( zS`g5Mn`%XXbyO#c`R_gD2Q5P3Gt6z-R5=dxy?c5{_#qti3%5mjUyU1axS%P+IWrB4 z>N@Eo`M#xNp?>FbaY1vV{7#d$a>HjYBew0uno)7bkX2|Lo{DiX`4&KYIrhy6*u*bUc-^Dus9 zggw+}pcVgCSjKZ9_49ep5s)I`$<1C18Ib0||dEgk9D>c~p_VTk587adM z_^zpV-Bd-6mln`X>}Z2+O+omv*AgJPg-B}5CQ&3Kww}ooymtM}>1X^k)};gTp+Eth zp|1%;{lktskA2cSa!+iY6kIQ6VIj^(Ov-VF0hp3!N3oI6I~n^RAc;KYr@D2`&02QW z2H#!H@aLg+ayO9bq4}Y!@4wHr#7>~%5$98%kgobHb3&nEZx1*}q%W7qJ!8Wtz}KH! zwT=U*YV((DSA_;g-Vie4yZSOR1}>WQIWL29ZUNl&tY(O`JbvW>_Pmj*)B;mizj6u9 zgUI_<>Ww*ueo1bGE+ASO%T^Q!uiM~UY)~7mBM-6==n1!aO@1+uuRu<6P0+Rrg6Nwu zAxI1wVbKz-bVX1>3S~~#nSf((Thuxv7C@npd!-Jdpl@qNKYHmwJAEI>QFmrLYaH#U zr+&o!M4gtYA6kK%6AFQKth6YAfi-vLctfKVQL30`Pr*d)o4wjO**U?`;(fUEmSRykM-mLcdfUM)t{U?HUs)pAqav>51Uv!tqjgSt3g$2ksc%B(`#HxhKPlz#ciJq;_m zg$iPDBYu=EVaL3aYZ~>>5a#f|wI&)BKu!HT43K4y> zGB?d_w_9PKuRMi9`OVdWpc6mI=)ouI^| zl|6Z|$x-CL#@D;a>A6uF0-NOU6zsm6oCDrEl#HXx{^~XW>YR=g4sBJPwgB)^>7o5l zF89>^q0VcER60Ri*e5|g*A0jtIKLkFmI2$A!6F$ubB={mLIQNi3Rj==p;0C~&KiK5 z+$4d?g3%XTy}-EV^!LP7yD3HVTcFqiRQ*p^HjFKV9m8k_BAXtOgq$^j<|Wzeh_W73 zUQ%@NB+TPGS%*Hyy&paFf$E?Y06`VjopOxsF1)6)Gs*;wgY9UWCYw)9$jMvNyX-nN zjIf*V^9o`lbe%^?8};vRckuG>8(@oRuA;5NPf@Uy%qQ5fX-`7-$MeUId!@xpwip4& zs+ats^mIlPF<)!gYwrNK-j$*qqAbwHHpmyyKG95fINwveCU8839Ius9UZn7vYHjW? z#tmy_iD-=f-MY;Msdx+gJXdLXsl?pE!Xpp6p*1xl5d|bNU{oqXavCsp93i zvt)WvC#?fLkKk*;GOQJlbawuq8QLMK9UA6fJ)5MF#@qo55`R>gDi`p8Y zl6YgPK((v9UPsH0;t0NdQ`djaY&8-Hvd~@Il$efKlCpK{q~cD23U|H6@?mNch}FwplP^z3$w&~YRu-BF zBk9<#@r@ls$}4Zo4(?b_4`lzQnAEPwS~);d_p99aR`aj^vI|@ZtdMIAhcQAIqcpco zX~2!=XVQJV_k-o+%QOoL@!=wHH$gWTNvZqoN+cpYb8Z<#Zs^_PO00eGhhjcNV8)Ge zU;>MIVAx>q(m0!G_2O|Q(Y5IJ{|_*4TuNR8Xe@)pJ>zXf{+!9s#$%Xi1AduD&FJ0@z`XTLsYP%m8=)Fnj|g~n z`^w!yy0KG^gW;sX?Cs+~n_|uEa1>H%9Oe}o)hAkIj-lr054|pb&pj8<03`lU58_rK_t?= z$ad*Q;@A;(`BEymxuM;EEDZ0_8H@9~K4UJ(H`>$7?HRo3BSat+k+(>+p=8#zTF6H? z%nNkIci&DZ9I^KL$0oGq%v@Vp<|SHfd>9jVx#0Vc5k*~F@V$5s%LA}6YjeJlv(xmn zV$>Xj5)98mRrTX9$i19uRWU+$O<6B{z@O`Tb~lnu)kz)*#w_BQHaPativ}AYV-oRX z){Jd7>g!qV1%CQH=;IXlyFke>R)kT^=6VW(9o2wS&ozO-kyC%; zvYulXYb2WK(>$0G8d^G7@}!ZD_7E?~c^sope3T4{!t1B%0}TDqYq(yj!N82qb{cQA zc`S2Stij(6YrxxAML!%ux8|W;Yksvc=#CSZt%NaRQ`dGp*)f^=Wrr?!QL8Vs$zsRT zPo2qtt6^L$>`PdDBOG5=hHe$vGeZ8*A6+hm<*7P z@GW=Cv6^vTXNc3;D5_5$LVW3r8_eaKwzq21k#H!&QOFN|+GFK>u#F-K%<5lXx)lZN zjmLpD<7d7lk;!0HSj{VnyaLZ4#RUWwMKp=c&#B;^0U*n?mO=WgVrAuv06mE^y`d}7 zSvN;)CzN7)MXFcIXi%clXby-q-sQUrAUyZ8)Z8k6MB)L=Bl=&$v~-i`7lD@XMdH68 z)B%Y*jsq2fNSxKVZ6; znF0Z;=Hs7cX$}Dtq(X)An~*S4+0@0YAbMNP%_4Gg1$Q$WWmaLckxl8~3RCKN+$>!; zaM|}kv=ApGmDda8%dHk*$NY=10ChLwHZ{S-uHr|6(L)6Kty8=ENe>Ev~%Ty3(CpDIdLeZM>EEPK|ic zKpD)!Quj7#!_+I{xQm~$u;WeP2l5vuZeZiSE|lw@=)Z~D7>*B7?J&o(EB(h97W z9%mWxMj{oAsg`W7PKlxn^<>))0%$4hQkvMr%gt9v3l&pA)irN!wf!UR!^1dCLE=T> zgQT#zwnIE4Jj9djylxvI$4JLI=V7og-ErBB)C?3$Qlm~?@c{Q3{E+-FyQa=|9jAy41sRK zIT|pOo>aL-$4X+OLEo8qxIjZ}8zrwQ=9CFQv;=MN_fl2uSc3}}wHUxmN-mcl9LoC& z@2T7A%|e@Xy|u-@CMs~s~diZY6ZLbYsKw(>_^DZU3u0 zB2lJcoCn-f+7I`uRkaY2K|}CJ8}JV~5giTz=!x8a(f2eVPoVxi@_uyTk{xx&BR`Xq z_r4ZcG`6$*9M_}2$^ING^}0L)x?NgOt5^0Zxl0X$#Sy(t=V|Fa{i(soNm=g=SAuX;a71(MjTr>&x$&fZ`_SpI%Mq1;PU$$jkgN z`c_O>$`w@0W|!mL52kC=`LHqkeao5{T-N`<%I)pl@qW4)cQ6>_#%@IcMi=xJrMU3| znp(h<#hvz(7zTY5SKIB=bIj~i9&bmaALXwh%9DP$&OOjaie@*`!^fB0SP%=8HlnC% zMrgI_-~`de+lxnKi}DLKUPOx6BxFnWRiNT|gFXY2;hWN2a5|G&@dEBfdKnVxjEtWLJ{cMruEoDnU= zz`IsDVIb;u7NfP#9p8<@sd_3qnHX)Bf5)AbmWe${z;>2FG@p0yPDLKO?Z)lU zN8BK%9H69+fnu#-m3t7P3GpO_h|!(5LKz?WtAy9ntnBuhWi^2VOZM)Q1migUL3X@* zF{!TL*8Nx`Gr1P5Nwc5Z15$D1pye9=TLKC5sxE0#42cd9#G9IdNOW8`ty814Hl_*c zsRnVD+!QM1DV=`)#Cn}0onFTOTWxz@w2U%ikNAO`@6SLtY1>Zvc6B+a?OjC7_S2ID4LlAz!A}_@UfrtdjBZ_yo!j31wJ~)Ydc_5srG?f*AbKTUAjxW!}T$Re^-n9f~Y;RlJ zs86?YXzlnS{Z*hG;iLa?FW0gUHID=T{+=n?53>p+#2U+mTGmsZyvp{^q9!ne%+<@W z&dsYFGIl3iP~CtaCDAvlhPH04My{&x8%I`F_p=@2IR1l2AkuyPU*zyxW#dx|ph>_Z zIYT|`MFGfPxRi)z$Udvex~Kg-+p&W)&=rXb%+q6>c!KZ%y6Q@q7F9tq7VyiD|YIA zTvz6-tY<);i=wSB!~h|M;0`Tp-UEa}$pQNmPtIusyd9n>j&>dg3ZSJTK6ttC{X4#A za>RlQHEOmb!+pmxcKQT{C0x4-${!Ms=rqu)#bQme+)ri$J;u_a9iQkwJiREe=PMSp z>(9!MBJD_S?Iv(@+uWcz1IE9x*3_4h18)(wffa_vKGV15ZeUty1cnc>MLeam?v4rP zPGZ{})gr_02*O|n0knxxAiLOef@o-JY-of4t!_cFRkGIYV@{aGdIeLIF;6#D@ahVx zxbkKmjLUomk!nvS=d3oZo^+IzJ)USxJ=lsz?^8WWv#>(xTyeTqw-5b8-|i@9_wIa} z-qyQ@f9-&);OT^lk8*Seo(ss|_F@1L4WirhL_I3~Ff~}tnz8F?3x$M@z`KI^?5X9K zNzqtmFA0=FV7&yLKGfe;@n&u7jf$7zAgZ5}zK;XyDA~ZrLIDV!`=`1+e21|@>H&(S zuzV<$n{P_X1<;a4q-PWNBy(`{=7@V12pu^=`#)H%$G)z2d4`aMU2>G1>?|#0=gE zB?B8yNW#_iL>{<;|I5MTj_6zOJJAoQiu#7>W4hNiWs76U+;}0o{kjgYj(e!Uv zjYTg+D^iCxJyHJ;5{nr{=3hzZc%ua7fGsA+h_!}hB?V%qw%#V2f_lnnWlx1E;-QDy zD@bq~=Ed*9Vmvg{_r%$&tb}GBw8GpvWv;Mpfy&R~+a(xba1Uf5_>ZWFuE2sYTb7+* zE+J?FL&kC^B^}qZSE>cSFGsU_!PCuFyO)R|#Lz+b9m`-(m(sL_qy}+7zs^kZ31s;% zq&}CW`q@llOQ41oqEle!1fq~?(-*++U?x=1W7U#Wkz}1!~TQ2$$l(nSvc8UPfaiW~^9}ogBP!kZESt}^Eez6+1j*fQa>8Tx4 zx%FV`FB--n5}(5|N6^zi=>USLdV2}wkS7u%MNM@htUtvFzYDx0+?NL=PQaaX!<2oH zqvO0+YSxGR-6M$B-Pr|pq0i1|d9k$aN}4Kh;#|BLkpAV%@z+cHlvlVk4f;s1{toYl z1=VN=Na$B%lQ~zDn~eG-nd428BoNpV_!{pyShqHk#A_|M_j2PG}2d~q6> zYE{XXTxoMmoxLc>@>HVk^de_4P~IPOw!k$%Ra-v*b-GDbhsT^_?x{94Id{!wF>pC- zwFoM!Re7oSFU z&#`-~Zwi+FMUxKXDw$4Rn3Goip+f zy5e9HJNjJB9g^FzUj)Zp?-utvoQ(aVnu#k{!7NZD)Ij0&4~#XBF#`=Jp3_gOHEXuY zW9-H|;{)j*)O_%WJzIHe;+%nO@_^udQYm!;7KbzUM+DbZG?irYy?FiBY1F%7KcuL%%w|$$idrSx3fgTK;>cXP3h-IQnRC_1dbtx)98a zCj6Tpfv6u4|C*-3DJ83mu9PoYP^F5rWsUpn&*s^%RLP!-9-%x)w!_UYAv^&`^BW?n zt1+l>nm?67A5@k0+<#$!Ou-LZL1We9Vi_VWc;(n(x_(()3Bpws26bw)19SW6o-Jnl zpxlp}MIjr4I2a$Sa|refGR-cI@g4w@II4e)gJoarbBA_sT8QWlH+a6SuJRXM-T)U)w>3K8XNM%OG!i^Y ze3Bjn?4=3 z`EEh;ThP2={VwpHqoca|xleRKy_J&%0WdVQh>O;>yR!2vA_P3V)*osuLYfu76^PS% zw>q9r9bN2)wE)&~cW{kcC!HGlFqW~Lz-B)`PZ`>3!GT8~hbj}EHRa>CTr_S~wkbO- zlTRb=YrR14Uk8>(O>hX?GY~%HeYnEJFi+f>uWh7tBBbMSox+0nWFru5C9fu4o(!$_ zjczt&f+aTKMXtO4+r|HtxVo{NNMigZ=V3=Ht0v)4cyxFoc=53`vYMq!8iZJ*m=Ox2 zO>WkJZ#xThP4afdKLQWMvCmXK#gi(u*CM~od7b!w6`#s6Or#cMNq79sUd?xEGRPSbA410 z_JI2YSSR9Qc75`IpiL;mka(30XH9I@%JCWVB?r|JFSYIpq`3*ss9vd_w6CWC|?!$pM*zFnu_^^SJw( zt=@5>p(xU>+=Ll-AhZYn?vLRABxg5yhBj~(aOm&+LOt+U4`ZBMvSfUUU|v}>@f=hL zEGk<=!#)L_+q`u1J%R;D_R>CLd#nPkyzBQfbTn5X7~YAEa6g_M{ll~(fO~8~U$^?? zg`gV1SA_7XA*T;>53sjB{w=GM9+)J1u4>PSB}siQYA_Vb(xH8c4tsbkr9(P%3hsoh zu@r+@auz^j?Ng;I0cbP4k7JR{+O=&Vhjaju?d(|;44Qn z+|&$birwT2p9T42BCw>h3JE~}`Pm@n;?sVHdF?CDUPB+1>o!@@kFxMv( z+D9muKT(I?fSOaEH8o9yGzx5NWFO!8V`UIC*X+$)`E>%~cW^zNKqfJB>qM##`T+)2 zZKFpAb^x04n8iIH+fLKHGgE+doAObnSPoISZpma+H;GM-M~zJQm!r4cZ6#NcAJn|;!d>x~JUd?Qym85T$I7Vd zWLb&;&dLR!o4BDE*mTDYnXyCxEbEd^J_3?ja^tGEPaen!A53zJNH-XG;9kFwgdYoe zXbpP;`T&Ud8?&^lzYI&6pD)B=5rV5?Y&vI%8=Kvy?^ANpqA+=hJ$3oEK)_o7=sVHfj-W_J? z`x6S9{4beZoXiv4eZoikIEbO+%9gh*<0Izqe&Q{RIN^@k2-IZ%`~}mAj?GhSrqEZz zf+I)aEuw0a9jrpGH)DT&yrco4voJ`+B&F~O^fdGGEp-Iu(<+n#0tKt3sBV-0Y?G%d z!780Bg|XyV*#VV`A}(tsTOkFuELRaGA)SbI%CY47fxpGCmekES*=!}%W@LRneb8^T zmZunJZ{V{?lBrh@LX~q0+|vV^BJd5nn{jzQ=TVMYoOn@hgj3c_{N;yaXwyR0LWf!y zZE|*EHGM-|`Dla0xvi)S9|&?Z>8O4XlN6W6r-|=Tw0?ru?0tk2Wmcp7nZiqkXLc+p zq0!qOLpA7kR?dWa8bgY~g4O5@{uAanmTBR&!_Psx(-@mv0nZT>yjIOEka3V_3JlBV z)w8DVbNx?cX&60%=&%e2y9(U% zj!gYa^cEu&&WDMa#S*#j5aM46&+P z*)lDxZK$CN3wEi@VY~3LRT%O z!0=K>nz#LVEcwR-g^JBs%P^}wP|$u1&1cuBTNOmM?F6#l{vm!fq|X3p^hQ3&+vSwe z-!SJxI(I%T?8X7Kodg6eVFsn_08K!$zn6^{xwO&Qqd{hU3C=C~HMa2lWwUdq5VPh1 zs&Tb{2YrNGN`TA|98FrXwz5qvu^puvLtgPn+Vb_juSRxt8t~BFD`*RX_@2OU_=loN z*<#U4pQ8(2Cd9u+fN`vKJ4rt5oZV`RKv@O@|sJjP~f6?rHg-kX__Ug!PDx93)bG~?&dEs;+Qtyz-l#mFnmPi9X> z9||)N7h{fUnOoXN?FNSKTB;c~oVCm7+z>BmkYS+|mJ=tis(KV29jG|iIH=jr(`g`J z;;}&yzm5w&w(ZQ;4n1I8!cML*HpO)e zNG94~DC4M2zD(2Vbi9*t)-E(2x~&q11m&V}gmTqc{BFK^Pv-tRt80R&z)^abHt&`c zUr?MVFWYH2|;Roc>$k4Dctt7GpPIfKohJ!P>@E=f_>r+?afJp;4>9f!) zF`gubdq3exQeDHt2ieo4SOtfPaRt?yGz$s`GJwCD!DfdswD53(u=*L#I;%~p-&aC1SLB{M>W4T?ap@^X zSWPiNJMRcg^_3H7qzPY+CQn|w7cvygKy$cA_XpSFof+{?pq-@?n|Y!gtfRUJ@>zu| zNgN4fM;6)r`rT2PPS zw#X*{PW$Ke#L0!ZWcvhT>f#F2%Sb^}p6F2$o4qCL$^u`Yb<9S~m{Ru$>Kd5K-#M_; zY^M^|bIu=K3>-AUms(wOZ&FO;cx(l@_{tf9yVo;tKXRi9n!>{iOVtRSUhYTO0rq3u z40v1Jpk21uTbfq>0E{}3HkG1BlOBx8ZE4?+VD6sV@5dHB(Lp+iW5Ki_H96I>9j~12 zTG@i_)>|wn+V6dI#N@b*F6oP?SxO}p>%M0fwr#@$mVx1wB}{uwW)B2b9NL82{;BrJPY=_)NC{fVe$VI zAQ+vuCYja#`ERrWu$3=EP{P<|9Wd^x0h%6Ar+#_z>xKOB>xm^*n*vd06Ddg_Fz2ei z4X6rb+Om$ef%+`Le8fq4viq8LD>+W@l)f9wI=_36rKZ4mJIGT60KAn7N!{xy&s}9B zBXTOG@_F&|#?7CW+AkOg0RjtS(~Te2GA{{XtaE1Ngf)TQDq$QR*PZ9?1&qfo95#Mt zq(2g%Ij?#R>^z$VRVJqH3R%K{oCHy@v4=AbJ9Xkb*sOoAof!#W8a$mfI<53({xZ@V zG{SMD1=m#Mx7;~H+7Uho%(8Yx}ByFJ$^ z=_!9}Xv`eHgB~R%QlZV030Ey9j9}=%m-MAE@-A^nr9?2#{UB_C+C(vp+U+cm#d>Mc zKcGs&WCwN%ms2I$wlZsg7=MzSv*SsDW3>Gj$Ml2G>8&hSEo;=mTiB|85?_CANA8*@jU5j)mYJ;&s4o0 zdKJ&%x<(X@4E$S7?hUODOLVn}vKrBip2GaO;7zk4vsvFE0}anddNB?$_?t9G-w)az zI{535b2JYu7d5qe>Ug^Y9$)7~>jcQ=PlJ63wFn`G@#(5ghlAp>iRrP+w@_}3UdPrR zA~`?djNgc+f>AsRNb2;?^36BLQJ?n_mFuVUFR1ps3C>Stjc!;>EufI1LmK{qE%-Q%=`q}#tZ&Q&TwH?OQxbCK`GCC+Xg`+;3rJ^93f!@!5| z%8WghCe?rBhoJ>|; z=^v=L-bkAo%bs7r&imVyMDwmW0qo)fwK%sWA%vZ%0j}#NG-{o$e?v&Fm zM5%l@&s3w%_?QfCB2Kcu^0R5aEdz=nLby%Ag<2_7rV>7x7PAw1j2DE>cfo^tfYPoA z1Rure*uv{`UNWz8a)7=)j@0gNTtGpEVOJq`qs>0z<#$sEnVnebfDqgC5Emi@S2V$> zcgSP_8>JOjoCZP!;f~7`1Ky5Z%>x)jGMkK_6e7V9W^LFp<2sk$=G!^-vR8U!+rs?G zvq#h_#i6ieqase^chR_^O7Wxw?(nY|6pHkv^==1#@M6U@C?2q`m&=u0|0;rb2(nP^ zLOZHa(lx+XH<0}p3;Pncs++!6jG?4XIeBLvpE$m`@cqqEeaawp`;EfnU8T=WAZZr* zy4RPGFM(hY)(52zD%i+|?7sbcrm0cGn;B|;koTzem;I;EfTdTP4_U%CtZm=bi}x zK0wTh$6Y9L)fMtCOzxn0MglfI>1eU5D!OfI=>a`+Q4yJ!XS^aL!C1vbj2!CR$ zf`73u%%h}OG@bM;Sd7)|8@c6U)#GmT!pV4pmaBPUh&9Wz<3Dxiv3`b!PGG#!XN@K2 zXb(>ZCHVPc!&il{qi{S%GX%gnP+NyBi(ww83SqdTti7%Cbcm&jWB(17C;YQg0|FyJ zVBsY9`*I+k<%0zSv5zadom=C=ydQZURh^1P;ris&|M9B?E>+dWygk@<{1HzRPe1RC zSCRa)YTTwgO?E^MY_l^f2yU~;`=nw?q%t>BB&k{8Ph9Qmv1u$U>Y`+U*!KC11K=&7 z(&+COn4KQqonUq#Albv5IIlOlOC>o);;FVSG28*hBtGx&oqO2)V%=X_&7vnQNx~dz z@x#tNVv%6Hd@XrNqr-8d#k{<%lz0mBzdRAUSH=XBc*~6v*`nubmzh4Z(`^9BLaO7h zCIR>XN#5Nylw+^Z` z9z89rM%~VLjek?UtB#qj$oqmbMzqMVCseL+-CC7uk&v5^vvs1j%GB@9cLjr9Mct}j z>_UxDnvFXcOzpJmaqqfWQESs_%7AG#qW+S2jnfC36E3DiVT##5tsQU+Y~~Z9(T)0* z&5cgNlbk3;oR`N4eyHk``y@reGn|P$L;6aXGkqk~9%M#W$C6>bA3^JalbI`)EcO%W zN#8SGK-QlltU~#0bjf}DdMj4vk!zhFoZ1SG7~ZTcok0iJ90m5liKpIQ}&RGhi#r9rU}pS&hV@h z*H~-xqtEk`mNKbbVCwwyym3Y%7LX{Bq&^c)pxe7ETnxE3%N}NI0$4Z&zS4a=>17=9 z7Kmov5-&(Ify8)2J_jEY*%y10B9CMWUKAH^iMduCz%18Dnx&%jT)bp`5aT+SZXBt4+-J=ArX& z$d+48CV~wxKO~p6ab7+>&pN4L(e^QJbp9skk%*qKT z0b}9;*d3*=l@DDdyf?7(Rnf}PGwVKr1&578PDytZ)!dnd-9RVfK|~`?!Sw_AnXiWY z*$ch4VeOYg)RY5twyqIuXbwWVv%?U8^`lHRjvOxbXx%VQ`h?;pw%T0E!U(v93n`9%{w?TuHRVEFI< z)_O%b!8UeUKBRF!Eb-!0f5%L)J`Gp@`D#yZM~74?MkhI?i|KA8bWc81tl$V8``XRw zIIp;WN_`y2;gT4(ubigQCKYGko~zTyheP}vXn%Dzf-;`@wNrul8JawPzi}3ElqyCX~Z^nOYXEltS zupSOM(gXL!zhkb^=iXo;$R++*ezFEVMAFLZfR>ooP4VPzcyfW$Zfy_?<1$+-+alR@Tw1 zz1cw$_`4@~+zL6q$nhaFNh_wHVbQcRLT<1g*q`gVMLiK@-!ubdEBqBwl%|ABW-w7b-JQxZJ4u~OcN;?X={*FO;*$1k^~Q15 zmk2erh*J*obO`OF8hh)0?Ld(c^h8}RrZoUhsEM9O-oDR3+--#!r9!c^8VtXgeUwOy z>!ZtHwtWew6o!Y-a6|0d6aZ5h$mtiRC=XI$M5m%|`AWV4O_#tgO8R>4gj{4(mwL#* zCaX2H|M4?Y5*x<6a6Gx^VInHYCxez^3v*XKzs%o><)&;DgR(3A?#$vA(Q%~)s+~e!E zuQWoqgG;Xfw<24CS+`>YH$@c2w?~VBtIj<)*VgUV6)FyKuodojRFjt~DeuWYpw&l# zHDa|g=hCwYob&!)VM0(|II=)4=?3G@7nVFO z;|iHm18p~NBJnSA(}7;4Qa?#W7|j7@=Vr{2;O7-hgS7~?58@uEeQlAW7Q?c7-=K7wf_a3hG9|-~48Z))cf1`v)B2O;?6 z_69-6$k5|nqO~ak1WfiCBZXtwY`iKI@mLmjF&{QH4)2HH&4AZ;5bcbcSA{wQ5(etZ zq2$Gj`5wThD{Tp}MM~NHlsSz>cyPUn7(l<_AYQ_C%XJ@*IF)NI#`Gp{{gK{fe z1*+7NECLB~c}p$YKRF1T|I`(UuT&EpAvA8JuRVDP@tmzFg&_aGc(c==I*CUWc&oWcZIQf*?Jra*&U+q0{ zy1fpRYf!!rL6};X692|Y%q&zV_m7p3M}*45#X*FaN!#-&h5N)2iRX@8bM1Ik#7Ywa zu>7txq|isC8VQ*J|NW2lBQA-QIhfI+(+wM#n?6n=nu4SIcrYpTm@&U?i6#n;#Ec^<}_9cW8)L#+3ez3O&HTWr=oSk}3# z!C|18gWK|x@fwJ6?$!nS1XeQ_b>Zhd_ePwOUm^Vn4lpSAaVHP3fF>Y!gkEMeU#1Cf z^X$n^kFR=x!5|HMM9-B+nVP=#oE)ir_vYKD>^CyCs;p5TE1hB+%3Q#^#kg&VH7}7P zyxkn3$laTAuKbKD*TpD27n8rPIVq?)DGs4@5pm|H~c30QW2vMmgX=C`c-cioL?i(m2aY4>}`+nTX67qnS$aie7f9#?-gp zJx@g63e+43C{X3@&N*kn-KuS3KRvVU^xHSe4B)(7Q3ONBVRB4 zDnefX&A?@4@#O%$bPRUQ8>Z<<1C zIY_KPX;lH0UwOx)I1LsR_DAltT>1kcmUC`5kX)vY6_7^A zeXa1satE7Z!zVuRE-(tjpvY9IhPRTkNmt!3A_>4 zGrFkZnJ~2fC_Y=luO}B2F@bo-2y|9l*{96me7Ocix4u5t!}@x-vA}5e4+>k9z~z^G zZIz+!Qev+GZk4rN9e!V?U9f%6U)G$)AwMkhZ-=ZmnvZzD$e@-jwK+9OJke14h*#Os z^upf|xX!30^7lHrRgvFbJBmqk9%-P@-}dn9vYRhBxnRp(<;uo{OQn2UI!88}y-;BJ zM_9QsurkWKAPDH%4f$vJvDg!Se?`-sfi=*WJFUa#-{J)gA+oCzW%FUw46Jxpjr^hs z@@B*VG>Q)P$qE-~#beLm%fLhLm8r=bRkO7^82mC6=YKUm2s1|`Jw$4C zdZ`K{D-Sn$8qFQKPy~{#47h)gaVB7?$PmlWax!moTU3bWQ}2<*)@uM3%R!Kveb894 zDGoy!>O^4c$++DPF@NOCM~59RdmG_|f*l!tdEiNn(BM;n$B3bHH`BUX`nc!2RB(zR z#;#%tDsJ+<(HLu6C}t{T?4;Jr2G<=(Hp#9lNJnXiGfsW)n9$If+y50T z`)H&vy2~d%c=|8ynRH2dGXI}mf$ev`vTl1~QvR`Cl1*Jy7f~n`XD1_oM_)S9ue4?H zPBz)l`@m<6)$mxVKuEA4Bme@gu4tA=1O`jT31iw(m{%Y-UGAvWGqBzuy>Z;|d#W zHV3OGcof?CjU7AQz=~_&)uU`o1-HKJcFq&$3v{GEVbVUy%8$(y3AAT{G=7=E2Ctil>#_%m5h6(h8~z~*Yu+sECK5{W1$(9_hx!+DfB7$7Uklxa zKt-dIFuKIyGRuz{{5FaneMyrC(R}=&*_Etyd{-Y?CXu+O-dk8eCub`u=OD)Kh=IW+ z7Sy)wTza&F4vE016>abKtG$l+4uPkQX)w1j@o_qfZ%3<>#Ca*31MU_&Lil{r!D#vb zH-fE{M&-XxfQ<9uTl<6!^nJsPFk#i?uI9QBbbqsY7ZXOZ!umK~oDM@JFPr+=J7>W% zm^Vy?+7B8kRV3mHGJj$88Bxx)N327hoEy14kcu}!{rX$~kfupGw0W>!1=(vdldlYe z(Yy|{F8(g>Uhoy9)G0}~d#2#q=i&K=o6qd~d2n>1ewt$E&q@HXV%+r+syTC=PE@?a zmRR38d;1#_r|PNDZBBpq^lL6I_$#?EokaXTKzqGupFzdIqW#@!=y#Z{sJ;?uup*UB zIAx#MwQVE*k>Do>BW+3lAk;@HN4S8ZXvaccfX{?Q-o+6-XWl9_0Z87&ArLYQ8LP?P zL>mqMvDH|Mc7nO%88e~EqU+Xo7JU0LiIeVTdoMRN<_p?&Bv`mRRW}1|57)r}UADy~443}t8F;M-1Jr$H|M(~WW-NaM!a6J>*TbK(68)Qf^zGPF2ILl1k+ zKYm`Z%U?wbmW+bZZ?jJ}vnNu_#^<5Yr{=5{_qfw}82F(RA48yWNeyW1&^t}`2sdMr zj4R-AJ`Wnr zzY}k1hwEzvfzj~CY4JfczyEUyto!%^u^u5XT1SYM9)0)Z?wt>wfkz5j*2odg2RxXk zh5Hgu&9O5X5-r|M5{DvzNez+l(nbg(c!O~&i}-~^eJ;Vj@too8>HUd0D_5V&f2g(N z1xjUpoqik$|HG*ji1{bwQxqlNb)=jeyK9kk0k6tqCWuTEqZ8T}MQ@ULR zdH6~&JBUXtv~N`rGWLOBs*_+~-Q9bKfd_K3DtXAB3Tx_kn-~n=2#oS5qPw=z48m-5 zV)KJU@nahs2DvaIlTrCnHAUb1UvCf6+#5s12&Al%>qPOb<^ZZ*ez}xi6pg*n?xj-a zE)sswE)fIX#zgzQ&&$vT05aA&>IcZ4=X@u~(ra>7pT_Q0+NnN-Z8 zL?q=bHW3 z_mF1+*l=O7n4EX{xr(l@vZV~io#C0gCCA^SlJM@teekqqDzq^7IIXnFP4w&z+H0>f zve0)19#1SA`>t&J)5d?lGJkT04Fc-tpcMxBXf`KSaL3!;{aQR1QCB}DI7woZ*~o)u ziVj@9E?nzX3T1#2dfp++M$F=h+M~c0kUnSU?e5Cn3VO~)?Al~ZEj1N+48`ju%74?7 z)OfEc9+Ee7L{U!PEe6Q+XyHF8l|FFguo|^Tc#rYk04jY63xOzl`Nx;Sv zW>!~>-oN?RUgC5F|4#hF&|Qcp2Lac`nhF_u`NhM%MTB7xBHYSF)@Izepx?@@?y4{l>QKb@C?23;@fq%Cv=) z6%lrff0V(*2NR2Lfh&{TuW=n!<}W1tn&!n9p>?B({p8s7zxn7kImAlHj^E*HjQ9>=#(5T zM~D_|JN+(eToUY-!vX|Mx`a{9PUM@@#_)^nJi`~m@~RoQ#D93c=4lYB+hM*w{Ib(= z$N$!^@7h*Ixq|mps1_8OM5Grv^LY}(!yLvWbXVCACexC_7W8bN z0pMi(`S>z+aPP-AZk8hXoe#@r++2XgT)PB;c?Tf-%!QAjc8d4=Fq7N;))Y)mRaR{i zNuzzHeO9(_7WEo@#~_mVrVnA=J2Xxeg~_#s+XT2HvbhFKF`~2I*ZlCDQEOmR)57mg z(KZ+@6noO!(#-vzVh<}sZVR_cw-8~gt!SSEGp(kD3sx+-2#}Bhww2QrE=khoY=TJg zJBgr3ya%i9D(BJ=iSD^Bg@_@h(PSbTP}nlWYuYh-8Fm09DYRZWJRNGc%0L$ng-m!q zT(XqMgbgLNHd7J*RmQfO2$9Ae#=I@C9poPjZ>i5Dgy}>mgrB#jGjl%`P&?Mi!IO^K z5-{Zv)UPel3?-T_(rvm}Ar_=WfwRvMDpw zAvcQmus7n0^kg?@@3JSv=!6%&{aK!KsmTK~YoQ30H@Jd*1@ki1m;jewbNge4V%18? z%pNHuve`r??r{ZKU$b}?UT!Ex=kD#vl^>|8c=tPr5$$iT?n;xTte14JMCI%k!pdV_ zyZU6sBW|46L0+QKS7^Z2g~OZKV<+_p)b`S~|99~zPT=V44nHeUE&|#Eiiv%D#_#Fi zS0kbFQRr@AjWKZ4FKO90XR3Ni_SK}PHjdymA-$AiInXOx(tEd?*b>u_JI8-(v5;Y9 zX8J#D%Q80YFneqd6m=+TUlCl132D6@>85hoN{ihAP~qXLx7X7 ziOk|YVj0A#YI-H!Gb~+tIBW(XvAI#JHRZ@=_^5Y+sPtu-uZd8+_~{Dm|LnbwoRZksF&$)9#b6oIM9V32m=!(sm{pfy2&Ik$;G55ZCZ(?QFMYyB6h z3QxBTKp$11^STIIC_OCn;#N5-F~$lq<~hU5<(xjMt*c+5^Nk5K(^RTfCsf|R34Y2a z?dzz*h>Bi(_65b2P{F$1)A8VI?y^Z|K^h;OyQ-f&WNXoOF}WN#QdMyx`4H(3#@QsS2`~UBq!&0m z)x&`_QZ~8WJbjlRTJwfE=No{R6IA6?bjEEbFWy+&42(Y^nL^COPWik=nU+(Pu4Bf# z0Y4G^Hb+3Oo7m3H{g}JIoFaYxL5lpB+MqB*L<_26dMcTOfZlCd-s*?5*o6P8L=4lp=SC(YcC+7TS`0p|wuxCdgsiW%)9ft;5%JrMR36yp zNHjfi3tWVy77fz;N~T<`Vg|bpgnf2;8@$tEt*=#Z6KUp-JX&-g^wA8hw-%No`To3D z97c_wrdzEB64ljRa*xyrBq~}7oMPcIo0Z1Hqt1EKF;`{}NR(;;+S0&4Z{NDO^PphD z^9Nw}Jb=vw)N?J6VK;0HIHO-Psq*j&k40kbFye;Jw2Dh8s`9r^N<%8WPuK-LN ztD@hK28vl`mEMA$+3GkdPz#wQq?;^jemC!@-DdNeZLfrsSTIDKnOPCe?GvVYr;6l z2GM=p6F|*W>6<#fYdd&=JDlpgiWGb*c*GkDjP8Ev^sE{zQ?1qR_zP>JAs+9eFRwL` zV`{|c)V|ysRTGz1XF&U%KTB;M*_}h%vZ~JITumUD#(n!+Aj$Kt>djh{`YAXi)U`Cv z&82RW;Ftw_S*jk2%gr3DL!xdI@k9ykWkNugb3)fuo*PsDK@P=$fK(l0v0^%z_}KAS zW5xcJEk+=lfo!@A`BnGA-+!ZShrs*cy!D|WpT)fN;ite!F6lTlY2s0N24K)lDPYaL ziqZPh?@$~iip}&P=h+;#>Q<@9aIlqHi?XOWr<1ns?>o@LsBgWCr$_P^ou1F@S&qGX zo0NH&M(j(D$$ItE)D%R)8Pht7kK5ZKlN=4||4=%zLKc1j{fWx@uT(6Df0Dk~G!gcvYj4WhW8{}`sG#m7SWPQRMJr<~{>41&W>P>9YQtYCzTz9p0 zXrShx9GCwd?M(Z464Tn?0JeIaq{^LWDZ#<~AQY^pRM8_{zuT6y>pWBFUgg*qf_+Tt zH*6L6_q`Qxd1EEj86|r{u@rA|@Afk@@+6~51VDcjnH%-lbCA9g~^(a_LAZD z@H>Q83j=HjFBMQx_-eFn$~AoR{LNQpF^GH6l%iN|aes-g!f^~5kyTOXM%5E%2;%KF zf|h>aYTQy0VwFMZ7jhedFu>gtTyu*fWc~Bzt(n>}?q5on6e~aHh#X1X1 zh)2W_LE8G6tstyh|G=lM&oV4GT6)Ipg*ebnnM*t3bd`4AqH=txO4uWphQOz4HFV7k zj~8I)SIf^8fXLCfHD?7h`jz883vs(OUmRu@VyiK$@*j|o6D4=qVE{RSkz?`0b~PlO z@j+87#)7!_Prr?y6E-*_ZOj$1%o>fuCJ<0E60zxO=~t#|eXAJgh2-#HjP!)~BrThx zh7&!~F+CkjF?OLl@7G43rQyiXh6KyDVT=#78J)Imi9{*1InoGfSoLb^%tC2BMM^!`cMZ}d1P9lXGSeO zN%^g1{F3gZ?y$dJYD`k~MYyia@BpOlabOEi7g?F*u34Zvl2I} zK@z`1$!tS_LdtlrBzN5(#6=|Z7;XPJRwvY$VF6TVMgR894rz7_nUVTg`*vt^e8$s# zjGvR=TW%+1N}vQEHzZgSVj=cmVZ_EFrim*a6q7Mi6O_U@vbzq~H59XEP;eeK&j%D8 z{7i}%Y2^r)G4Su8^@J%y8p0v-RglC;nW?YZSnwVPeYW>rnxJnsc#_p${8lQaEpg&2 zq~wGH#Vqq)?Ypt1muSPy16-t$lVYE+iqY+ppUllJ!ohU@vN#;KfRgg-;gx%K{vH3e z-Ft452)?L?2_v1RREX#7em{l)G>_E6(HLzYpOJ9ne{z;irE z<1(87)7=H!!GwMvi!I2-I}B%U7rXAsoD7QuAaaj)vB3=u9zRMPj2JRzn((3zduhFy zS@Iku5qINUtY+@Ffxjv~kptkcobqKfY?aPpz2dNy_WAb%K%+4mi|*T zC*AkwSC)CSm5pQ#^WiM;!l9%(;R!-Y!Ca@X-uae0r+1S(VZX&uQ`LPc&-V)< z<9n?co%f1%)xVS0BaLL@!-GET=g;shD!5H9TYSpzEt9Dq=%|))jD)|HU$8%ksuZtO zm^ux_VT$T@Lh0Z9mpob~e~9Pfc&TX#HTmng;*)#8i;YbZiDha`AX$1;3UeKq7P0j4 zw=60b@2=}BQrwRe^P+#GpbO?HX~e6ZnVLM!s~qk#8Fs@6&@@`do6(2j-g8r(E5L~UZp_Pp7R`N_2HBqX znC@g`*z(4SS5#Wz{@?CiZmJIRuQP6{ZAax%WD-3sEA1M30*M}sH3?o6Y|uf^0&;*E zVo<1jMnS%_PS2n-{h4nBQU;1(mZq&5J7Pne^o58MsN?UN)Ft1J8fRi84ve37UQYfT zRuf^{pkiHr5PoHA<t5+=x>?Y}OK zyHb1+IqIp4ONCD+Z>fzNPbVS#++SPnK%U4tt$1Ji6|$Jd1Vrp=#GfErdT*^RUUIQc zHmN)87H!A9V}T$9xOwM8bUj;2);LDEs`L}A3SJ;5r`GkYJ83=R^c3}|)c0f{AG@n+d3ntRSbl}sJ(spfHRU*D*_%91hJKlip_T`2%I78Y737x_Cm z6`g0Dbz=_dKPz6|5LkDvX}~H#CNBJfSAlz_4O&z({Kayx-B9dR;(LlFWJev1i)7qH zIhLp)618zl3T8kwM8Gp062bLyZM5$$cJ$4UnHvVw!WE7xpifYzI5Koa1wXSGfq_Vz zLlU>Oa%#O*7xZO-XuE(NdzZVI=+V7=W4Jo`?o9!#$GrWDTKml7g9SEg?@7LIvoWoV z(n4kn_MV@>!AWqYVdAW-TGtW;Epw~n+Wve=UM+sm!%DlS4eO&1h*zqQo%S-bCBY?G znX2l+2eB-pNFCqP(8IZuKYI!$=m`gH-ng_DK@2S^i``qekfOWoX#5!x0uY~^CGAIL z?^R+;4^DE{7uz8Jhwq%m5Bj5C;G_?ZG()v|i)8-Zz;R{ypG|RsG0forWV}1k*RY^neWVO3NitEr#_rn zguIiw@~zqZk64=wGlFEJijoqizw*Qz--!Hc;m>9~ptf2rOR zj>9g?ZP2%uacK{950Em7&Io^Pj3sH~4ss@9(gBFx&6wzvS>(f3&eRo3I~|IVgbiFm zHD!$2{1fS#gzCe|h?#G5o}I8?_Ng^MQ4R&;`Ps`eeswqeQab!f;_!GL(J%>+&yHWI zo(sn*pS{e9n@Q@j5zEa5aQ!}xju@?TJkAnd-d*kV-?)c3z4otGb$%+zj}zC8FTNgsorc z{}s~jk(NwwWU+kqX_2c_U*(ah{jKMgG0J-(n_fY^@T9W8GUV%>i9peDY9E8pb)rXi zHEm32X74wkK5H~077f{m=0~ks1SAs z0{81#dllCN*_~nQ8aWwDmFZ`#=R^Oz;3>IWqx0Hi!fxFGUAKOD>DL;49)2xKclA{4 z`7rxUm=cw`tZ^oCRxduE6PhLJ&2|Ha3OZSgPz$EP=0yRABFh>Fg1HE-neb|u*99It z`T>N&e5TUc84GdL*CkF0Nzof*EzCkf0&c`xTJghkm1vInbZRh`Hx{O}tlTpv-{hB$#A&RW$1 z4vX)N6lT3_YE1yhAfcLB0EIfwQ)X*Age&gEVz4}>AMg8`aYRL8So4V4@8-xmMuB^F z{J%Pz4zpmRwL6!ZB0__&DV{|)KAdfj(ImVPi!P1XxY7) zldHrrF2O%#mK!)-OgWW;+F^T^|B9@RpeiZ$qqo+&a7}9ScWD9(uBN>##Hj4dy6HAGO7oR3VBa9dDdW> zUcH6Tf6Qo%juiA97v1Au}x^ajADBaKCU-(nh9o? zS!Vaax{C=EYt_<;~sK=Y) zNHt^iNQ9eBD&J?-6_Nv|8(IWqbZKRQmLS&FD=HSDUG3e0v8D5FL7{DB2#9_kB$q4~ zMO#=L<`SMM)8?epa}~dv53`w8CQ{ypLb_HFUjuD^94{q={GMUpM$Sz`i6+Ckasi(G zH&FmTK)}B;z7OMz(~eR-eb?I@H|OsFBg+iyB3NZFQ}qBV4-!q*VipP4C7oAOErP~* zkNqbb&3dtnCe3b^Y?R%o^R(Xn+OYQ{ET;jZhJ17c0}(oGKNkzIa7kABn7aCxQyt2Xg9q=bThPAsa)99XHpSnUG#}Y?cmZFK`Bf8>EKLoe@#STfpZ|XSfeCqe3Gwg zt2(}&HX4)O^8)!&4L{_^7S>e`vsBkoFdRpIuA;QOq12Q?)#4nqlAqqx1N4aHec&}D z);9pI#?j!;n=AZFQoIBwf5~C1IaAMe48%h5DO49p0`$D@t#&&vwqQV>;+4HuRO*bsntE4L5pq*abZ6QadSFv40-#fV*NGY_#+C!q z@Vo1eR8ReWI^E-p^E{J*Jkpm0T?k#_Dwuf2ugM8WYR?S7t)kkxUoa5UtV5WmOGqP0 z`>K}XEo5y^u3d0Qe>`GkOcr4CRko~myts@ljTd8A+X$x^g zn~$j0KY*$15I0eAQl!$>!Z?Siyuev&q=HQ11hA5n_6OE+gR} zfVVGyS+ezlT_4~fWeVqIg6}nRSAGBTRB>f0g`Pr~UTpcnY<`V z@z_NuTtdjV{3Gxlfr5XZ_RJ@*`IA-`OY3$!6?D*NM_2Ic`<$0SEC*5&?qDdN8jlzC zeO##JRC>89Fk9tw>D$w&Y1M|9kIz6xm*QWF3VZLDVm>LeKGRYl9$q52b;&M%LPT&f zWXD?h7#k)r@Kz2rT901$YD zZ3cvSk>5&Q8@pEnbSxmD19QT-|14B%&^741NY&D#?Rb7ttK4Dx0|gx4(`2e?2bZ=Y zfnkD~tqBdcW#e$e0Ps7fx65#oid$n4n9@4kWbGfVa6w<7_VTEY&Ij+tFu!iP2WmPT9@+U;vZ(}2+WYMw zjQqwMjsl!edY`zTw5YTq9A=q1o~D0|qic-yh673%{tk@&=Buij?r-7KNlTy(?T_IV zu;n&_AC`nenYtGeL=x*Tv=4DnQaru{gjmtzKDm&>VQu$xG)vL{M&!B>sv+_9Mdx$z|5Uc@{P1BL?5}cA1cORj z!;uaq1*}MbjK*lGzH@dR@xgiT_@yX1s3{)XC^yQ|3IArPIi5n5d`3HyX@gD9gW_p= z`))KMGosw`2aamCzt!&Rz^Y!u?|9Sgb6!y9#T>2%fE2XM6nG(Gbblz{kyXVs))}ht z?I2N_`!it5c;_#3Xj;Q1MDvTTJMWcv%>l@hXL%4XO}Hv;K07nN>n%K6scezwf~f?p zx(BkdbGm$4(H9kJL!rO(*tkKuy<`CRx$Ah-_N2M=(`gdbE-^DcTz&sg>BEv5ng%(V zmI+as#6Xd(0E-(U$1cD_!7mU_9**-nJ$~8$`4O4)r8xE`_49`g z-cj-nhdwLnQ{Mc5iV<&R$=&u^q~lw`<;kd0s9+P!MR`0c3>5jd6e{biO!5Bt(!TRVv3-0ehH|&CZmU$x`YG>n#lNYLw!K7W-eccA{^juir)J zZ)2uH&cC0g>SY!xItYA#`_I$ohX9r6`fFue^0p2O_PpD4;(T;zfs;g)Ep$6kQu#gRczF7X5MZ(j#Eni>_*IFSM{-xX}!`0{yEH@sVb$tbU$ zqHaY%K(R@*koF?3JHLtwYRL53p66G{Tgyqh_GyzOk8gYbc2+P!Y*cj7cmdq%1vYju}uCK9C#=Y;MA zt@-Ub7aNg6?eCaxWt?E+hJj}0nOpAFYt;zkRv$wQf|3UfGkm{kecDrE%&==OB>P7G zjn}*iR~wu|wB_YL8fxqChx}uky*ge%nun^z-T4Q;Z>224)Sfpjc%#KQSA4aR=zfv4 z@~LfTBe4IFjB+`Ku?rRvqXbDU^d*gDBcRDKQs`AHOv|hH?UQwy~ri! zv3gx*WoabOT-c8~8doOEl&k)jFCxbSvp*>Tx%c!-;?0YYnsZmf%7YWUxX&9l7Oz?R zgoS3k(3=A;JfPu#JECSrv8`gbJD-aN{t=fm&#Ig^lrg}3B90h|9K)lA9jLA@v89Ew zUp5~%px{F(UN|KBu!1`TPZPl8%?CPUz|v}mz`F-mBdoX#mGCXJs^&ni@BdeIFoRr( z{F*CG%De#@rzQN1z@?xME&0mSpCzgIKcx#PPQ?}wdAlY43&To0b{4R`hYY)9P$y=Da(!1~09CmOY)}Z_a8GpPk|R6Y=8yW9lpf7u zE=S-Cbtthyh{#>x9fD6YN3{W>;e#|7X0CXSoKJ znt#SJ6^X>Ldk2Ns0C)w~yk7rAXxuz)240LYgx+hqd}%CYZ~g&)+5pX%XcMhNQlT_b zm1qJidb<}(&WOa^^672|MhgRW=w|Sy-s*?z5n9i=rB??ULk*mo7Vhhagw6M#P96`> z=esz~J<6|qU7lJlIu$jJ9?ngF-HH_qC6KTYP4kUN?YzfA((C-6@ z=WPD`*8y179LFbfC&db^Cf1+GV?hO*{S|phLwZe)oH6x=x$P(}oOI1VbzV z`_@Lo(TVbKogtNOgPbdN{A*&Rj}}_e{14sWn9fE)qNS^`3&KaQ-W6a!7(xx03Qpe0 z%EX(D1qCNoGusX`+)!Tx#rUgZ8;-Y4w^H=PlaDy5Vb4a~myo)gSA3Lk*NJiR9DM6J z(3}Xw1+I6Z!Tf8udFAB2p(cR4gP3g}4EzxHr)6AzO;(@tyf}0hzbu_0)Uo0A7~)p^ zC2MuI8fb|ANM2}%S(2eIu(Qx_R4Rl+BufSCgHS8TyMlXR^aH)(9HB!y9}Wi2X6bA}9t&3W&NZy-WzyN&d^ z^(O58OoWGR==YDu?PD|p<6o#o-OOeu!9b2R8>S9IT+hT;TiJ0ET6OuN(5t!W!8r^!$T^GB4u*2 zplHjfwm&13(~iK~5-zAB$Bb2$SM z7B2=L73Z&Zu=`A4MQyo1!;_JQ?@_xT&DrR-fnt1hpO5#8m;t5?Lz{u z0WzZHtG925@hU2U({UtYFi{7*Jr666a2u~kGi83i`ESXCP^K%l$AKMM=#K?QVtf;}hr2#zR@`r~c5`f2$NzM>djMQk_INJ|-G zh%w&bf698tC&ws8y@nlNgkk7LAq)!Q-!V!N^|hNeqzfxL(;>P*spA&uDhftAJH@Dk z3JxD<3p+5k?w^xf*%Yh2cV2xdP0vKk{2H;BT~&9ZO550SOL7yVa~L+w1YJ>1=o8`b+~1?P-ot6J78UskZr+(GZ@u}vib+aNYbM-^uV2Q^@2jxka!vPYsL`w#q# zs@ve+N%`3TlG!hzLlp)%+gbk7HDz~TV7@^M zlfiTXnyO#b%Y#&=$qYbL<4{35!EV6aeIowFLVoQttVx)R_4pW><_D#_@CilNl<0Y%zJWW zqgdX|m4?;LEF*%dT(62+Aom3wOc4axa0l;pg$9COe+4e|+)^jf`Y(hY^w`S~jUq*+ zDRbYAxy9?=GGNou?HS(34NgET=K$K53G-35$4}4WVR4P7YLzO=3VCA%2WA$vZ@$MH zcD!n}P@D6bpn<>jnm9DsPRNd#bRw?kb-JeMc^&SL?EjT^>0Ff&N#1~#%fSc-`wh15 zo4m8XLV2?b;2A8&ucNpI7ak{H*b$*;FntNq()`^)7g5uRB`^2Uh~iSc1LnUxc4^&b z7j>|8P+8F!RHw3HM(&~m!`UVosCut~^zJdzMmf}PU)$Eab|kzM3it-BH6r#Ax)a}} z=^_+&Tpl|yGER}m15|IHI03lwIp8W<66+Ny_lzu>e1CrEH{YrdadSa}rYv%)4v4g3~1*4Gn2@ovBaps^Cq;DTK=U(IhX6 z4m`fSJZ`@tt|TO5JtA<22Z*7twFN>!Jo|yr_7TCu?cyTdnsEcv@|K_z;og2Y)SV@b zCTAuQ)9Xq3nAKNM62Ph<%U6OC`&<7Tnqn(c9fftP#+ILwt8h3bgK~eFI5BM)9WXAq z@lc1OHB?`6*TQG?wBD}`&8xE$Yr4BRB;v@N!+PWQ*BkNte%|QOyV%;9??K(SwK1vj9pEf3nHP> z+@ial1}3-H^HTnP&C@H7Hu36bbcYcW>m2;_p>Dm(cvt!CxMt_qX*7f%pF79c(xNu$ zJgRDDARGY^t5Dg`JwN)O+$V!2GI3S01*f+}n(?I33vaS)Hc4pgQ0>tw>Los-;%qgiOyX8dFGJ+#eQZxRg%V0Jhz>+N z5%$lG03LMZQl8Trw><}C?rds5g@-a1q%nmDcw#)Ra4KI+&vt29>>IGfAE4kMA2kGN zj9za) z%6o^sFLxBt-URLHR<%Yls)Z%4#czml`e|V=Ck$xxt|eR>Q@P(?)$tjOFJr(YQqA`PCkEarxFQ5 z{ngp4vkrl}V7CsTzaqq^ny!kv%Kl`vY;XYb&Rb2~E|`KxQ$!Qf*;Sm%$ymERzmK9o z;TcQlc5~TprxoDvq~7BgWpCn34S3aloI1$U%!sfR?xn(psWcxG2c@Y%>vZ(?&v`A<3P|{pL&7i}@rdI9pWm`ZbjwmojL4dbe*$md~&Z8+4T)0d> zaFJ-@Jm`s_ch@y#%PAjEnaYKu@l%<$P|q-kh&a@5w)QK)*d*53rC+NX941bp8paOI zCP%BCz9=3+Tg6zgxc3Y-WCsb#Tb?qtAuxw3CUO0uAM*^4BoLiht_7393i(`RoRjP0 zCREv<1`zn`9>hN0H6dhF(XD@EpQSAd%if>}j%)F;O>QUm19ziE+PVi<%)8YX3Ovzd++8Er3Z{9e6%Bd!+#J+SI) z<;QlGP81*iZSuq5!ZFO5K6RAWQC|Poj+xUXWUTZZLcc^w8-zKg$9>^WYIGO@^hy5G zf>j+4c>g7@5wYj>N>nu1NHau%;T=>%a@z)?B^zh1j3U&iA<`6hs5C7uLv$ z3L8>SMiOTi8It|PdnobrMMn)s1@)jW@S#IRk=n@5KPhGlFwSl(7FUg#x8Vk*wrKaR zgz!ceRcLs1_Iln1u6`Q%9~pKttbANRlMb}J!u-}<`b-!Rp0CPz66#%hXxaDp!huAN zW|L8BA-y#FLimmwYGa17*B02Qye|o(Xhv1W6jsc%kDe@4N;z^mE=#`NQemTGVE4EV zVRHs!%_0YrI*r0n;9mjvL{?g206cT&l9Cq=e4q%D`I<9=PZ^>9NF+;)Q2`z=23{J| zw%`hoYDxeap6Cu+u3`HMiM5n1NMA(dSA0a7tE+RQ=huQZcKa}!i`-UtBO8Itcq<&^ z>WrrQhCqAQ9O4~D9#Xt*Y6Y<=Wb-7YPIpE%+X$IX?uJI@0CCGzWm*sp0ZFkIza_!bCDOw#nzRYQw=B&sXJc2p@wL(Y|=9BhXF&^L;R<@{wBPT&Sd>s=?oTe z{7ZDzYz-AY;Ku^?mePOtc~EKhNW;DVeBB~N-A&tLkhrJJjv_Ne+wmLj_gKZv*ZgRT zn5z;Tn9H$^5|Q@IehKg4>xkJtb&b%Fga-dG@4`wry=3gARwa(NLqF#rkHh$yurbS}e|^IzfOcKd3`IK7ZV=M)A_c$Q$p>d zqm%1{FW;owZI0>sLjgaP*P)~RP}V#0dx?tR!6Cyno2nS&kX%&3bq|h&@9XLu_gxUF zXe7l5uuwIMV|LhE%L(lG%XwIpLfMR=op0{1?;h>3Dr*-CLne;%a0~@-F%}&rP!#gO zl~!!kbT}=Jz-_GVaBj!6H*0YG2$jcwU%sZgGVYF?t6F?WF=GR=S=EXY5fXGDI&he4 zOE;4Jal=%`;`hFE>=sm~6GS%; zHDMN1()MAoJqnQ{CSMtUl&|S+en{IG80*Tc&29L8WE1VNTX}8R`hX1SeFlpWkDbS| znm?9mqfDI)w0(YX-JhQOBBCgdXePp5u&}31trQd5`X)SUA?0CmG1$qoiA9uXqORf4 z^>S6gPVOgkg7UqpHV&WYcf2umSb#Jjk;N!%C_LcBKf3i*H4Ka?b;$aIEYVD(fW@)B)N%ccd+!vv_uA_SAAs65IfTK z5C163!J z8AVA%`M{<2R#7L;r*`eCb}SeTZ?hS|qe}%KZe3H#Mzu?B(^SaeZGSpFRA=E941gT4 zm3Sg91Kg~zeqDK#k>R{;o<%(TZXJQ6rDc-c5`Sl2iyPXF)jv}%H)r0>Dj8t7P(J1~ z513+9eCIC~$gDKS38)w5$5~tG85#PJlMW=#yIUh=~blgx?uZNw9vif4osVX z#^DfRP}(xbN2^mJFK5D)n^nX-MW#~erPiVfKNA2Eg=3}1WO$ayVlEiJ5T6BktptWS z37kbVC1Ldwhni4wO2`M(2xklUP4SuQ?i3#Dg+Qw->y$6TJ(6(C6p~}}HHni+Vck$A z)yFC0HFv$NAa(AiUu`}J{<|vOtNn; zhS>bSpr`loS%je8<1Imd|Awlm&(!hb2#$W6F|#~9QqX%k4RLathJBb}dSAJrYT;jC zX5~iouq(WYWSZ=JsC8+zfiM6%Ie(lr(DoW=dZ>jM&lkFP*u7i9!BsyZEw(2~0QBsg zy>e;s3i_J^`o?C*&8>`vg%y?5(!vnFay7|Xp%z@a1lqq#8C-M9O{KX33gBL+pXTH@ zF1bDhZ&hyMfieL&buVQg)G`jaSa|kZFLn6+JuTiBnIYA$zHms*nwFmWrUc19zc)-Po)?GnI``MlB;$c+b)glo9j)y6>~= z#6+NnLN~>L)&7Q0xwuFZ>OV>F0Sl@_nGkmr0AX55K)hOkiiMI&aYS$Tj2WchAM)0o z4WGb5sZ`2=xpVYQbX8xRLjO?3EF5k3CNhr56xc>WJ3dD??iqU%#+i-2ayKQyW`DEg z@r4Qg3mkx=kJzsWoLg3}rf=efIPE8~^^A74(Kv1@4)JCDgPRu_JfGk_K+B%&vBnQz zI984j{axINu(vXm^)+qg!=Fb1aG**AwZxtkgVx%!6&}TsQ#GH}k8Q(-3c2+Adqpjs z0BzA0V}6(h@aF)46rKiXP=8@9uv-Pm+KPQnN=Y8-PN$lAJaocWSAkQY( zv?Kl`74pYl%l zlC0hturo@4ob%6@YPG$+w{E1NGb4!Nq#q1#86c^E61y%g(?k#TgXc-JW`L?CwUYI9 z7Kohfm_}O1xcaQSu?}O94vxN zfB>j1iSx2Gk^fxLraxQ9@k~lTm^$G4RYz>Ps@!3PF@vj(m3{`R@KXW z^$Gfm8Jt_gpl?*n2C8RKz~^C~Mj7q5hLOJLTr*76rUXM&%fj?AjpWT`| zZyNTdXb%j$1#nLFDo3RlR1U(JlMcwI(q#dVi8S`PcPFJHFfOTH&ZYyb&1$Sho<1NH zwWJ=0Ro44)5W?~ZU;xF18GGG`8$!nTK4p!s)p}~Khn{0Dr^OwIel0VK76fHcKKfw1 zK-s}#^jncNz->(lcnw)}lm$GGLm?2l3s6sQahr&iJsCqpG(Te~$ou|0R-)Rjgew z=}LSkgk&K&50tItC1Sl(q+c=kzP`x;9K@5lqFvpLh?!u{sm*<4qd#FCP!(ptU|Lvb zJ4I{RVk?%s13OC-dBwWzuVC2ERX^~P}UNnRC^fRxO!eX zc4xRY0_2Cqc})I#yn+t*uejKNAQ^%j3SYz~#x=Cbt@>lPT)jy05HzL0Yq294?%mM$0 z$HMpL6FErxBbx3@9I`u5R-T@BWb{2q5!(BPTy`PMJ2=eZ$)E-w&g6fn=qYLITS|H2={9T+ZVtrwTuhc0h0C{FA^vG1RydEh zKv8mYgcI^O8qBC#HQKUDZ>a1mmcgk&J2kzcT=-dY1jYUlAw}&Y{1fQYjlhKhyqthT zOB!<2VC3*`SXC;FGysn>Z7ZLkTlbJ!S(8wDl8Xx!$1X*j#aYGLcu0o8sI+H`M!sA~ zv}B`eD`uZbtc31rB7xxMis}YfK;JhI=p$*|N=C?J2RJ|)T=c_`=K3wy>wVL|7!Z2) zM$Fw~UHh%!V@T}E2+ZPn2b|kyA;UpXn4OwHTe`5TXKSAvIN`4oM}xZuGr)koH2IX* z+_Cn&vtOamz<>cU)=elvK*g?JwLaB;=;!)Q?MYyKoqlB~ptN z;~q@dxYTmIabZlH|7-ib_#KQ9k4-e+pImm~Sb8fsiMpks!d|~HU0r~x&uOsNyuYOfpWWPUlr<04DqR&f z(JEHlRCs6Ot*)+n9Ul=x8t2Sx-5_B#%?pcIKAlWwGB3_iT0W8YaZzrU#&=imTkjvE z=b_;U0IExOtDcSACcni*9_&TgQX`a1N@xq;wFk?g+{tE8nd#?PfcFOjrZF&o&IA5mZmp=hG{f1+q+aU`QBzis0&4b2 zB>8*8LI~F!zHvuOK{*(N;~R_+e!)s?MInxz4kjxJqzFJVVyNrcns1&vN5^}DjB#!Z zpIdN*#H1H~uPE(6pU5|)T^f1>mS6y|5Jvx4Gi#ZV@TtU+ovYTR)3MU3^w z0lsfON$VTEQ80nBBgNY1NG7&+aBz{a;25{V)yNy##6gg%8y}&>Qae1l=_=XjZU+FD ze?a7Nbi~rcU+Mq*A2l4j8bF zdWLNvi#lxtlS`QW2JLYRXk&>Ux*!VbVAQVHs^f>@B9rC7i(pDB|3Q~1qI%abDAuyL zk}H^%A1UT^D_AWkZ+2nfOLUfo@T%=!`f&r1v)yVCZ#e69a=et-_`#S&cyiEw;Re>E9*x zA+F48b8HGRv$0{RK(poO!_w;T4wIEJ{gqxKCj>8Qj%s?6T1(?R0+p)0R3|wX=6i4N z^ZDHP)ZEw?qs3xoS6Dj7Ods7zh^H3ff5-G4PHN4pfN@D;FAL&EARgs3e_ zVkhC|%(Rvy7F9+@WpmoG958ww(bq%MgY!hNtVx%n3o!D>Ytsw|bF3tV%9NLTTe>H0 zv%evAZCc+q`(R}JZX#|UQWj(HnQx&<E`#>QfUZ>lbk({uVp+LHrnXFU&ZaVAg*(K_~^DYtOqUqb(yZeCCTGfgLkpbn02L+Al|bK8@kW zErbu%QgG(Gj74G_2;s(l9qSz$7A?QSzz%!|oS|vjpTwgPnIeP*LP z{!!IUBZ9L7PBev4%<7`(V2{%1irW(j3|Md8D?ND!bJ7b5YE%ExZ24PE&rNNOqAme} z9W(?L6=5rIS%>k%dU2s>RR*nX-v)E5#h|^>X6s%tYxUrUSFHWN*w?FcO#8Tl$yCF zI;Rnj5Lx2IoU2raeHP=hO$M0DKw4XFI(L*5W?~v&TNkaX?X=q_21&Zbn+8%0aPE(~ zBd_Y!k$&)NM0>W-Dr!h)c7FQ$~UkO`#Ndki8hl+!+M5^Xrk|0sd` z!gTML{GjAPd}%ALg0#ygeijC$sB7XE{k**j6LI|7&LuSbR_K36UE0NyZq$hIhcY5) z@KS`n)=LxGs4T4lpPbz%43o+srt#f~M0OT;b;;$7=o*G946=dkc&-M$OQt+Lk6Y9O z45I(FU1(q0f@gk6Q>d8vcX&raHEd+{vgo&6IqyY|QzY)6V6|G&nDDoL=d`fRg#70R zAet8!b+Kx{{Uac+TK0BFsZcfI_XQS5Wy+qdR0-(W*9U9k>|EqN{TYCKw#Vb;k+{sd zO{|Au zSR4KyjkyU^&Qnfk$LEiWO85(CRpjIf5l_a3UnCuQuRW(TL>e z45s}n)%#13LKHq8WGr4+8dk^!-SSa_rT&3wWp;9R8_8*dpoe@XW`XOUyDBJ|#?wRk0VVb2rM2$tzprC zbb5p)uN*&!%WnPY-mY6A>!BSz8{)Nk&GGOQ7lbBcu9$yD>6gsN77f|mOE{g zr<;U{q*gvB&0b9M*!A~s^f1EZX3(tVIfDYA4_O#+)w|zX--t*?xIlJmH4tfTQ9z}_ zp3qbt6YoAK3~_+|K#{lXhN=GaRgJA&e4~dU0WK4u?l7KMh4Xns#z|L3Q@79^vSA@| z>deZ2+DqW&bRF-1(KR*FFWLC(D}9(6Z<@&J$P{U>)5}TN2?#m8NDs&|nSD4+Yp5s} z1C`*2F6+pLNEGWLAbZktt_CA~t{jI2MR$VI=C;*(!7G;j$&0Z9Q0ZS@Yy0>ZtvH{{ zHC>kfpcyDp4@X}#3U!@&mRC$yj&?d3*5;vyg(vhxxt)0;x8Vh5t3lBAA-O}ennnzX zY|xKh%v=JX&h?XxvZ?d+D*d zR5%I*0MirI_hZ3?r!>1#z&Vu!KB%$^$1V&Y&G*-6di&115WlD1o@gKNgjMU=h5f@@ zZm$DW#wQY)`h|4;ov^OK$OL}RswL`&y+TiJ#P=?wARj^kojf?~GX5)p`Z&@MK(6S7 z;}B>hR?HFd@Ekxs)_Et-jX+I<8fpGD5ud;@E$Ss2TZ?i}-GddJbf`HF7+b8aZ?)T) zttZO)5pU2+mA`p&NontRv005~v}9!;Vgfl;Tu*MW#=I_^J`+^C5(2?slP*KeB^F0G z1>)wK{NlQx{=Hqy$!8*s<$I5GV)6Eq5Y)V!z;~?{%BXQpZ7(w1Hv}R7!DpXc@T^}w z>Y)5oo8&;DAx2KP&KiT*q0g-4U*QNQ*1%`8Lq#)KVC;gA(-oQ19U>vjNAq)tz}Nsk zLc~y?_=&Ohr!Qiab721||J;(pE%7F~1$}aS_{-(-BI;^L8wuV?kr_$9y*89=A%4tS zXEp4LT4<>a1l3M1s)Fy~TgNZRCO1sh#JB)*WOJzP8A)_2V&-D4vG&?w zl!2~ivr8i%w+nCl^2wv+vN>p3JpvSG+fBTHXWIxZwH+Sx>l$Y0x-~O>+-fQXult{T z$JzGby(pB&nz{19T)ql37uCTx!klZ34-;Qtsgw?05O&ob1WRP5O6ZI;9V1h@MS8Nj zrwm2Yiv>~SZLjsoo5bIIs?)$Xd15B*7?od#dnA%SM|JWDnZyxe2f}5a&P?&IWb;Sj zS_pdRc^G4vRmsdRgI8umOx~~^aTt3kkFHqV5n%$3vTOZ;tdnX`o`T_4?DqMd$sPjy zY{66>Mt=C3=Ba85{8ec_U=fe+$l}cQ&2|!I?rCVQshk16n8;If(N1rHeRH(vd#KDM zj{1D|z0C28^umS5%mJJ?D#2*j66fT$_=UW4P{uo5_6EYY?mlqhfc~@c z7a@fRA*=YS$Ntn0(JQ>+yu3Wx=_8QA=c2jkMs(_Da{bQU1)M4XMgh>Qvw_ZMqVUQT zh$BUhv$LuYC^iql`~!8Xg)dhzz`#;1whN$NhL<(u40}R)dHL*$$KSWUvBKaI));WT z>nd}DCxEO|C&)B9+S0y)6obxTr>7rru?Mo@`JVCng-!D<5Z!_keeyoEgKmY%6K*)i z$0#LXta1SO(x=uH7ydq}>J}DA+-eBsUvzcKBFpU3v>k+DdD+Uc1ao5g*A$(I-BQ(`p7&7%{T`FjqdGyB#SaGJ0Q*LL6u{e^$p67Z7lmy>(r`{@iRyPwSztwVS z04uVXske@bQjp*6G}I$`D$PxKj5++P@j;pD`4IIkxHW$@kvfv;1OKyBd)|!()@IL* zoLA^0l2jdjuc_+ZMYhGVL2Qke4O$pkSgMQhFVhFbEFo4XB*L})(cwzqd*|SOY1(x9 zql7vo9j}|0|9rYV;tj$z1YufX7U_*?Wn-)R+jWZVOng3d5Btrx0%V`$P^1}8UDG$#kI0gtKZ>=?cr)*%K!^~qm z8Hkq!#4wDqOHia%oPs|NeSPttivAJ$H1z8BT7eBr_$Y6gv3DlshT}~2GnBd1cD=v8ZajJRUmsOUL3jXLTQ1_ zv9ThW7s7>`ic;oX-j>b%~JIm3QBbwH^nxCDyQ()s{cq`qCC(tUXL2!YPGr`fG2 z#PVx5rwU?^Wtf(O$Ox4o_KG4vYBMZ7Qzd?>bDVE;?rvT1oMgB@WRKu2W*fq$6^fVh z11N}*RbAhsP=`Q-5A87tqZFtcfhqVk9F$3Q7i` zJFHn(D#>q3pD~qq#hK45`{ErJa=8d@I^67 ztqLoJt4KqYVm3`f)({gAI;1SSs8xLq?`{=o=|#bn=fw3V1exHXj<}JVyGEUre9LhoBd04|+w$1!5OEKyK+1!f5I78QV99><5dxf>E?5Aqr2x1kv!Vd6d+70F9 zV7}h$)6&{t(>}U~EJqgnNsYgGpI?1NX_2ZGH>KlaV4p{s-ZfC(DO#tI?{Yc{NCQ+zD!kgQzp;>YXH|WPc^& z<8FD~{xgM6x0L9+l$p?@fqp2w0{5APr1*G=`n5m@%8Um8G-jLAn5R?PGRhn0)1tw7 zZ^rNElY>8zH$9evG;nu0S>Q43o>m17qFjf-1INRLNoh4w|EYMe3EQhAIJ>!J%AamD zIdbp>IGBXAudd6@QW+YL)0;Agg};JFzmFsHEClw6K&dbesZ$S)Wh+rrz_9i6yhej; zU&N|euotQ+mf*?(xD476!)R^Rl|juw5n@+gDWz3-en-kf9<~3P#O3s@LrUzltaDP= z%`JgC-@U%%$_|r9Wa=FFP%Lai4EbM32(65^hi6I~rzFU~n#KLQXEjm`(Qqq~k2D~B z_9?iCB?}$K`CN*!XPktr5i8YuO(`$a!1wm*TE(ij7a^dJsLw1_c4iy!=3V zz16tIU=Wn%Ado*q5?KOXt8!|M@Qa-?(i#p=0aBl)IYd2~=~C-O3P{~{=SJUni;L^o zU<~$PSh9<5;U&rfv!uk=(f_5Tw53&WE&Jf=6Nfu8WlO z+ee|$W>R*iiK-rguk#3Z>vuA26dQp}GzV}Yp@x7 zBV*lxKa($g^U`Zl%7-?5#UAe@t98W;k-)Now^QeO6g7w2I8^nC*`zr=^zN4%U_2we z`u{BHZv1lMZj_A2bJ$vQ{~Xc{&~Ru3AQ&8XD-3@8-yXZDADRPRgmcE66TdmB569jj zii9h>sWh(H9N6ec4I(T6o6*#`AtHn%k-9)4{TKCI#zAQCv$}q{yZrc|2dEhO)R$Z9bu!m+wdphB zD|!^Dsa*r|Eu;gTajqR)>DZ1qsAMnUM9L4Y>ofLLcC9=-^Tfrb{8NyV!*eHjJSyr_ z_0c6#fG;GEg}3%^7S5#G6=GStK9NNQxSrj@)h05d>@MVD?B<9tf#lIO8q+&&M<~cF zB^-~`RY4Au%=mcJ?{RrQvrH-rgz+p{a~}ilryDsfg9bI5lA3ZZbM|%@E|}2t;18#~ z^Y#&H<9_n6e#J2G-v}qy44ki!O4$@|c>CpK*M#Fut63^_7mCubV|(L^?9H$htBZCd z?+I+#Wx0Cy;vv?*L)k`Z#wKxL^sy|c8zEwCT;Cw0B z-k)Jg1useT&bnW{&j;QmA)3FXd_4e1K)AmP9q`?=ACjW4Ogu3bD}zYX*6ctmUG|)j!6XSrkjh?gi@fxvwV#e0 zt)=*-mS+ZS1`K^;8Uu&+=0U@8YSxk5-8TzTpMBB~Rl_pGXxu>H*n zoKGXo$sVZQ*!%7b{|0;%15-kaGqU82me>4(ILZ8K1xTY#e_IV{5cPorPqXb0c-`SF z!wxozD;gp^=r7F3BaBTAt^S5oE15wotN|lOvVy`nPYsV1)WKDV?P;_iXSKA}cm@aW zIL@9sLZD%eufB~P5k+5xpm2TW+9$=?r&{M%3Eqh8=#OO-4(I`@Ry>`GJQH|By(L#d zlsE6`nT|4HgH-;}pq54yF><(CPnsSiylyXLrjtY#4B>3RvmBju+wA#DYyD_(KgSU; z?wKloE{bnIVr>WgaQ;2PRu&xE*-QlTJMqLNJ zZy!;ngtX1jsDxQ^VvM&AI81mXir-v2@}f_M{=)62%tA?Mnjfm7*H_*v3bsB%LlzUX z_N|lPTfxzyHvR7L-^~>O9u!d}vV;1{K0%~;y1VPJGg6WK;ui$Zq3Nms+kacVB`v!o zC&G_=MCR2xVLD4zIZ`(LQQ!mSbb)1Y{mCE)Y!~g$@UYI_&+QnTwNc=>gDb5S>ot0* za%1Hw#p0$M$591qw$tpON#m%1Io`^Q3)0PsvkMsj;wLZJozOebeg!%n8$m%y&d+C{ zf||4qq|l13`5=tsA`IkQ;B_FuySo>Hm0Pd!fN)3&;!YIk5?AN1%4@5qnl^--kvETK z*8}R?jjZK4C9N|nchssX$ifT(S_BsB3lMCQfrrSHBFAtr;v*2Pu1sA4M^G+}P3AYA z=n`n>R9|&T2cqEV%Ny2k$o?i#f#v(}z^F>)FaakTRq`Y5=KAq|2Q{b0SsJ*5INs6t zAB_|$Q?OP)tde{`l7S8)E3Ign?`2Os2#7-h7WB7DfLVH$=khqal`IA4qGgMogAKU7 z)u8B!{cVGGnYy`7-CdsHL+wBrReoClXDV_XHu%sBQKru=`3>kwub8j~YDlb=Y56^2 z`F2jyc(Tju@SegS=EnS`ccLzNpwX(*%!Jkl2hmR8$)mnS68JP`YvUOepv1}pAJv&M zy@62yO5F?biPMmS_$Ryo-Wr%&WZT_6C_fjBILm88Urn+sM~O9qc|NE!VT(IG4v$hwZ_fm2$7B0w z0mo2EL>t+_1m zYSNY*$t)t;aw5h3!-KPouiD3-J|66kLOp+PcHn3slLV>f1*!mF%58-4E8b_}#^65Q z=hxbRxL&-xPX}`26Nwy|6Zf8uP_3ezdT3O&vDMec#(8DKjmqS8eFrU#SbpIxm=-;r z)ji&u+tHt+&NWKBjiZzR0En%iH1zON4X3fm1xGcAZaTxq9+i*ao0J3EWS=GiQ0+?X zO%qbN^+@@mDGR&YWRMIy3j$G>R0;3KmsoJ|T#;^igrm(#tr7ipoVoqiV(WJwZbsytD*HRYg^8!lv~MV8p5lEZ&|d zeOCO=GPBlLjCV4@@;*RdfnV0bRLVkH+vpK3w5UMqFcGiTRqD(85@D4DgBeMHwxU-v zG;43%VLw*?@QZRpr+JSRx92q;!ptHW=a`?*ltX;%C{dxxEp#Bz31#Ao0+=*?(`S$j zWGGh0Z5R0Yj{dZwHEVcbRIeG7L{+6U0`3hQV6~E1rri!Zp*C2s5TX^onkPp4H>csTeqse z8MDeD_zpDq_>on7zXVuQ!%l4T13oC8dnIp0!-}4D5v(0K-kG_~AZk{`T>#0>UUtq* z>E6gq#^XE7!Fk1%s0=jnfLa!S*DcQ}7bJ#G z0%H6u9T+TZeeTN%hjwB)cWN#zsJE&Cv=Q_U6Xur63*lK8k-Iw_Q&sO>@p~l((-~IdJ_R9K@8FbDI9@DnmO!z8w=GRhj zVHd^pZ%L2`Q^l_6$d)WTGyzEBso@KwUn(y?B*T_<3=<4KM^d}DQN7enVPu4HuB3JGcB|? zVcO3;zu{?W7pEexK91Bl-Fu~HQ0n7VY%>B7`u0zzc!Yy{%@u0G5pO{IuD4b z&7x`aSyh;;i^DE5+f!{_lN&|FJ{JpHFBfTg!rmHSsNnbWV+Bww3RJ>j2y&LDZog^o$gTuGK6JTqBur z#Jh~)Wn^qh9|{I)n#hB|waxE3@ zNocqoxH+**lDcd(uNL@zMlvCG9DVO6&jr9nVF0uUfTBe*p~?|puHDnZMIIO`#R=EC zjvLtH4YjC5G^Z$PZCpZ-DR{uP^|)rVWDwE~Z>PUC;n4TR>mCS|ydb!>bvIkOYjUaKqrC0G*K6Iee}qAIY2u>1eq7s z6#5)u3?g1!OQoCD^n^-wKK)lN*b?;}0xJ$|9G8bu^6gCkW2X-R}mkg6|+VMPS>a9;xh45tVpWT(XBGi=$? z356Siz#mf5%^K9Rp4UGb=%8dNw{H_(aBJriR^3^b^<|~0dYna>U26Lj=oIbD>#y&a z&3a?2I3f#gWbh9G`^+BucDcic{0PJhJa7`+P-Qq5Y?j;8@>eW6~$@6+9kglWE(ThDY6BTv%>NbICGToMIR@_Tbl;s;o<`PrZNtU>wBQ%(Iz zcwhLMcgYK*^87mUrjaZ`cp#OfUhu234e^3w?jE7``YvOtB$r4cAf43wz}^{@YoxV6XFP^<3uW{4P=?Miw7^)|qROt@7AgxB#ZBqzd zga;%PsKNF92>rXbd`OS!Y0udzoq4ulh}d(6Tz5B8r+|rsR=cR zo6C~96Myap+b^If50$Yw zbr7)t+C@d0!LY^b^lgI!(T=MiYvF3q zX^CCne5^z~weJ&q7y))m&6~pEHhk>+3-pdcEHiOp1Sm^<9n)X@E;AVv_{@03B_V)y zDNH%b*28S25&x#A!03oeV=3eF*saoiK7XnoK&1`polYbLXbOkR|kJfRrZo_;TA5-~o3(tHxP+dP&^>B&R0(^cF z>@kXi@(mS@xZ5+4tklYcV*WVkTtSVFgRfkT%;h40_JyQIX3J`L*E>4mW?@vLg5H?v zB0G0oi)}`N0rRL~B%5_5*4HZ|K2^qxGwG6)b)53sOp$FFkV}l?KWz%H$(XXSM)dZqLLsK4TWQXFLuFwxpt?5buht?ZI)77Zhv(|fibI3k+1fR|_iU95Qjja!I7TJk z8Fyh*B8MAO7!8=)SG7Hq+r=EZ$0XP>?K4r@D_xkFAMh~I+fV~XK~?zlM6!;W*~7tq zW(e`jE$m=-I%Zn(Eg!&tL!eb`;V|UQzMZ0E1Yg`*+fOI#(s2 zS5xfoF*R!iCpfs}R6EqJ_2{^|MO;~L4Pxr|ZLeV4-;?4GOOQjSNBT+1JRHcTO3nis zQysZEPdBjpThD8Jsob*7Uhg*M^0<3*j+{r#G) zbRu$iWgj00yupD@XvQ&rB$$BeTj@bP1VAIS89;eJr~B}c@u^AuYx1%P6)>x=>RaE7 zMX9WU1WELUr zJ6Pa+cz|TbKa>I?C11@fr8$BQ^7%$mcJXs)Uzw6&oWLsO7UnRSUH*3scl@F z43qWJ48rKDD})YBA08=A#xD;7jnV9>Mj<2=ts6g_i02V`4WjR7)E%&lFR8OB52y~U4akb{01l}0Pw{%WOC^U5DjKKHt@ zw{@JUN#5gd$Q}T+;cUz?5Y*i0)Gf%s+`yvfw~AYO9Rup~r=GaZ5Z#sL4lQMv_r|U? zx&X%GXh%1WQ^OZq%dUV}eV)XzKs012Ddq<&sAnkDlba%%KD^52Te(ltRN4Z_x7jd^A-A_(Y<>lhoRdZ=q7hV-wvoH72 zE&!0>nK#KH@;=|bWSq#6sluWrz)?PFS5Bd3pHgxqk;j6$nLMYvwm$ciTPO}$ITnHP zJ;;d;*Oc>gJ1lJ5n;M;CL$G*IP0p2d42v~uRo5DuAK(sKEokm%jjsCLF!m(9M|O9M zf|PYm_B0B|eX6hw8UG%+b08Fc0d9WQ26_N=`SaAJ8KpL#YBr!IP&x`nIT0zSo zoY%j*jLYMBrF|M+MbjS&^U_xGDGagI_K~zm22vr`;(=crxBrjG#ot?g7>mp8%aIxT zQ;b}bbw-5=ox?mmMc?EXT$54q2~tf}I{Y^Nqi(-GNgkBVlA<-Ra;*wEfmSBA-0rKJ zsktFeFuvnpw7Dvu!I!mxsU9RjV6^n}vIVY~)}Tl?352@ZcRi=)t2I(iT?r~Wk{F*x zUAqQI&1kew4I^ow_I-(O#l~TtiG4GDx=(-zx~;`;Z1%hB(G;gaxmR!k^3WL(w zTS>F#sOJyzXU3K0es1K}rcUt(-bRt3+O{j}G4%yFp!Vij97>YF)C!%u*R`g(A6y1A zR&zZugj%^<`tNe@8in>>Yn>g-puT=cb=~Zj4|@G~vqx;ywkuTQQI@f%{#5`RFRQ{v`#m3PT<9GKfEn(O;Y z6K|gaDKGFIVqs#6zrZ<3grA4n-Cr7zn!9Mi34X1}tQ1s)CB0>@ES3bLqZSn7&NA2% zG6=e`bg-=e4H)k+hoML9K7tlwA&lAKwuC}&bL*q32j@u z)Pam*yC#p}7}MXV^~~z_^mOb9E-bImv8BYdbM>?>_n*UHi!u9QgT@t}O+mLZNE@PUM^YC`xe1rSP{#R*5GAtQpAXJp~Cu9Ii*uNO+8{ zOE`a8w`Hp(9c`b;{ro!K66ofdej@=L?rMN!AL^fTGy%;=u!WJaMRAM z74|PuZ}Hn)JqcMYEkKN~nuG`XZ+&b}q?A960UO$t~Zb&a|Y*8ffH@beI4Jj?t! zId6{0Q*|TqM;O1{KYZayV~_l==2ndx8&B*cA<8+j1d}+_3NHxPiy2c1U!{735V3SK zG{nN{Knz>u1buplrfC!>?EprcE+$`%Xpv&EF*T1Mwsserf;JrvtvrjMXMP$D3l**~ zA1k%x<4vma=y>EfPMcm8t2pY&yc)29(!l)z4vf;19lQzY9YQ8y>ooOkTIG=>pAurm4#_U?Y13T+bf?=RWIWFZFfW{g&`yvh%% z*221~A#SF4M$#xr=gM(IB~ps8cUD)$U=M;I8WXQ6eP>t=zDK;|{9aWAFMk@fDaaV* zU4nRrGh~fu@=Bpxwj z8$}~F-%y1|H=^3Q=|uVItgbFsgea@nRS1U?Q=C?PHJq2r40h zXE9ND0sb;e0o+IXK;U8N;P-Q04h^QkKQh(cbEEd`h!~!vt8C-WR;b!TTh@kj|99Cb znc;59S+!B!W2u$T^oy`XWKm#ufl!A*basP?V%kWyTQczpANT z9|DauN!cU(xfA-S-d=;05=i)i=yDYQeH{sg?hTGL0-emZI zFtqnk&UimX(FsXe1vD4;u{SJUPEQ@!4*sV|DSm2s5?EeE2U`P_Q2IYtztjppz=B~M z!~sXrd~cPO6J9N(VPp)R4d<~P;I~$m%GKog0n3c$L7}a>TKyOek9&J0y;nv#6!7lT z?$VKG&8P@YU}NlBnEfSf4fFq zAg8t-jz3vv0(!q49xTtraqvLPy%6@-J$WUn|JKUN>}Roro>w@MZ%Lv%n1e9oILcQeqA9R7xn4}DIGm{0;+IN!lIH5T=0Y&*iyiJidUWBS8?qpaiEO-{6tVwDrGV6pGuiGU8hTKEhtsGw*BgGUHg&CIJ1>D#4FwRebdGh z{w4}we7m?_aPQ3U@bwftIqYW%;e!ddjsBpl4_64SYJC9|m#NM<;lEd@h5mB^vw zN@i11G=Po1vpetq*2-P=T?rn6_mK~ntbP-N#qEP+P}NaPE8zvsc6%H4YI0+78Y$)` z*BqUV6^|U6tKsdGV!+#mj>4k-17-{OSdB$*3QpBX-IMDP5kPHkw~iowT=R7&JYGd> z)DL%%xHEIxN_0T@)Hx51v#0-Xnv@=g!#G9&=W3t@6X!%1>(`nqv_r=>y-HstRY+zl zk24d`G@cz7TeGGG_L-5^V=R8T*uLiXx}tjAf9l?nG^mqqb0bYEi#Pg@q_-0Vk&deH zG|_VIFOHXe?8q`~Vc!zslQr1Y5z)}Qk3a4v&p+y{7-V4wL9uFG&e_+i|78@0YANu| z2r{R;ofMz<>g_LA$~A^x&!s?eUE*--M5R@Vjf?HNp59m-oQ)h7vmD&Ml`<5kDMKhV zuK_qV3QKCwUF#(bVr8`3$bZxZ#GjvCa@nR9E>H;9E|N%+k`uDU0oFd_?X%;kJV8(U zzbRxpjzlF?;ZiH=Z1aW?=TQ)`;Oz4!43w2$cZh0bxR%5z?kM391-39;Ag9J=#mJ~1 zrCwsl{W`vqAu``nc#NobVqBv~3rgl?!}|Y{6bLG1wu%S}&iKfmhk(O^?NZu?cpM$EbvjgX&->6{U zA?pqw<`K?*F#>qUy|v3tl1T3mX@-ryy5PkhEaq&k%cX@O4XB14P5iS1k78r*V?7^A zfRH<%6SoP^D!ZZ8S`p%Q*RtNFQ($rd9h0=2qcWyn-(GM1P<*+0k-#$hTy86-Dpizq zBN{7uKm+&#n$XquD5C6*8*AvS6|6 z*p|jrJnX&YxvmYJKQTnBjqG%5A03qdx?HA*S4Q}wn`7HZP)PcQuQ~zyKxDz$2d_|Y zt1my*PXE^ooW~>zPG}RgHo6CeyniH<#;uMM;Z%x5=care@}3yvBpb%=cRJV51qWPk z8;bC?^2m5aV09zYtJNot+8V;5J7)ADjs;-73;pNBACTFW>^#C}5yWJh_YB`ioiP8# zGP80^ur z=k3rrt!;#i1b}qimwYwr)R7w+-SiovFNmeen1wl2g%!lc)hqU0H{;O@2@wLRB{vzE z-m4Y=dM+7pzEy|CzXM))S3Ms9AB+-%k-J_~sgOaAmL~oq4y-Wws9Rj~Ne^-TaJ~rn z$IQb%nt_;+nF7U-R{CT~FlTn!%eUwW$20=BUP`$2yOyZl32%)m?rJirtu)WI`)_ubS?cGu9f8c9@a(@37;WCD?* zu`68!f#+_e?S;QazmrzjN$+^_8mVO)_z4Xh8|q}khOx{l*yC!)z|!5v%P}$t1m8?N ziYp|o85<>G(fM|23rU-0P+7JU^tSL^6;)M(1l?!nW@m4Hrw(3-8wOn(E~L1DapYIH4|*N z8H+!uE#&X2*K|oQG?yDaRbU6n7D9jkLe!TnOTm{7R`zl#RoDPdv)jjlnh)6*#IUa8 z-SUrO)lY?S(J|jcaQG{BFZ7a#KStCrNn$iB-nOjRVo>Wfk=i~*NJ5-EED5s`!*^k4 z?e<9LQ(-$~y1H%wXZh_Oq^LeYuy;rrWg}I) z6ovj?39$?nXHNmmvg8QceKmdW0=WNQtsDIwoMzzJ>VeO%hqgDpp#pj5$o-B7b=1?< zcF%zVnjd^H(J3ArJ_)~KYK6v2#o`JM`2WC>>-W>fC1%rhXg8VE^8|^PSVp-mn6LL? zV2o}9Ot%DF4*h9CXi8)};(XkhP!N2TQTaaB@JW}v?dQSU3oh3|3D8vfn+?-X5hF>K zT}N1@)HU=q+v^yMtYTvCRG4!?4$%5ltxB}q)-0?pgy~hsc`VV52ibvV{A45>%e0V% zsR-i3PrgGFU+q>YL&P|2LSlaMgF8?CT~GwXz|rOfMJo9FnA_dUA_VFjUq&8KyWiE1 z&La1hU|BSWQK;MmbmWeD8QDOPn<#v2|qy zO0xT&Z`lnzFsgPt@7F8x@I6##uesL^!;PdSxhkdcI}bowU<`lp&0!fE0AQv8&v0Wk zTY~>fn)s#3=(bjdNAe&dUOlhUzUtn9zo)ZG-G?JQ&7K9O;#zT@rl?ERo17mc*y@co z(po+9X5JJ|Ho#|@$t)p}rhZg|EbM4Ust}$iQncx~EVj`RV4puc_pn6n>d~8e*B0aa zg7REu$O}1}wGN~V9qVg!WxC7;J?4m*VaeqRC9v}6f)pxu4vmmrqcQ|Ck<^r)|A3EL z?Fl^ccp^*pdRYl!%2pOwIe(zd#$vo5ZLrPa%R_78-S*a$iSCPoGYo_*M%pEVXF7-6 z29T8We^FxL1CA^qBpN(h!>h~+GWI{`(QF~jqW9FZ*phu3l=mcZE3_YDeo77kA%h5^ zq)^(D7~@(gb}Or$_kQLpYMgZ9Lz8COI!)55_Ex8aI>e3?3(8XP>8jzgkaV#h;v3*MD8&#BAO1KB4ZNq&($; zVyhF6N_9-rR}{l!C0gA%cwRSp1Vd?wTMp;gzLp@z%QI)<%a)*|%R2z+ULj|9v-w&e zn@d}aAp4V5&#Y@5^}`<#mBWG8{MIi`d>(s6hZ%6cuW&h>*09MYTzc`c_R^iN6nhFK z!`_J_D4YaTvLhK%sC5p8;tgrnZiMIp{=!U`Cym&2m->&J6APQT*?hWnFm&W#Lcy6c zu>8_!2@jM5PEV}0RBnJ~RoFa4e@}oJo($T)>MWnG%pK&B()pH)@$$+_`eS}T61tv$ z)!F`WwwmUX)6w+muNx^2QVz#%x3SzSyLOe4ix5z?m?VkmThdL=^BP9%#_R7Xm_J_r zQ#__=r8oP_ekr@PX5+QlGgRx=CkIVgqi2f+`#U(Es#nyo^P*&VLAJW*>l9LkVW0Sc z1`-tOZlD#=lJ<=X{U67Y(g~7vB&nVhMgp!`K~;bixK7z`9Do{^hJxZZf|zF_V^-QM zVI*5^ZbXQ+_if$=2;G-?Av411MXgyjE$Ozp`@M8~?K#;Et^+pO1jLSoMll-$JUR$ZGc0GBSOkEMZb|a z%Z9l$SaA|afds*1IQF*qGj}%qneb>@`<>x|NtpLCOk9281_zMQiYqJ7Z?#{2J0C6# zWF^XYH`w%wQR=L#+ErLDU^<*=Xdf5!e3S+H z5e(*@dDKq7OEG(Yb7gv$qir?fH%`G&2?h$OKlR#~Wy*MiZ1Q~?(b@`b2D}(ywxtMW z;#}(3Vr$1e$I`7kd%epwY|4Ar?1N?eUpGMC>nrr> z0Z~07HRHyHQ!77bxuQ1)b3S%lZR_nG$j5uR9XJLxICoixpBw+0VtOu;GBFn}x^6nM zXB<)=iIMa3WU$?w-&KmXC>;^5(SyTxzj%hP!h|KPv@URAuw7WCcG@6EX;8^F#5{iW zs0rw^oCW`PV%ADk^pS40bawr>F6 z$sw7`Ns~-GbOY}WL&&XSu-p}yC|aD?)!C)h_9fj3tD}f`(;5*!9PUDnoFR_&0)uzJ zX?U86&(RSEb`X1Y!8TvQUE@9ZD#s1>EE5x};XS^7)oNF>(K)8zbDw2_Q*YNOy4Tsc zY{1l40}6T8AsExyMEqcet6qL=+nhO#;_y?KXGTR1oNVA9Wmq#YHG8|g`1rXg4RZad zYXIcC!fADq?WSGKyU|yj0}%S57wv<#4lwOP&2Tnc_a-L+%@;}jaISTh-%BKsHB=<) zry?=K2G@D?5OJ;ACe}rP9Y4oF;6<@j#NO)k(gr~rcKsLx&8v+OS490Kk6CnWRz^N- z1bTBo#4!+ihopCP^E>>1fUDR8z`ZZ8S&!$lQn?&FW{%I#BQ+x1litoHg;*v2!o|(`U+(hs-)7gbQUhDN$ze_&FGXbUCsaK1gR$ z0!6%2)uEp8RLre{G?5wW+xbz^NFSd8Y$?qFZ*xNuHEgitFerVzLRZ}8?&~`3JkEbe z%1@A=p%e=o$04b^Z<_hgd(EN*KvTJd?5f6hR%xDH4s(xZs9~Bilq)=I2Mu45P~y&; zpG(Ehmp{V$ERc>ZYTbwST;Q+^o(4NB15AT89LmC@1cG6c3JBVM6~Xxyuecc@1!?s* zW2?k__3X(*z&p}G2h0-bW>lpQgSOa_gw0$ge)5q8^SxW<3OxjIf`vLsj4Gqnr}LhJ zDG`1_=VtdF%5;Ct-WK{VHM!tF#piy==)!=;SgB2zsg%6e@>WllBbFQ^0|XJCcB?g{ z&z7(#eUq2-E80{Vv)|%N1Y#WdWO4g}w^vNz5qxm*kX5QJH{v>NSxTt^kw`45s*Ccj z=WZ7rcc42zXanK_NESW6INLx;URQBidIyLU84Zvm3-2Wa52MU|^H`#=k3;|8QT4h( zQOZkO4D5faL9mY|AC&$(x*{jYAREW?dmG>Z(Q-a~Hl_%E=S==AF5FIZy#@xF& zWtsD#``Z^DGqOk4-iFcw0~{|F3WrftAY*%^L^6dM=+jbi`IKTItXf80ocaDoQv~N9 zM;nFBI@wuSQn%wVllx*}vu|Qz61h;f4?d!SVI7C>e=%>q%7A`Fw1OZwbo|p;N)QtL z@6EHpqw_v(f_FeeS|3Brb{^WknQnsrXV*zjwU8}R_8!#$N>YfUCg4ECJoyshXc-FM z22F8BTWbpv#N|^7LS=6X2HuSBCF&G6l5IZp9RU~4ENMz#*3+y2hFRe5wo3)T2W=Q& z6JD{{8?Zv_1=R~(820NyFr=z7I_IFS$wSncCpGsJwpM9cDINy3@Ph6!8_Y&ZG}W7z zcid@QuuIT8`sd9VostW)H5kFfUOTl8)=Y3&7AlkXpan3!PFoI`rXgzA1^0{8u^oZ7 ze^j)EcPPC6sxAilHxiM6rdXqmsxxPU4kw9A0zVHFaVql9Po>r~3D$?sGBkoo=T?H4 z@*fs7XF0Kola4FwKmS?lD`namM*;J=4}?4Y^dZ9zYSO#Zf(N4-u#JK791PKlo`hUJ z;g_mb@bAKPeA5_hRI^-Z8ZQ!>S2gWoZ38I%^JM0W=c;+d37^L!bg#&U+ZDBRm8q<5bP+XkW%6YDI$ zQ+IF|&IQH+ZWhe=)_6`q_C!R$-KN{cdzD#iTYbez^K0el|IN5QZwp@ix3IB7qSuW6 z!5%7xPGMNYQ3Q@3nRYo3(~S^7>w>hk@Z&pmpVqgcbspme9lOX<;@`QwT zCpi}ban&{ABY{c16e_xx$NbE7vGlc<-`WAZuA#%*x z4i4&@6s2Z;4U&9%&B4Mg!PUJowg?$!pvYr6G&ZaT71$UO!5MXdtXpFs?=ZNht^9o- zg<7k*V5#W5&J?i$wV991&|~(uxf;j0B$?$n>irnIuWNyB5fa5LdO&-xTcNggwp(k? zX6B(z)xyCTcRw8|nMhQvsHPb{sv4i=5zMDC~ z=jjNv($G1`=@Uk=L$E9_WDqd5>IcO9)_)1#{9BMJ8hs~s(?K9qWqHH&riGL|_h13) zawb^+&6_?G1FJ1xyX?dJ7u9BhR+-p8pmFJ%EyZS;C6n}_WCT*YW*>b_0HAmqyZ4~w zo@?}cN%dxMULj}3S$Ip|aa{SVU{*_ZgRtveT82e! zI$|IRBE;vqwV_}x#5*Ef-<`9t-@>g0STZ%-`ej;cMS zhy19;4{5OPu)~!1^&w{W{tb+^UqNS(eGahg(lu!7*2RDUAbA;EC;-IRit$N;^A}+PdPHktQPj4BfjmkA3-+*V4k@H=sF^Dm)A$^2 zqLS(K7~Fr`f+hy4t)aR#ON zSi-gIru9cWcmMo$z0AxL7J6;g%A~m@s?&M|5;@1v=jWx63R}=xA?{BzD z`48ZpfD%>*2zjh|{c5vymF)i*>=p*Q+49w4-D$86TzTL=Vk$1d=pKn8PbIp&uPv_h zt;UMU7}BP8LuG9Rp}=V6Bo-7Fg}#qpSA-hL@CYS2Hq@Hmi!*Y=MyC}jw#*~yF8;ml7`Rorb6fE7+EuUIO%V*R7 z8C;bWhj2KPBeJbE@1>lk`H)wa1A~jggE9s>RzEds1`9-dqYUWD&;oqP+^~Yco9~fn z;pV+KWc!(DC0=)}${EdJegi^6UbROeto;4GDV|uu$7b3Um_2k-#(__Vqw}DXCKKNA z*wB%B@g860F+Ts!WfFY{~V?+3iZ#;XH=d1EO9{P)}(BlQx5iA9# z2{5pe`!BLlqgo{b^ahoiX}_3@7HnjsC~;pU7}vNGbc)l`!;Ki_ZIlI&2Aa^(rOlg+ zZ84*@2hoq$bIGvLSZBb-dCM&4_IxMYvln}Ll6{3D{n~~T6VZ1`X|Aku@GcxzlUH#a zTbEwcdU@Ab3g5dT(G$lQy06>JhTkljz-w@(zt04jHFee(&lydZfHPXSwk68!HG5Ce zE~Np>MIPICc7B5=lg39Dw?$D(@%{JzCbC#wu=};$uVpzJ@VG}1VOsodj zrc6P`Qn!Bl#jsDbi;S?}&P(fyybiyMsQ2cUY}#vnixw5;X~rWhP;{ZQJ+j08af5sw z!qH2qeI*QZ@KdRR&E}YgAgd)4zztdKe;q~D%IVMCN;uF`#_|fac--6x9RRa953<)N z0~2GMV7f6BysR6q%AIjLgX`koGdT2q#9CV1DtZU-NoVjHGKhjjuk9q-41uz7`S{z^ zetALhfOfp^Jj#6o%0sAXV-bQT9Yb&7i_8`ke_xiVNVTY1u@M8ipAt!oReP|_EFT}v z%z3bvW=E!D$*mO4m4eqW{;J@R#JA{fQj`;^p(pw2?LL0_XVnAB1VzPk`Dl)Wf2cT1 zZHJiQv`G&R(KP5~XZNM#>bzgZVT3$BF*E{ABhOy45Rmo%y%YebtYDIJUi5Jm^hQKv|hfayRv zJgEfyL4B}*wF0i^YHgw{Q|wWSR5`<+ZmLe~FC*c|K%+kzdW!ipw-;T_^@=%UQLZU$ ziSpEV#W>1bhfk$2c$M3>aTU;!5N?WWSt}!6GTCcg$Pqg?^L)ssqRf|LRMdAk7c|$t zNeBk>@-#dml)d0S6^t@=LtWlwEWyzSjxN=Ts^Yuw|IQa^4 z?07{oa53RBflbE(VeD~yR%(I7ZqiwortddCAfvuGmD;d81RY!{5VkHmU3S9#>SW$z zr`n(|B04<~hL2yfz&+#r904?99Lh!y}f zK+3-wurJhVNE(S3KYLh8uQbo(5FxZ(8E$rCwKVo`u8!7Fe!ZR~j-I%*S!FI(&E@)? zMcJtr*KW%AY^p^tdEWjZZ_(4&K`S7EfjVb9|Go%*cT3gAi@DU8xJyB$m}%E7+1NiM6NPGge{GcI=Nq&)N3;+MxDs7$2qJ z^8ojExP%VBQ@1%DI!Qr(>9(D%|9RHHZy`jg*T?;Ry8^yASNyF^8C{x+U2HTLJ7W}} zUj~vC`8NwR$YS@WC3u9aqO6cQ*^jvftMxH+^8ytrvn48xxev$*y^es|hTIPgglF{1 zCZyi>ZiUHAf+D_2ld5sPti6?H$#K2@CN|QPCqyjl*`#uzobPo7wfH`{WZUUoT6j(e zy?d=EXULQ3ll%MTcS4gYOC$}$5O;{i_{3`3Uj#D@4xAu{RwC(8A&i^VivmoGP;V&PlKkV5^g8Y02g| zDL%{<%(*#W(4jcy+!O92!+k^l_Sg-IPDN2O+1u})6%TiRAcpqpSpQY5e_XZ$3^$O!u<*Ry@s|Ax^lYQ+5`VxwaPGnF+BSY>G8PiNszArMc%n}VDI$u+ zzRo>$IGa~xwyqm(j0~Go>8aQ7MkmyE2llJALg95rto~vW+^6m4bdRy-gxBiyB=%L zggW%1>A0XIPRGZBth&3vtg2xDXocY+2+g7H4+Pbn4^ho@mWXa+SOcwsc zK=qWK=Qra=|4{-Fj@4=xhbme^=qbO4O#M<4l6k8e^}rBW2^XbQU&~H-o=#3%Qz$<( z+<|6VcALfs;8bs!x#hRP5I|CT5&iY(@HIF8QOEtvS?-WVjWk-BY2tl~l~awr6C++` zT$a?#DqB;3#+#l6iA&zoYYBpbLpmLR^HXwfvB+wDZnP{&&Ii~`#%VTl9!wH$x@sj4 zx#5;G*`M;53xtg?BZd+!6se@fG9@_b4!XKSTsw>^Q|9S5$iOb6$uHvCrgQ`Nt*Di7VSAUV5+tP$)Ri-I^nYtAFwrTA& z+8!sJYe{Lrw^ zNTDV9`tbQB%_W$~GZY+UmLt=qr)_oW2tw&tedM}a2fi6{F(t2A;QJ5EF@HZ~e6N7N zpOh)XE%aa|hKnIT6`gedD6TqUqTzF7w!|>;{93_DzcNS}*P9bMVlc$bmwOCi1P)}F zMuPc}=lydBU9_~@;ct4A&b<@_sM*z+ikgAp1C~*B%WElzJ5`|In&jKyYhip|0%1;y zYL*uO{Ppf&h=@WxP5n|u0KR=qLqj5?p8Q-6!^Lk&6ilG5?*mQ-qin* z{PUOyB$MFkuIZCGg`VGS(OGRG1r!>_)A8*Z{Qqj5f4nN#YX%!0`@i{sq@l*~GmP0a ziX}<|ohe9=m~qL-&(2f8j(8{oI0)7^hNUd=-2Om>A!m_J{E_G|iBV8o#Dh@`^fCFN z7jg{Js9!B5qCV(Yl;6 z!D|?|tHeN^7;BN7PT-4;6Cow^X>8Y5LZB$#cj}iNhv3EJ5*M&7!%xN>tA zrKy&bMhwKH^mMhXj}JEVDMfD+S0j_gw@REUVJJs1H^o+54iHLkrBifEz3W8`S;B!H zu+g`aB6)so2Y_OXwY}E&pSDoOMg*Fs{3roLcTMB@a*26FqbE6h>3v*+ne1pcjb>Y= z3)S8z(Ke298OI>qA;G7TJ>zo+(`=F3GKuvR?gMRnnB1y2ZgN-W>WV3B?})7Ij_#=N zf_nVF+<+p$w7{gf4XkF0<6)9zsk#kxx|+cFJ&H{^LWJrWcB?dJOQTfh9;Hbp7(!@izAmpso!(3fb3g`M9>?-MBDzAtQSz|DZ)CXo00`Q z1W?=bSC2laUTVy{O6)%$WWTjMN`VlZPo_ks?802}X5@*X;+U}GL;)U_BU!y8K3%a_ zbSZM>sVD6{baEU}wZ0=y) zK~AcV;)K48N;RA;Q2KWC-^JZd!z5Kpr!A82`V_0~l6k0j)z^=c&MMvnrh#rO6e7{X zudJGaRd!leicyR3qSRAucFN5`coPXJ8xX+3El2$vmQ+x+t$d8-_HJzW2y9 z*M4_g(Y-a=s3u667&!u9(5c{BGXOD9y%l89YLjS)rFL+VIG3AY;|nR*dXv4Bt=0i| zL_Vpb#gwp~frT�CU`y8Op_GH@n&RKbNJjvD}1_FT4BQq4Y%qYIR>ey>WXhXCK<^ zD3y~dmf>C$-+6gejiXC*uf9gEkjwYCsL8hMf=FNTD5e<3jwo7rX~WA#e1M*c-gpeVsM8?Um

ck{ZfoB{SmQ*9TfUmt)9+ z5O4O88qGW2k31qGfO&+S3hTqK!7Rv4(N7zzDPsmD%@#(fMw&8w-{APT4 zSv<((+)(R!PnW7v->qMZXm1oN5NlK$y^`P?}D z)_b38rCTjLLhCrQN)Y6~6s@X1WRqPHQ5V2;YEjAeFxD-dWT8}_;lM%slFU%QtlP}iV4U_Eu~;iP z&*^eJ4^Raj6R?us9mj5=i80Lua#}{kG}^#BBCjVklSNo!-=)JGBFH{E5<-emHWb#T zdU$v)xn^@^fPfE0->+Si{$k*7>w<~L?V@_DbvvC)B!#utMHFTSB?woacNZmD@Mj@% z&A9^5;Ihlv-XV$;$^rGjCwG$@VV@5h0v`9R_`n&d##!<19F;ZYQ;+v^?SiClD`+8W zBrAx5SRJxtMo79-o!Us`O&t z+*fhQ51-=V%kIV=P7)$VG8z*o{c<`m*Q6gSZe?zTjkudaZII?};=&t(t^`?3#dd<$ ztVni;)28&*`f1bVP&>q)t;+xMe0KL;&Kll~d%aaY#?oy4MgFP1G23 z5xR5sp#gf)tuASY}ih|vFDJsxrW3T}dZ zpU!Mt0Md}-lqhpNvAsXd*e1r#&3z;dMVe@fizsjTZ7hQd!MmKafzrZK^13adgiK7j z7Hu?zQs+`RX{KgMdQnPYLz-NGICM{B5WZzBO4`kC?pfZB(Z^ub(SkonVnWH8kRzZ1 zX*p7G|78Xn++QMSpN>KN`%JxPc(Tp86sVKL;k_j{f2b(WW7gyH`cu!>cNZmw1?Q-)&TRU%I^!= zXyOybR33sV>t0g{c$%rbUll!RzB^>k&ke>BG)SPs?W=ww zdlV!yYoev>nkO0{5pO=Rgiz83T~kB@-?#HWc{M`zmGUm178Z16H`|3{R~KmxR1oHL zpiqZgcI%|8LIpD2L!z#eo-!~a;7G68AkixHt499q#K>k&Yf5}P)CCJGJI~N~~ zu=>~_~_5|GEI(lWKHPAv*vw|>ZyM9xeQwcbMLr9y(VE+$( zO&`!f)9+8PI#8#FsWs`523NT=!`(|AM0-HdWyYSWe;2Z;`H5on@fB96aPBAm`LYY| z(d|7}VNN^W(AmGmEZSQ{yk`eWcDmhXJHE|0XLSt^PKdvz=wBu-9AK0#rjHO?ETIL( z`)yNXzF>Ls;2pg_RLaz}+{AItaX-4SUFON$5lHQSRG02pst3`qt{W#5+_j_?JYJEn zoUXQ-J;l^(*a5#U(^<1wd6Jc^HQ%Uf@%#&t+W%BZ65r+xP&e_!C^h#`*np_gBTP<8 z0t7z0i2Cpq9ua{2sk0+L(kEMp4O6Y}&LYSu3AP=q5ZVwXno&0QN}WW-213Fhn$PF< zbmlOvU{t3`t1{OJ{*n$j^G4d(F0MLVhpY=(6RpyZ%Q1xDB_CoDT zxBp`smGNCZRU+sa>W-h0)ElPp-jPHZ9JPi;y+i{tDQu_94#!#>F++IVWOuax=%)K3 zHH^E=ksrbjmSO#w?!o|8M5gvpX^xgEE=Psbhxlrge%DTbbUuai{#}(8 zS;nbau&fxth9s_*gAc|Hp!Hmb{YGz_@m7vA3ZbPDxOvx}8&aWVvTJyB z6L`%Cuh5JvY~swEC&d8dW~I7*V%9~Uy9Cg%tAH9ppK8msWYS9xzlCRzGu9eUo}!bm zvouo^v5D6WqJO-ti8kYKHzLgJNg|&l)%WhiuvYT#cUBR(lM<2uUu8@M&@I}P@Rc~r zU5^FniOO86#0LC-c4R|9CU8O#An42LPdxvX%+rGZO9n!I~Hv1RYnP2%}ASyi?52GID;Y7Fjhcih8My!G=H%@zKF#lLQ_CJ8d)#!FWD{j7u9hrPHnVMdbczP;Qvrmg z%Xx7|u=qf*%)tPNeKxA%eD&tQ&51DqwmC}2H5ze;qI-F=e$>m=^4o|=*JFcmK7dp%IQ#WX^_A? zcvdNHg@PG@Wpso#bO)M>Oq2FkV+?n(m3Jt2GGu6wD71q&Su0JtEw9{JREfR(`Hb7xm zmljWs=&a>;5_rt#hb8W*_Aq zxdAd}sx!xj6bq?d=vF+iz~Pv|EE5_MR$0||A^}g9?(UU9*>y3hH?eSi@ka24|Jy}0 zV5w8dS!OD@FGv)%cwGZxW%_I?iK(K zP${gcfbqyU%m0KshmU&k?JA+@=(o`yV%h?}~y zES(cJZ5G@C>57jw(Xlf%a@*EN54Fp|?EaOVbrC zGo1tYf$y5I{}7S5H*xoq-8WlV_8Q&Hh+99lIC^=3CpHy(zRp}1k)?~!_&XyXwhPE! zy}Vevq*WhY$qcw)%ggdrJLlLbrZFxj9O(g@tOU4`5oj4QQaK|d9ovL3hyWvCh303r zWRJ5&Dsd`ys3Sr&wFP7Rb5_?|l?8b`_JQ-VGvSgz?GUmZyvjRzxbCBB%GuNR^Y0CI zh!ZAkAU($LdyB{}D^6eOD1J|#atw`OlvNJoGx$`DUYr38)MA+}VZHW9_)5)sp<)$s z+5T1BC+UjYtVq6Ia)+r37y=aB?}`6YZXrNJO~?OIm|=w9&VlP(i0uG5)fN=crL@ta zI~&fvyT1cSejndRvD=(jg#cND70*QSI;v49v4hwIX#D%zncvjaC1dO9PMzUJh7%{`3z#qyUCA!Wi#|GGDZJSttuG|#g9S>;hOH_1%oC5$W!1TE-aGqb_I5N{QJF7G2bOqY>y5cj}>zfm!KKTv2F)$%UT6G7T-r zEB(w6;stRQ%bRa$;4k?ahfr%92E1o4TZBt)?ziP2?}Ig_$6+W-I~xRR#%qT^(O_h^^Y`!*HUOLOO2Jl z#@ke-oLV6$gcyKE)b?Ft5)}S4-R9Q#lCSkmp_1387=1=?TY72y!W?k)8{@E0gn=1i zxVh?q>9Y)USuZynSWdO3t&Sv;!yw|!opWqEL4>$5G5D2wlOVO)=UKD*h{mKIFciCn z_Tf^+s-VF#1rL}XB;z)`g}Zw#Mcx$F-wDrvLBY%KLLBA;u_~-BEk^3#X)9M{lR>+= z2)*gy(GpFt#XTlDfb_>jf)G)XGa<#KzT>}m-|D2$H)4(CX)xZJ!yEFla(2f=%6TmW zF}u=r{Ug+6!qrmC;ss(cu}g@wA|kpA@t{cg!6H-BQ?+r5sOJ~*>?^A`S{ND|DUuxPsaZo4 z#8_M3Y&RtGZpUEQ;E_#%?Cn22O6qFGuiAZwgn9A0nC?^*^_mC`j(G#Pu+lDxV>z(F zb!yu)w{2(g$voaxnD(>8QUSjOm!yzSbEp&bp+x&=O0rqc_)4f+U3e}%(Wu1&RswK? z8A-)9qQ+vq2lMFPaP!BM1w-DX-CO(S-R>_eKTXsOofavot$o3${xG7ASaP{`hn8Gc z5K-?qt*-IGJP`~3btHug>uDn$)mM;txGLj|y6L*1aO>(CivUi|3GImcU54B0UgM?CK0jYOGhE zm|!^)t7$?H;zuTl^(sJ9I8;8si}_(LP?IpVvto-jfRp2wu69O^7!2O{ znv26!_3$V24{JJ^+ zmWoaD_B+)toI~6mU(J=6p2XCg`pUE=`7YqA@K3}RbpetKm2s`Y2=nn@XX8N|@ET%F zS_{Z&K)B^q^?*&FE=YP~JLJGHf&zZhx!i6w+qnkV&nNbi#>8FW=nid~DatL`)E z@kEK$#6S-@RwqaCzjBot{A`Vh`4FS~KscCHc>pygXT(~Kv#ijhJYTGvI|rn!z4S2~ zT7l|rQO$cF5=j*nY0>R)P6d0&yD+K6F&BLUmDn5= zplaLlcwC{Bt0qOcT8kjtUB0gG(&!MSpbPMqnbWq5v7(ql>q|Xivrr8X`}yg$(?`e9 zfbInJ{_o)3ShBb~bM{{G7MjS>WUwp%e_UcTwbHX6dQXOg5(_;7Wo<7nd2&RlJQl({ z{Il$M5#WO2L(AR7WRezg}2U8p7z4W9q+>DpM5o zQ2%ocP`5m>x_NdSgQt_G?O2pH;q%QjfC(|_g_9GX&@MJBI^09Sp8M%L$JZxBR*zy` zkn{Osr&>84f7G4%vl6P@^vgPnEi@=CL8FcIpm}QI;?NnEGa?r+dyN*2Zd>5KN0=~v zgrv{2_SQy`8TfnSA=oLearU52-|f}>gFiBQ@ox3gm2q~7phEf;fPqh`u2$hmDE>~B zM+DDg83%9x^U^$qrl|Z0Tl^MeV*qYXUu|DRu7CGiItKBtqDNui9 znV#v!bPz3;*xUjaNL}3ZgBrCSf>TdWJ}22XM5CNJ_p%gE;wnFOHO|l1=eF-ln||uJ zJR(i)`2>>6i|sCcq?lOjaOPZyS%?MNWVS`Oc^AIaKXg6otY>nI=*8Lx7NpiUS-U0n zyWgoL>-!-J9Xm#P*v%#&Tj~?y{r%gb_TPlmnttDH$%KUcos&22@7X-7aUb`e&mH3YIQUpevQRaC#MW_H+gZ#k5_V!TuCVD^Mgu73a zj|7s86`0llBnV^K?BgU)yu9>%85M_|j}nX~1OTOfbCqDkrZ)gkdO9Al0kAdqqniB+ z^n-5@MPj~)%4K55b9OKs!k#xW-5s z5~i+=81o&(uC&qu;L(LmDW2RyDd5&(;~2Fxiu#iwpSnhAx)hT|zS;s)y&8-v*3SDZ zp^Etchi~GX0ctd7K+(qp$2R>X;Ys{ck0P{!RGSt**&Q@YHty3bydpRZ?k36kd;=GW z^A$NGi(3T{P1bJy1P`O_m+B9sfI7P*yu5a$Y& z2)_f?QQRDxm`T#gb17~t8jUcstatfNQ_xV(}=fD(*D zCm%sm7ATaunuW%6=?imtJ_;8%G#Y-3zM)_XUOo(4i?q}(r(WP;6*z%n=4mG4099s6 zit>}}DAqZ+^ z8_K?{H_jv9vRW7J(=!bUF1J9XBZL$fi@eMmGv(;EyAXUB#jSC~Zt&+D!mmwBdga_C zF~PN^ov??J43naoMb~e%hfzL7Wg8BOUoTQX`LLq`gckjtGHj`4VYe4Qq4l@Ah9DQs zHw+Wq6y5ZDlv@@_U5kL*4VG-|7Bl!IG@7Y`N009{q6xhsB)_=#+(T>{z;)A8y@1U5 zxr0mZnu*?SS~Lpke7je}y&R#oBha%;Jr!q!td!fzweS8aOO%M$;gF)lS*blekR~V3 zT3-sZ5jXh*^6d5Ta#nBs#b9qg97~&1zT4;W(wjxo>&U(G3dsSNJa9NCv)y8*@f1QW zPYibuqcun&lOyaxm(-)UR*u-UzgKxtDIPV% zHRzu)Pq|I;Qso3Q&zg$eA@wJ7w144ZeqNqJ4W1pHgC8^iVjL3@DVF75spO?@q?u(a zw(7&c)Kgk7lKRs%j8Vu@nq~EJXHBQ8d2a+^l_VTg`<`6{n9Dg&l7>zpj~io(du*>Y z3WvO;hK87zJ=u9I#HUds@18&kFw&{<`~zi-0wf~IFV7dX%Vpgn9DRy__KsD1Q#Yqs zU5cWZzm$oeazV)axc;S$9ghhE)-~%AxZiYp<1Gf9gT+jrd3*sD^%$XDlYhOmNJWGm z0vblHBG*akGs-smgePfU@#yAh-seW&HgTM!+B)HlQ)BU#50)kMIohe!h}?&5R^(W- zedXJ^dgSuyI(J)=_ts)4K1iLeYgE|8_TdN~A)|mq`c{Q^2sGL%x7tECG;MNBN8Bh3 zURp(dpBZ@G830YqVRGclz-3Y<|JUZ4)QN_6tr6G>VIZ?UX-P7C@PWlbVRo<}QBt%B z@lB4EJ=*%dufZ2vOD{^6lQaftS1TwjA(`3Oz#Er*b}Oz{RM5VxD2qBz_xc1IKv^Bg zBC>?IlC<~QAak=A^zdgNosy0w7My~UEqUQW%nUyL*geJJtq8*TqdxE$R{Stj-rL2J zh+&WdByQbW_K&np<7_M;#Ov?zU+>=mJRx~0`*2$H1O8ph~<`cvaF#|hN164@>*r7R1Q`yWr$iBC+ z@|k=dC5W%WgaS{+fR*j-QNt7iJ&4uwS$`o_`o@Wzb)(j6P zk23O6;rz#a4Z!zGS3*1}$ojBGA*N-Co@S{ z4Be8@6H7~427E`&77GVzv!Rv zE6QST_MWQOjrgV$M+OY;@ke6DjqQ)>IL^S7@rq3(0p1ebQTIOmUSyHRX2G}o3EjvM zH0|6Mt12O{x7l{n;c0E?{ecZzis6@n7#zO=TW&udE4GXm;z77V!`aesB2eg}=Mw2S zTI)sN|997;QNQugTq3L{hDMH1x(&_F+olmIc#V+dMTHxUbnEN_%;;6UkYnPE>^Nn~ zMNLQC{~ZnNkyTPdK72rM8{bTA~qIdu@K(miDLdaFrA&s|?B(_E^T6_&ym5hQc1 z2Vgb?|A&L>&E0V`D`j*!%wekqj^MF#+XVl#*tyq7Q`J&fVaMvZVSGuMTc@@eJ!OJt zm1_^DRPZ++Z-|*u{LsoE^e>aR5eYyVNhYp+M%~Ua&9>qe>=Tyv@Ax~~*d7XZk4KvO z6aG37!A$n)f>lh_Q4U{AKVT-2xNLs|4AwlopTc@OfCzNic*ksaGZk9o5xwZ!j-INK`5y~63TkgZR}41Gi2uO?*of&RAwGGM?%smPt#<9RaIem zppFQ;zrXx5k3i+5FonbH41nrW+z2TYGpi~=(@>K^HkjHfYl)AsDm=s_=BamI%b5zvwg_;^XpiGV#y=j8g z(Z@m@c^DM#6OCJz6nOPPTZhnG^yx%(=3NMlR=vkk-(K!6HmEt~g|Nt=c>a&1@-+vi z=meCx(hMS|5X@OJV8f2r}K<&sDb z8jQx|AZjaS{Zwy0MDjSi%j7DNu_tfOL`XMrsiZ`&9Cc@mf=#9lZvj(-=r&x_IEKt6 zXKcCc5h_-$46#7Sl&bQktdKhO#7)e$yxzut>N;@@eLqJ_-7#skqx}X#4Bc;xVbsx0}>?kj1k*Lg!f7y!~>lOA8 zqJ-@xf* z++myon|RUE`5V-2`U1$)2bd z4S?+Fm(ttM@b#f1o}BzM0~LGzd763uennP3*3yrYl|+oFq&{VFP}>F8(&I+eWG)R) zRR6&T!4l2f^b`AsQZ%*&ad6AinXO|cQa_=(4#mecUWPhmy*B@9i^u{g9OE(_z);zL zu%c!1Grh4dt0P~9O3f-U1~9w_pT7iO3K2jaj$h1`HTmRL?yx`FrMZhE7j; zR{gXZHFn0TxV2@ntfP@yqa0`V8pzQwUtKAkIHlv|>i!FcZ46bepb>YJ>=yPVS2;{Q zOo!IDm^0Oq7ZxPpit1_13YrmryIR9$+uk3zYV-P1kW0Tp`)@EKDr$*oVS&P}SX1Gz zt#|Vk471BYEQFms9B`$?0pG=i8xn!NT#71hC$z zx5fQKdhKfO`23-fNqy=77J3FQIN(PgCgsz3R8EJ7bVf$2bXM>b5DgkbBkg9FL3ukI z`n>g}^R4z!r{8e-`}!=*)(I`+v>Vn6Xk?Y04!PYep`UW-r$bJ|*crrP=PS1;Px`Bj zlTH*7R4#1Q%>SE#1&`|)cg1?ap*i`Fja!|1r0u!EJm{1AvUB4vSIA)i#6o;;a==Ej z&518@^9sN%x?yd>=(@}KRlCy;cLCe?$fDEGqb9YmE8&}@&iyju&H_GETMeg}PU72+KB&qe&BlZb+jT2{Z4$DY|0DjZuPVtxio^ zB@e{Knvk+3BwB?~p#qIHBpN6tR7%+bEJ5>iERBDxZTrx6t0xoFQd22sOJtzWLEa^Q}K(##z#@I$DBntxubsHvpx&Ka-+i|1YLA{DFg;Mf3t-o|M z4GaQ>ZsBR^>1BCWvDv$e*gbBZX1)>8d-PIB0SG3^*FmoKIq?w396>O_>|VYx>`iy_ zAeoKkBdx1QH81{pt6h#Xql< ze4kj4kiA+2<(Kg`qIF7!w=892)<%o$s2IJ!G7v`Pulf5Iu_asPKNDN@J|BvpS4J_+gZ@=x7?uy6=*HgVt{9gJw!pnM~=Tbitv5F2&#VGL{jHlc$w-=GtqT2$FhX? zHWE2t{a%RI8s{ZSK*o_T{J{!p2|Nzd6#aiM3JNuskZWcwHW$^U7b5Sg-o+f3x!;+S z`Z_z`Yxop&Zjis;RNJ!3p$HbZP8vaPrPtAVI$f?m{+S240-5=D^LrBT*v8I&M)K_< zd)GQ6`bO|e={WG*)aTa9spufa*8KzdJaEd_%sga;`<47Pmqbazhp9=xUkSHeBMuH2 zxwt#+D+DFCsW(gZoQ`umHV^iof8(|8)2)V2suTBwXN96xujp{r+8V)F zqS01iA?|63D>n50tv(}G-y02d35_u)i;I{gs<;`Q|~G0Zv*cs0``Zz679J6 znkNOI!8&1huPw&}ho>gmbM+q0N3&?ZP9nD)wY^P+-R>Y9aS>02Xc=q4?sY%@6vF7b z{6!0$9v}S5QWI`l4vV46+{YCo8dCAD`UEw~E6jwzvuzGp^EjK^=!V2JITgSd(&SnC zRgn(MGl{@rE=FBpbbU5|wI2H01`yzm&wU3;j;A4l?goH7`?o0ycqqvJgZ>6kSx}^T zyh?fj%zD8;y&p}M-Mknk7fB6<99h*1VwEcWaS7eF&$O1U9qNWKUG27t~tp5tDqXb5a=@B5opk#)Tx9!)OzhuLtm%O^MRte#r)3|S`Qo4urdvd_5s4h=O0Z^=At`PK=5qyHhGKC`T7ju z1oQO$2FbiJ%PJJliC~fY-OA{cJKD25rxb z4Qm}REO2U%?s$)kq<}u;V`s|lF51x1|Kns zBcVE&bfT6N3lF$>ERs{F0I6ZJCA)Q%#q#Qh1uA!l5VLcxTdu(OTO^RJ^0dsow2q#F zcrpW?wBk;dkl5^I|2qkR2?-q@T!>eUq@q3=CBM;mY;|)gVyeQ`~6L^E(_)7YOt}|0{Aq>CF+WR5J{XD!rdNd`jCw? zDA!M>N4?(Ziz$gsxnUlN?*#%ZQ6ND+#H)uC`m1~yU%;Tr^x3bY-l4P99FgNcE?FeM zdZHf6lh{+nhAesBGP1ZN43HW;E6PVZy6#zgQQ18p|8kR~p&$FCz=7{WZRdbdCjgT8 zOp>3efG$mZHnR^p)XBWKf1&wPl(-<%;}JlrgUI&I_qLNqt3mf zXE;OL^t?PDr*pTuVZZ2;FtasE6c}q{kMdzGWfIa-0^@?E0#!<%2v*}3PS)kR6`+?& z3a`pXIRYP-7u{6FtjFMf!OQC6gJY4D(MCW_H##+e+VC{PM(m*(+%+_!1t@wblD@%+ zJJRtMv6(`He{JT37gjX-p}LMFe6MSeZQD0j-xoDBLX;E;r6dZQR>BAU%(MlOdxyxd-np9W&1u!!JF^-#uohq4b@q_7U3PWQJk zEmn>l%!YUfEwhk!N*?r)s=mrmF7x&n?xLQ62@a9;S?9$>ofmK zN3Lqjr?nlmFGQ#>sG9>Ls0>c;EQw0Lse*k((mg`GtXf^rig(YasjcmOh89Z@o%As% z21r*+jkmj>eEk{@zq7v=7Yfc_0OZh)`X^BJh?hebg$G3GQHmRfa+8}zv+^Khn$7G( zTmD#;L9qnnbA2~OhWqz-YJm9;#|Xp05jsjq&#-Pd5X(WDJAu);@;A6qpQ0ZRC9!`H zsr}OTa}q-vs}J~H{By0{=MWbme>xck8}eviobk7mp&FmVH}EHITQ?$(uZ&qo%Y`PH z?;YG^BaA3RLc=_i(v8>HwK|LF4R^xbJNRk9J@wx_t=h$uhh4c$%{T3DnL76IA*Rt&buFe|Q^aW{knEaUqA z%y!HiDht1MyW6r$Lm!u5)=yF&BR;$`i@PEscK-+=t?*ibY{oDGR|^Fxyqc0mmSuP0 zH9dZSCR-v?n-meW4Ckg}S;?3FAt`%p#63grsK>ZIFP zgr%Ku?GK7`qh2>(gW7N%Th16XJ(t@nk0r;)WztUczozShwP`miJ_JoT;VJi*TfV2< zle3`RDi6A&k4EqvwBZFRg0eQ6J0+}h{7(U*1~#uRbQKE7ANbeLr0#F58?LfSjwcJ2 z{6y}=;On|7Q!(g{%2g!Hcg)zTqKw&9TNQYGE0>W2${>$@EWHVwyafS$9UUP1viAAC z;nICJ4`|+WwqI9mLP8#v6x)WY#TP5MCIas=`R~LN2D-*`Nj`E>bI4iPws(-YHSVW4 zwb>WyP29OS?6iYbzen@i#WD*Vs67sU4G;-~4mtG6fhv(wxlxguM;v7XI-L>Tza^N-1H`CZ^uBgmY~5+RUq_E7>i6`>m1@y{tyYTr2`b{ z|De083wp?li-Lei?G~c^@l7d5#MjW!TXzDp;-8Lw5Oo?L@iGjzE&zGoh87gy!3T;` zt?%G9!kzjQC3HVyaJ}rt?b{2PkgY+}4PysaxEiBtA~neyg1(4WP+uKBboYYmFqAC+|eE418?pJz)@Y zJLCrx=JbpfFa#YnYa0aVjWVrkNPifRRfX=P(pzeqq~G49ShbE`BQys2I`jr(eieQS z3*~^zg?7>cCWr?&)baO<#Y)y~hJsTMBQ|peOnfNi`R0$*J=`09oyrw7eYl|s_+GY^ zUo}-4Ht}`xeMUeZY{JYu+b76zT3kobm_uix6`{zA@BGsOA(q5Ur98oa#`4S5$nyOl zCN#c?=WUCTOxx;A%|tVJrXar}P@nkjn;{&>36DTrEHu&) z-Q%n3L6*|!glHj?;(l}F!oX0ftd5^!&4;GmWnjvogz4!QBlEgXUgdTaQw1LF!6&ys_LWf(0wp zh+-=E&jbukOf;;>Q8Q5!k83N~O)=_UzNT+tx^oZ{!{zNX;J`h^z-gDhX`EJ_^P5vs z#gFEPq|uV6t}SqyNwH$8U_$*1P6=v@oFkg_JdVYZcjaqsMKv5{)m|wBMN6@^?2Vvw zc9&r(x60yXrL8qQ<>>YY@OpY<`_#_&>PJ`iD!jZHB=t6jKtOSwn2~O1Z7tFF>}Gaw zrFWExXa^V3{-$m`M}HN?2PhR(Ooy2u%|QCt#D-dy7c7FLlpY`zUhZ+mU2W^E{D+L4 zNXVwNg*%R0?N>=015GR_wHhSV3BBKnK7Fz=wXwl9t$HpeBq(4cd084wqhywW?^JKq z2TcePf$WV*Y=acyn+<3zA&)qF{7dk^YqdD=VTG2fFx2?{&4VO8VlTkocR`+5ZXY|F zExB?gLl|>yNZ6zkY7<`lN<|IEvUB-@AosC{iV36Mkt{^5Z&mUZvY& zv}NIR4@ERpp2&CWwV9woX#Vk zDEIytqhRjeW(tM0$M_w2uG%>m9XTM=XVo zb3AxfTYmP1>6^gSlNf~&lZW~8k^tEU#(^N_fD}>mn^s4Y_mdHm`QMQ_AZK;lrighI zos$5%QoV%EIyRw2x*4~aRQGZ}`~1z7lOEDxV(F0N_&j|RVw9qu%1z3qRw)~RyG!1^ z~oP493i|_+0dsM7JKOT->fX z2j!dD)tCqH08Bu$zlN~GEQv=^gO%{lovX0ux}MC-Q2OcQE6FiUdfp(AaFbr&*5>4z z0^#1bM0dDcV?w?)tHu+|%cHl?;!YnXBD_q_oAD>_KClY0luP#Ozm>&0F?D+i)A^66 zJP8~}JX!QqByOn+eDmWswdKEvNdHHB#bGm~;2yjd1$L=Vs_AXR|4VZKaJU0nNO~!I zv@Rheu`lIi3hCH4fxNVNEE*}+Z7l`e8sITS64eW)7%AMMJruJolA+3bFxIl>Zy#a! zo@_Uu)u=oRfQ0;WJ&&~O0HZn>%uG;uTcDO9$68g}KQ(z>9d50PE(kY-Y{>3w?f$;R z5TRVKXt7o5R0}FluFj?5CQxSTWa_}~EfozUe1f&k*c*`hSBvuyOBY1WcOp-vb!F?5eg1U3M-GQxw-Ila!BqPC>o+zynq>OM)Hw@--*Kn`HHb zZ__bur;eqzuKjfl;q?DL)WrWmCq z`7T1&fW(a6e#?13)S_|dz~s4^A=Y`|>WA$uN&5*(Q%G1?m&78Ait|Rl%m@{ka0`1N zgyBi~HeYgv?SAohOf(I1m59*%k`oBRQ4KH}>qF!%*FJXva_yxqwrgOF<2_0CE&}&{ zC@XXyMa%TfGgeY=WRTYDY$VORj6-50p@9RIdUfEn(b~9r4NQvi_vMpKpso{NwAIbS z?1lDIs-bjmg|JE!JYFFq#{-)cz$`!GHz_k`t)+7DUw zlUoCTFs+=E3CAxuIY|{WXlWV1y4G52!UR zLE-GFq-kHjh;_E%enyRW1GedQoXE##L^fY_#F%nZF$+OAu|6PB!Wym!$2}E#^E6$J z`eyi@4#1DYtuz%a9BHk8G-lhrb%(s2{YWA2sBMJE*>wbACB&TSg{=GtO_3E)|By6k zzF3He-`-230Sht*WTO3319?eLeSR?n6fKSQi>1GiEw^X~L9?Ac;@yQEO-sm_I2_|l zzGT_#SqE{&aGvkL+%fg-;P~<%^+tF6spXf&+56%Z){LPvIc@^E&n#+2nNWJO$BnAo zZFN{|rZQP7km}BK#&R197r*qFC6Uo!gQp28H+x2fo9opAmZ7>0N(jJ^x3i>y`Db5& zy;8BP!N0O`qZ6cKCh}%xgauk)}jl*M7-5tmO*r1#%$D%75JN>i^=U_hJD6pz=^F_BY2I(6TFo2^_%^0;CXM^g?Mt#+ z;J%XHf3wec*$-YVHiO31IMtbJC+tT%eFG;tafWgO#k>F1HCrijjidwZfW;~a8>h5! zQ%AFWTZnKOBMg~Xk$;A8rK!|3k;%d9XrS(JTj>3yTZ3q->6dfjY`l6RAK^IKM-Ar# z)jgwZXf^$M6Z}-Cg|+VOupnsB3-3gRIR@|QRl)g92wqSOdXUUZXiiRTrD_1&iWo^P zbiBvp$x#M)Xq`!HuT*aIekg;W1#RR8d8lI_7-9&#e(EJT@Bf>usp+{Pk6b@^!jM`H zrxxr+{I$V`d2;Z+)*`-w`_{N``z(;GEgGwSHbN69gM7(5LlLygnvU@)X%Y=6FdsE#H z)%EpN2RVPpDfv(VV+?%3a0_LuZ_ip#FWJW#I4v>BIj5B$Tve$BMPWFEe5GIZ+~q!L z2TDX{mQzVQ8fX_+OitAq+AZNlv%8%Fy)*E)6qSTOFMrxt4T{^WKm(f6-23cSJ^t#3 z4L#2jGDcMzaO8c+Cwjc55Z1khlmfvi5i;(KFB*qT|Mw9Xp z-w>b)8stPgz`ia;Ha_*Xv8NF3c`qDAEe+D|fJo#2V5fym-LWoF#21KzwKW9G9|Dq>Jx&xInhEM1JdV6J>sZ&zV)YOb&qt&M9rLv{*4f1J26H?=J* zc(hxPuUn`wX}8%K7~Cy28*V6R`xwC8scOicAefhJSJVEc=rHM`lp=2%Xe2IijyJsC zZrW8fjlhUi5X!k{nr)#=$D=Coi%}P$xw9{bv;A@r(q1HuU>K`rzFJi>8F5F_{#P7s z=}0R(w1q0J@jgE~O_{XnXA)0q0#mVe2*(#|Mr=QxqUR$>W7Jc|Mt@DW%6XWM+_SC8 zZ#yf3GqmF^6TCUA0$}2H~nAxaa@BM-GUvOb1$JPmuW4x zvTkHB+L!5_ve?L)N0F56A_HLH(AyU6^j7BtKw{_<|8-;%#EJ$cSN)75ST;XZY~qcO z%y&aW+cb}``{Q2M8A(sA4L&`Xp-^cY>f=3xP)0Ma2B#H;3EI8e;Z3z5D9G7pX%DPr z`|YNs6 zc%F=M!l5vzaYFKip~0RBG^m*Uc3c~F-l?#&Q30GZu&G|SLsxu{e%56+kT3SJkSV+mZ~hqrB*yeYTZ%(~v{Lv-d!R$OprquigSq{m6SL#l@ORnJKc z&YcNp6J9MS2Cmxl0-Gc69%uab9q?%oW&}>KNPVuYEuEApY*c*pmQD8f-F!VJB0!NpYiT0QSF7^0=h zb}*CipZV}1pDojC3}tcGGsm*W5XlLYKwb$MVdUt`&tCKW7CWRnT-V=>riv?lr;#qd zavU_D>WMht5ql!W4wxjAS-fVjMb$KxiC5J`N#4Qh!x`Gpvw1_7BNskHllTmZkR?)q z6;%NR$_pCRSZ2Ga;@_O$RK$X0i=j?qNvQG7mSJ~yX)3^`da8ZP4wA>W7E%sD{mv;x z6E5M&DeE1cxEBhnim8cgBDV?(<|U?cy=C&NU!l2+YJ>&Tp}oxXCT??krVP!PC8}aM zpoulz+-#oTKT^B#csE@yP0ikHh!WSv5>Tw;@9?7O;|L%03bVQDO@koTtg*fT0ANR*KMU#$#xj z3nT;V>~R2DY$K&v%3htP9LrM~a+V=K$iK{zkz1>HkdVa_DB!kpye{hBO_~mO76MHa zsc>!o`ak1V>PqEEVz8rzM#F8M5Z95M;N0o!hRwgEV$;yrgUReJ(w9$jLbD2>A~ntv zXl@0ku%}nLeByp%!7YIfwN5!|{L~Y~-^$K0U8PU$-TIfa;$yAF1JV}sgfZ(`0uFqF z#cBv20jjwS8qX1K`F&9~Ym%1{PHR~eL`Y)VLB&!{)jqo@uXzjYOZ`*qZe)`C3H`?*GBsueHyWaht*ra2*W@U3H2rQtqpO zioGr%tA(lugQB?{UBMV?=6)YWS{Jz6fw;|^d&UwWx3fYh7{re1UEF8rBz+vdG@}x^ zTN`GV3;Q&*@3p(@`SspsvR^t6cCP{Lpz}Wo?3@P~^=EJ8q+-T}!$YUD$ zWJF?S-$QVo6;f3UwJ8p-hYREwEnhcqwDt#EE)K5shculoEkCJm0-<@x8RI9M$Ns^Q-()(C$paWN6ux< zloPec=)gj*Dt$h0-;CGR_l6RZOUATHIaxr+3~w5cK($I;mphGU`81|v{Ub=S`(sID zV*yVqMh&|>7?x}pFIhKFMUc#?2xfVHUJ01nCQQH}XTH)!UPjF7Km0EhWFdjTioxW<@Cn^3-3J+`HuE_pM5Awd&xj^YIu^t80CkcP)85jI*ipMcyY;Xd z>t&(@HF!)dWW#_|gl(}1Kd-)@E-5d^Eu%Amxioe`jw5gqc_#RZ`zy#TI2iEOc`++r z=K?;i)@lo=GCuWgT=@jDHk(KpFaFp9!NO!_M=>yrM<~9-^Njcma+|HVV4wC9$>7kq zrxtX^A85gtZvIY0YmTd2U{kEcw!P`>EdvL4$L~45!ybyFY`1ek&e^rkYFI2st{3o3 zc^~2pIp!IJ;8&(2?#Vz;=yJ=bRA0%5Zu~EKsaEE)i)>oyH~jAg2CMjkncg6V3?q0j zMR4S+62ijZ$v;emp}KzhG*zgKh5nz|1Xj^LLVK(Baonz7YwR{z_Iw6H<^`nMZOcgd zt+9W)e4U1_*O_Vq*!hPL?4U_|LRU}_)9w&`SO?lcM~kl#|Dqcq6}H^zYGhdI-qzdq zL+<3iO^Dm(`gJ&wN6gY7{)d#MXI*2m8- zgAWo6Cue zsJ$l2hS@kMOFKK7VW~gyZ;{DHtWoT)3m z7_L9()}5 zd`m3BwX(64E|l0Iy4mgWA`BYc`_hb~zj|sjrYx6yeD*J_el-I!xerttCE3XPqyoCi zq>*xjf$|I{3}nt3q%~v!a@232#?|Jo@}9_#<8OiJ{?=3Mffu!bqiLP=YUlboJb4c# zsv#k@AgCKA{OhzXC(K;lIC5dO?F25_X=Z z9gA?ojjL-gb_X^zCtH`}sQUc9Y;0Yp7Tk8gC&sZF{+SdauaMDmaY98*U*P~MzR4=g z0t4j6?e;2+*fE5B#fZD3tFM0&C9fzk6mPpdo)VkCvm>!(NSi1N;?)ue_}4EpM8lSJ z3&d5P2~QRj)5ZmDY2J=Gw1xSG-9Sq-!~Tg zyT#*UD=5KSOWf!hH~leBk2BKN^*dcaT-Bz`f8b{Q zq(_%PV}k*9cq;PIzcpmJDhFt7eaylxLA6xB+4V0CK%;9uso0i2ib)9jFg$3JSiiyA zu*supF~eLyvn_xUsAnh8_jEeFWlq`yTQspBFQCnSHkIDMyS{E4+sf|84gwpnBaSfN zz257Cfiwyu;Dl@3HaV#OrSp3g>k(EoteRYWC#%pV{_1X@K11WyiP?+TYvD&B_gSCE zQ4OjnzWGQapKyJOI#V~dK^4Ft$jgfw`tue=ER&8bl-(_^BEnhaxDN?a50bE{ zI3ADr=lWd{ASS1soGTVHh=dVJ-4J&hh+RISeYVvr=(G+ZqPb!mZ@H9OC<*7&^ z_uv#Fiel{5-#*T8c^pdj8Jto4YZ7nwRXI7Q;OSJh=ELW+`9+WmJIOU-fN1m9tLG9@ z7x!CTcI_jY|CxpE<^8B5rz)2@wKbwk$OB zfs%f##xlfNQHpYeisBLaB0vyeV`2@F0xHg2gj5!ViO%#f^5>bJY2CpXh}TrMfrPM> zNCNBz-%%nPqQkl{o&(73VB{WYn^RBF@010Sy?KnI;$Ddw1tRGxTglg@cH&9UG&bv2 zB(RqC@xPe`HrpJ?Dsi#&(YBZa1KUzztX%gAd-CZ;2GMr~%#CyK0E@Pzy`Rz*Vk{_i|}Wf3r65aVx`P2c^>}mjny_ljBokl zFMzl%B#}9w==S@ukEZs$QkFj4Fi&do43XAIdz4Q|g1{@eLbE+A$CZB5*%1K|@@LUY z`dLH$m0n_OJ@=#wtuimmhPJ_qY^7SR+jF^@9|^{QnUku%1)HzMjrTvlZ_Rgi||W_D|2wMAFjM$UO3_i3i0; zm<?f^IJ=8|y$d-qzy9NaLh^rs%y-` zy)c*Ou2%SK`(LLobN(hmRsIf&U63<~)Vbi{vr?I)`IwH~8J;dc2Lt8cThD8>IqaE@ ziHElW;F4by97!{*-_#bIv3hkVCW4J~SiTs62W^d;g|@i34UHVxC98f|U;YV`;Vxea z1gEeQ#8<4mi3r!d^N$icIIuV35_2&*HZ2kZW0Kl6g!gV?}%1zZc^q%A#LHwY_I{E0d3L8O1{AwJlKDE zu3yXk$o;&+CDK%UIGm95Y#;;oPe08=n5~SMCAim4UQWTpkX}1BY2p?7dhZy0a0RQ| zIy_3_OGNTc@RgV&DEM49@JuMVEu5DHYC1(p*wh!#OHNe}mY7}mo_)QStH#@01wsv~ zxH+cz=tHpaY=)==UctBI?RI~*Q?`w-z9NzIQyUVgJX>5=OxyOgu!FJvbeyFrw>A(+ zyV3prybJcxpy{Kb2ahs`+KpaAne-#&2?7!s3d4q?_KH$)=vBjqm%xIO{P2PTs*qMX zt@$GR<%o+4g>MN7lfriDC^yhZ3j!h9e$H@;`sU|V^fWQlj$U$%tg}E5p(-wiPc;xD z?1`>m$BEn8tngznIDyE;ig8^wNPIQfk52H@0|J7PLD|of_W^ z!dd1Dl_b_n9jJCnD|}gDQi@DO{vf3x+fQT;J}(Ok|G-I|VIjm};i2%2hwc$01=0(3?(E4T zPH4{SSMh5#x?;9$u(o?_!jUC!%eFydRy||%Qwo~zv*yl=NScS;sPor;o`hsFg5{?qpX5A)M;$klJf=ScanKq)%8oCYqPd+8A*)2J@PnmT zCY-i@*nLRq9nV^2@K#i?g5wI{dp+A@Sb;V`48L9_?09zTB|^P(zLDqXa*HZ^6bdO6 z$L6=bD^P!#?@xan3B?2`oM<$izj1s>&i3SHe>f^lGi-3zXh_%KO3+`=T?n?ApJU^Y zPUyCu_{hqj_%?|2-{B~VRi^b^TMJUQ&t0#FCwL4#4)n6Rw@I;$tc+*(J&oj|XU@tZ zL@y~rE3od-(coLrP6pmYU>}Mj7|pZPjKMlOm*iV+;>Y#Bz~kdV_0Y;3XLgFtWNsc= zn7+h4J@#^R)<|H0=#JaOf_SrP@~Vp;*kIuTDY-tXV09pZv_kL-^LbXv7oBxEyi%Fw zFrmgjwDNxJ1S20_5KF&*kNq&9SDKGy8bLKn)CR7;t(nu+lQ+7(ak?OfTga@R1O9T2 z&N!l&KcMAVFJ_;`(cf8ZenCYguiwHLzl*g(W#}GkKMK8dr#IATXEqALJoIOu^so() z)4+ZY>iYI-JqCc7G|63yUSGhcnfzPrS2Cq*i*RTQ=$& zhQcp(yVrz;K;!$a?}AvH-_Er1x-Ys$2x8(fBzy2`{*u;*m%v1LN&KhEu*A^F+cLsY zaSt*^>R_*^zw{|j&5uWK=Y7DgcA}8}E;Ls>&d-PulnbA1fp%(7>9gvP(3%XtwaND8 zoDsT8Q~|Upd*YyUS;b!3YEpy|QK0-YU1BUTq~NFlO*2iz&<}(dkJz}yOwEZSqQOhP zHNaTY%?Y36Lc(2PN&q{L`sHJ}$R&Yl@H~#P3w@dFhMTS*W&q)K&6$+>b~0zC0j1am z_g+#X5Gam<)&K$?7IOdKX@SIp)$xDc|1&r$BpnQ=A;zV8x-R{0pSI$OS(?K{^QHh| zb@YKWxo^jNiL03K;LMg$@sjsSCJvK>KZ#dpmtA&Y4Q9F)s2)sy@t1TjCJXc*)A48?cDB7^UlY4@4y zpc~}az4>Er+e$_C6vYK%%Pf_G2j$W-YXJXRZ1|?4WKj#)I-<78%#Q z-_M{}l5Xo(BS;UU z6#6rMEog);JzZPsVHXu;{A9+vQ>x&~ttfIS5bLTkI_H$=@24t7U3^b|i={xW-TnzR zL6$k?_h3(DEuODTt!UE(A6>pP!kyW64JB8OGK_yd=%ITB5*Nc?W4k|D1qv#3VdZi8 zO`CM%P9*Gh;X>U!U|%LulsJG93+%%Ad?o-!cqdCMJruMn`bDhM8v_*4$Xj1H8HA14m z8tW6&d$0|0F+I`i%1L6L+tgX+hq*lMNNFwBS#zwm_LS!giHokbQ_)zAs8H}^Pu}Os zjQV;QQ|31`LM3hSfe1N~?Fmo7YaY&i*H3o^rqO*%s$-p9>1sn zX+^n*0fsSMsD&TU_Odn;OW*x4 zQkw5^ouvj`R@7LbcPE*c6Yg@65%$OO*fuHV;fh1?mvj;{BFY7+rB=9*WNtcZHc6Gf zp_;c{BQD4r`+pE7HrO$wocc5H8oKfG zl#`OeG_w(^Vmxu;=e18a%ae7Cjh8{;D&WGn^&@1P=-7xY;&^8lO*!9Lg4dqEcF2Jv zhKz_#&IJcxUvVwKzJ3)Dxh{(+NU?niY9R6tpUyrj+YG=E^`f;TiT@$gQTiI-eT7vG zhqlJ{nq?*;` zhfO}!P1iT>au+8=H%3RKGOlnQQ#vMBkFJpo>cS)0E z7GJWYJk2CxRVfY#$YP}vLJl~BYW>c;!m`#RRxH7d}&@aVF;>kiT^8aGzT?zFPMEiX=PSY zL{U3hviwv4zzpm_IdtLwt2!~6qGyit9I?B}m>nMla5U0MD+H=)$}XjLZ5Do9y;3YU z@W|S%<^k{8$yI4|6(LdXBza%#w3u<3Fif`-&n~13Kf=gb-77y4ANdzEiJ(!f8Gvty zgQncN8pCSi@%zbW3-(>lz@pn-o3n%HM}xKCg$_i{;VSsV1rg9HWmNF29UQ7ty1}e; z6BEY=I_ZxSZz6iHA|k#)LtRuUU(r-0>Y~djihO*E!A?{xBG$r}yQ)GibdHa+azzpF zF*|bEl)Th7FbcgZKG&|WQpZ!>t0auFnM8UPHb+$49~=O;!W>_7DMx=d*QlsE-&=6O z^DGbP_tvPTQ&XoGxf?W|!xusTe5kRmNqUe13d=2Z5`88lpAkQfqtP~(yD%et(po5>F2 zQ!MOJqfEgS?MX*wAMz3#_cT$xVLAe5tE?^5GYX)c2in7>QV95j+uL(ZgD4~=LWnyi z^VGb)a*CWnSz$jEiKbDkgDfOFte!J0BmJmR$3~Co1n2JZTw?7Ze)`yZ?svZRleI#i zHb#28G#u!LP)4wsRaY41Vx;+8tDiopaG_=J9ZzZ8hr^?ja6mRO0eCSM7j+X#AG1r& z@v3>$n{Q9LV}bBtwt=YoHoWp-b=KokW1ZQ%*i~={DFrrh7NmgQ8zUAatatstw}aaC zy^sP4*qj^fC}|wKf1N!(Y`7gFEq*2H2T=9{24w$cTZaM&>@ClR_BLz$o4*9yZ$uX% z`Ua{k?@25X7kIzBER6E)O+n75yiGHIQxb?I+F1<{Z=u~uT!#5T{2jlUyb|jxz)3>@ zsVYm8QG!7IPkM#6?RerF3t4(gO@l}te@Wj)T=^_U67^nv!L=8q=-X?2MGeY={ipmy zrJn3;?MJDH8KC4(>Pea)QVs=4Y9e{7LSFbH1C8(&3jGltbSQ6fUs9u z@A&I>O8Bg?xFFQHt$0dvm0Tt4q&Zs)q!Es8+d=sOaZ6-H#`x}yOxY@MpEhuXMwh{0 zpW$EX0-jG~l^W5@wocFWv{wHilA3iqwP(NnFFIzn(T*4Z438)R?%-5qYMbPha{+wY zJd7(_vt;p|qQHoxF=fa2-o(EQ2-KE>ept`{=|H{91+8cBdRfrd@ z8%R#Vgq*Nro;1w3aYe(kOS?&oe!q+;8kQ}PMC3ih!Bd59yLebk2k0v%ILU^9(Z^_a zqM=(4xN=SUS#0lfAXyreek*aX^#4ZDPTi{WOdX!?1H{fjTWsk-^5OXO@aGQ}^~ zRi)k4Cm8&E737KY#>#5^!WX_>NgctkgH;PZ{I~e|dc8LC1C(~V+Fl><1;aloCYH1> z-Cr<+p{oZ%ceWjM0c}D3HJa`3&Kbe;OyMrV@4tbdDxsX?0UK-P6A1|_mlb2u!q$Z= zhA9DUAe6_?<|)5XV{z&Xt`msJ$;8NxSQ8JvW4Vc|#Vq(=RH{24#u%y1v{VKBO4>)@ z!#_BYkwc$7`W@1AR(ezBoQ$vIh{Hh{I+EHDq1z_X5a-Ui=%l8}!^{-9BkNMc)dCo) zUT0DbuMe5(*Do*%{S<@_b$(V_e}7}A(?+8gni=Yoz?J^OklS?SdLd^DczL~`bPh2k z1R*x)S(^lAOeNz_v|@v_kH<1XktfR$NLbB8Ydq4M(z{L`4xN?xHM`&8lrJ|j^3=a- zIOQV5=ZGH&fE06)upFIGR^Ys-8!}h+K$GVMlGnaA z*FmEsc#pJJIiUAYxfS@DDYpv_Sj^ioniJS}yR<8*wpT%5zp>!dbpZ_C-{)a=cE3g2 zgzyM|r@s$!|<}AhX4x6`kTnC2|)OZ`^ z#?l1Jl9-%GpYMH8cmDaRt!%STP#eHX|1AlyN6krVy!#gb5X*u*{GQaCs3AcAi(Vq_ z>)xuNzWZ#j=vK&o@IK?Vsj|x^2I*;F-)S9VPtuhz@*oh5)9y6+rG#fC#iK*L@WHzZ zgV6YiNvQ=^ct2?w&DlDb!T^B)xSX>k0qwI@$UL3u)^ymQge;z)fGnKc8j4o$$n%?k zgHb$S?dt1N(kQkSjASX*D39*K>ofnj#mB4ToK7a5OABV-C&_6#^pV@Q4S%F_PmnX6 zE@fgEYIa}Ln|V}P`Vh5&NZvx4FX83yjO%dzmB)FdUGt!;3~uqOuJOQhp2+Lusj|j0 zbuo(H&C#OmC>I}ZeKZdQ#g(58Cy0w-;WS+zljDq*0h+`?H7rr4*1jyMZf?YxOBsT6 zt!ZA<{D8NK@e$!)_jZ)_uM#c~)Yz}_Ler>2_rV#rv)@X>w;XMMj8TA_!tdC-xXX%tG05Q@)p>1&MM_2Bx)$hK*iV790vZbDcU=lzz8XV>J@y zs%a-S4i;hUeYoz*jWzKmpq11blg+iE+)xIy1&4JP zZlW{fb&pQQBg=+HBNu#AFW$ER%6*;&Q>yu%>HBC9(XtL0Pd-zu*6x*30x}lTIIrXMqia6; zbE~YFDyO!`zVAz+ty_(=U_+fJ4v7Ul^QC8&hu3LAUZYyfsU2Z_dG}yapYx@EDcCK9 zHB48^jyZRu8Am?PQ@~qlUKx1oyI|dLM7o1OqYEVv26>X}GPTn3!bE4i?e#__H=AzT z4nvJFz}aAZ3Vpvr(q9;z;9Tn(+s}MhHq(I%+X>e2IVQ>bq9Z2~XF=|lcj&?=Cv}v# z*kur@cwJn=qmfkryT|f)<}KpvUhZN2bs5i{#{vBVd^f$79fXB%{#g7U^M}N z2X%$>GNPH+@8srV!aXVlGRexkqcE5r2!#I-UrdjuH0HEEPZNo^k1x@;FAPIRvf$T2 zWx0sM#jdH1CQ7{mS@d%nUk#eICStMlMOmrL0)!2_ThpbXxb%aRxj72q`Zu|Z*esOI z{hkHR@H}z%1t@NLCW6oxh(kF!^9=PrVS<%F7t?VdCZ|zhwf%a4)n|`#saF9!1_c?7 zNiKHcPjeJ8RCt*fylmPK&4c&|hmDT?+A94Dm+T0hHEH=3=VEaI&xCE)SmtaxX}ahD z=j{bHCUft}lzp7=?f>4nS6oM<8-_H7Y`?Uif2zo`9*f@^OpCO4bgmnWgH-|b|KNl` za9h$S2)Q5nLJ*0EdC7UcGXi9T#DLa&u?hUIdqB*VU1rzuA!vbU`ZM$tM2C?(OT+1H z4CF54v60a7D?_Fkp&KIyE%+kl_DUtqP;5 zEbGlz^ZGo1baIpg;FHOPoV*Z6DSClEoqR=W-d!M++!`F__fYFnUh7hrNk%T{?2)ku zeZ+@pudUS#3*YJtRL`yAZwEdBO9Xjh#7A(fG5SsP-OO^FS_VB}RG$%Vl{3aFW``yd zrVN~CdzxPj3?I!N-08`!w!1A|k9Jt{gz!NNvT1imqYNb(=G}~eFNyq>$6VQ`!XE>Y zUas57)&qq!b&Mq-zP}kpimu;hIlTOm?@LcNT1SAYST075aG~atjEU>ogSVvv7crQC z)<}N($QT{g_Jd(p;x?S!Y-#Q?>cZth_rfkOyPFR$-(;W6S?q?6>tHpA-5*aBi_6^MHU*KNaB)K5HzOGZwS~p&7L;fHR3I_pa6}7d9r%3-Ey94mE(O$)?U#X$`r5+1;Bin0EMogi-NjoN&wrl^9iHsS8wJ>STaD3pB#S$ z7P-qsf6F3nmrVyTtn)S#Ol--&N>Jw_ZK~z6?eu(vfDa;sF&lBCF3;9^i0ckY>et<2 z-hBx>!3y7E&khrCKtlX7A7=>N);MdADXI8iA?w+&qfM?9z@G3zM^<>Fh1)~0$j*!}1bQ&iV&`o3+`BBxmiOaQ^n(ZItYmb! zy0PGnm5=0A(Jtc4A;xEdq>#=2G;?jgUqO+^)c!#v)aR~^jKu*t{;4o2!Ouux$+PdG z_~~>Dp>MYp!u0TobRTObNv^pY|KF@?V_UnH!c)sjpP zrQG~=>8AglqYJ%c#(r&qBOp)oK-_g-X;56onlbeZXmji%&$H!MPQ0iKguMmnaOoh! z!qZcm%2QhW1P8WYMIqGTl+WIO=3y9TZwQ4nR53%!H367NPdg_{@?$-i#+hGpT7}Y( zy>~GMY`5b4;-66r81!aJsJflMLEVvuN(xC~d5{C}Z06V4AU1($ts@qJU0b^maacvi zDb-ZciT7&SF0Yaqml8h&gi#|vGGD@BhU;Z<9|=6RGGRt6%3*X#GVUtJ9Hr*?VuxTqA<_pr}MJk-~E!X|*wJ4hgOi-cS2$#t2!w z@ne0RG8^2y$U!EBaNVk~VhWtw^J5IlA~S-YSdW-w*M&n!Y}|4#cisNTwS3YGJ$8=L zCez`vlZ_y~tFb*?uOu2dQ_j1EL7GU$p279o9`?X+>}1@s0e_jM0#=Ojr@n+XF9Aj8 zfEF)E-_+bpp~;!f@b0z$WEQ*_#K%qY=39$q!6@f6Te?VCT} zS>#mmN?ZognYoNQx)eke>EA|(@-H)rKDv_r)HU{sP$U;{`AKPM0y!+tKp~^NF*t_M zXUU{+M_Bi!d->UZi&%W|5Y@71`?;sl9ApGB>1lJ#w+`=MIEDwFRyjciNk1himpbxL z%`95CCx+tW$Lig*FErEtY8+|-6s@A@?S^y)iVl6jPw7zTO}Hh_9W!VfPuhN|D*>|t zAMseDs6{m6ic@7?#eim*e?tP3nZ{y1D+P4m)pdQaPFAdj^F?ztkkV{@_EY0EgmFvF z$WJrPtpV3~{ugNdijs_k`65*0&YcQ$rr+gVrJt}ra}23elwPw7bj4b3_$~NvnHFbR zPZp+~>s7bEVqNJcP5eQ(jdtDeL%6SRepTVghp;#2eK&zZ91;rvF?a!iRu87qwZW9D zj9w1%&~={;JKvV;^KVJ2JwU^0DYIKolnUNB!UU^>85#Sntw46Vnn24{s1A42k*c4M zR)ghV*e&kg41}Dd&bp7dz|0Y;n!Sjau2BkeNT19>lW@AEsXwD)ae^8vgq|Fl< zq+)K@ZPly6gi_egJc#3=k9CAI&w=Hrt4|0Ru|(bG*^>`iXMuy@HP=2w!=J%%9p0}# z@)iVcyB$#RG~CgGW64Fm#zK>Vv^+(_oz+Bjjai)=#rIdZ$I^eS3bD@d#pZyk1O2QE zfkN`#4%~>A-oV<%v2w~zgpMfx8sxBNp8|N~G60W&j{}zl>^t9<4>c=~68Fghhk9_J z<|b@s5p@3*q{MEI-D3_U{g8RFU1$idiUn^zJO^nTcN$K>TW(sz74T@%FKCs3B%^>F z1?-b;==iXj;Z$d;ZFZiG=55lggB!@m2zb6FEEu-GkV|^`3F_)Bq;GO<#|4~VR`J{x zFsyhv0_UY`qEkOb2@_*31Yb?YED$7;RH}-Iu-WAbhb@fIr*fu&hXz~*pr_i6Vv2p4 z=`+VC&Dd6n^u0N3k94YOUDweSFLzS6{O)pIi%wQC;7Wk_1ocTX~$&rr32%cR!K0lL0wCIBly)W6QpNU`=H z-g=Wj3J$Y_MgH|>Ts41)GsRTyh#p4r ze#WX`H^sPPdJ%R^RI~hG*>xNeG6NlaFzVkCw>*|VuFy+LgWRw(O&$q9Y*td{rx$F^ zAV}|s$+Pa(NDPRpK?I}G0uBT$h)IQiC3)}r1&>H!i7dt_RDV}G|CL@x<^z?`gJe`F zaRyJcgQmN{%1A6inA{VvFdD)BIK&hZ?0!~aI!F(eLc%ohHN!xu&@UO*^#T3PX{v2% z!i9~EyfTapjQC#@qfz%=g8=(eg)mfMEOde>mxrf}ibzX+a+3GQU^zjeh{Y6?YCT%e z{taWBSa0N9b#Vf<^GdzIFl%i<@aYJ7=4!vAn!i1E&_3?9Ep4`JJCDePrX^qQbk4BZ zPX!a>U4*S*(z_9bl+lAF{4L-fw3Wo>_$$-31WyoI!uE-v`0SVVlr9r~vdHM4RT)bi z4L0di_D5KW{e(F`)ih(iTWY{eBjRjm@UTd*^88p%9Te@r9G>g{^8 zF|1HjEjbj}`!xugGA$#F)2pT3MaYRRyMI@~%(C`+g?|4da+%Az&a8?`{+A)TY2lZm zgY{%2)AG;N8XzIreuht!pHWC&SYs_qKI}mAM7-pG%Fy=AbIp^N8$Xj@OK6DRZZZ;%=vgV@ZQV+-5xEx%C6 z0i2AxlJppp_8kz4Gv0})x2^<>nOT979~Gss`xfCdtV~Vn=@*(juD%drzY)uROMpw# z_Ss-TzH8E+GJ4fAi!#u$CBq{IZ2-AlTnWZ|$bFl&lrO|`;Sb!-f~0@E zUNTV+@dSO290*OAfh-@JmDdk0dPC9`I)dWko_i<$kYJc`;@mV(A5wF?@k0UIU6eFE zlTO4h#kRlS0Y9{h2QJkk$G(#-b0~v~e)GtI$>Qb_t?6C~N_8g#s_Y)n>KaHs_46ex z*MNU6OtF2>P9OfR5H!l!ZT~VTghittkR?Qm1=N>3I&+U1wh`q}(Bw*m4G!o7%Jbj+ zf-_GA-$yEUnj)!C>28ZmvhI(EWA!6vZ4>mY!&E3;`P%$AOq@M|rd8Tmdjy}e%$?Eu znEq{sssh8>54aBKJV51Y3ktAj?}HPVsB2Y(mN?rRS(fblcGef3elIkhq6hsO_*xna zYPY&X&h2zXqZQTZ|2$uM>M77>l~;e3`@t~rje<2X?#$@!K% z$m2B`Vn%|Z)iQ|RqK-OgBg2JHR5!#DMW0R_-R-w4)Aey#H~Ee~xwsyDMA^a{p@~c( z1-hK3iuYOh?Omz#hq~jDl^F`3Nr`HXNIxcB&JyzwSb-|8otcm5^{E*_L^9`U)C65y z)z|%YNwL6qI2jMZ?huqDg(_<0U~arD9XvcgL|@6j7aS4FegPIuRdX*g^QGoP{a9SE zNNA{kL3P4;pyjACt`mb6dprA-l=0nzn6H9PIO1@vg`b~Jk>Scg8t6AE5H+&y1aI2f zJ`qBw5r6eO5m)lGS8VX>3fT})e)2>_@kHnt8xg5p1BwlH8~T=}0GH3drmRt0dpXRw+y1Q`wejgWf#*OdADvVqsO})dKrCbnEHBXDj+#2%_yT5k` zgJoM7|3f8-Rj-$SG&XkEqua4?&srUH!4fzM%Ref76B7ySRSDF?IfOx%nWt{2<@+OscnnJ|UqJcf2+{_=yBq!&6&@+NAV|?mpkZQld?Hqh1&-K)8%(xNg zc@fT(!mPLb3#(ws=2`n5_xJmjIi%NvGDtu?$ z*Gp%PG=RoIi29ts4XpUbSra8dtTrE4zQchJI*1HCwS{u0Nja0+gy< zr*9ZKxhphtdV>dRkf)_JRpibx0siIGqXhSnm($ zZ#l-n0d&krciW^4BfMqgLLysaDMhf>9iy?Kj=e!7#cb{f3w3Pvc;@_dud>Z|a5oF( zBl8VRcy9m6i;VvGD2iy2vE%X%(NjaXXQCMu!_P+g=D{yRa3rnVc55QUPjlzr{Lv_! zsUYi;q~F%?PJsq^cY9I`<0*;NapBTW@q%nb z!1{^3Z=+!k963)9NBfY1Q}=vBy&&Eav(o>03`veIXbS%Zh;i(2CZ;fRK~@ArXR|0^ zP#b0J^x-UY%lP%kq5dXz!esW^CV7aA^`?V_v{nZ701n`^lTW~vGzUV?J z1lS_Iq$EORxnN@qS61?;ybn)v0g^KqOTAvNS=)Nuc;#5gn z*I+(%6l>cdI~?4&zBrSo9=Dj@v>jwI_)m>pOT_F;V8$iOJ8)lP=}7JAG^12snEz&0 z$5KU39Pg>v!hPb7de-^IlvH_g+g>JRcVE2WLBRgCEjl4!*Uh8Wu4)(J%ScqNRE!7A zWS0>)!SCnSe$zuYhWwls#@FW~21ccOz1Sc?D>vVh3;;^^PblS4tI_RufR0>6qoyiW zg0~8*Bz0Ctl}?qaUP$2SJyGoTd0Rm)JakXEG%yqC%kO&_PPW~;si%%sOa%~2$br}u ze=5RK_2s7PH@r(*DdCv$)w?Vk^!LP3*6$!tr{r}qR&ZbJQiZ7J?X@7^DqG*FrBXBo zFaY-)>$k0QZhdQ&F9SJ06uJH?u2gGNoiO^53Lw*yyh&aN8%^hjx<`#cF4RT-0n(7c z@@kYogMKUvGM~!n1YjqqVml0x4=uZ6S)+4qzDw|AmGni?H6Y&h=`_bhTqUdlmu2EM ztIV3}t}CntOh)vN4+aV&<_iP2PTdQ>em+1%sH#L9oW?6q7L_W(tx%CGV%@^>EeObk zejRx*cKSPyEI^}m&_0~7FljKJBm+8?oaiXA#TfsX1+pjkAKI7Kd@G%U!Ws;mhFgC6ax2`g5HY z@5Q7=m;%6z@M=ZeM~A^&ic=Xrq6 zTVuc?xrGk7?n3Io7aAKCc4hqAJhJ*x8|u8>+9*yi#p)iy2Qq3jpnzpDWat>8NxLu| z!8qCgUKl}DZXhwuX5xjMcw%_NggJ}bfm(bK#KdTCCG5DjIr^5()`A1q1UOn-iYnFq z&LU}RRda5OML-pp*)VW9G;R>O%d%PA;C8T6fye9`$eE_wvv_Zz;X{UaGXS9K$Yvu< zqkd{~oH^}T$FSbM#;noZ^d6O&2f%o|xuiwOx7sH!Jsoa?Wi#}{H}u~anTzj!Jx&7M z2VRM%OCcuBN9*ulGCc%De38~Zv{gdA<< z;vP~#WN_7DIJ2^Z`!$g(T)dUi(uj9{|?aP9ic5NpmY7 zB16{U0_OPI4MKOr8GU;^P?B^G{Fbc#6Jk;RToq}2o(TqU2<)N`G*rgu9RaHJ$Dz{I`IDTzzYDMw#0;Nng)& z0t2$vUQ`D-looOka5JVwhmIe-iT++P560Kn&WmS=(*pjt_o}vC;xxsREC4Z8@TzXR zq>ZsidzC*JW0rFdCYIp$TKcCfLDdzJVPQx;Wt;U6Q0hA1Zed-kSj5E)IMO(_i4>o^ zpy&Gei8@(ccmr>R%85V8Pu<8LURw+i<>{T=)kp#a)~Q6TU}V(#tCv?)BsU2-=^x%J^AayK||# zdlIa@L6P$o{97DGyEjHer*HA8*Nph-9PlL&!kDmuwh@BY96ZU0G#f34?CZR}TIvID zQ&6P(6k}chgzkG?!}5Y?w8M(nwrTgj+$Z|ue4bhthbK53TLqJ-$=4lfepHn@^YaxH z>BQvH*FB&mZTPPeaRFbskuM*}gZ7!jIXKZVc3KiYrpo?RM$!U22SVaF$3f$@-K;tn^T$KYZBTtFaS~u=rY$4P8nHVobrH6j7v=3ph`&jg#atHnpdC zmwW++xb1Zsf1`}`$TY@~rO{Bn{Y170R|WA>a8E@cx1Aj7#JXH973_aEDA&oi+m9Oe zpT?);al+h~%OpqDpT#SRaBkea_hZIQi_c=KkqIq!JAni|tAS#$yIhlRWv~A&C%Q=M zUads7#iAw~Ok1v$)GHvUwYh~i0$_l+KKfbpN5)I~kM=UuMmY0YoN03C?OaG(Y2kc@ zhLnK>KBon3wTizmwOu|hPS>)DtOCGx%r7j`jBxv!;Ij9&#u1!)Lg#FiOV=WcQ;#L>z?g z2Qj0_l31cvi?L9SEGh^EZs%Om^UdOhKHSOnNEkI4uyuStIOcxObT%q5_5Dl&5+eY9 zk7N%QB&kg|$r#wX0PrW?N4lKiK|JGQ5F-gf%R_KNe#ruwclCd4(7Ho+;R7H?>bQ67Kgt<|KqKu8jn#szaY6hi>Anj|qlsR|>l(;0 zK2Sh2b%L-|M)TN9iJO#nSewiodaJ-BIem$L>e&M=+%}Z9bY+vI3?Tf_GnXC~99*SW z<4IV0!k-|Xf~#pbPfD0C(D{Ulkiqmh-DvVT{4qjtkS$v>rB;NLUydj#`?iMMWtqxK zNP@m=){Zb(KfkOm0fs~1$OP!!p~M09JBez&JBdq8xw62_1#CyGgXCFDb z)E{S4NNAt%QV*g|2U(Dmp6{Mx;D8{G(co;H^(UVLV&imgm6!@UgeHN;k#*Ua4kZq$ zte(W|N#g`LXoOq`(a(;QWP^++DD?>HT8hlI;H5?L(`G~=y1#FP?68KAOTGXg4X-ch zQgK1{F~@nmugXQW9qvf?5Dz!u_5MVC#vSxANbVlR=LODGS<7p2G%W$?sVXOH`#RV} z*2%jR8}|ZQ++EFqlF9%x9K9OrK|Du>g)pqxl%+3YO4`Z4f~=rzzTYVmH6mrAY9xKI zD=9Q|wekyP2ct2X1w8tg0#zBlfjqmH-ydqBrt=pV-y+-5&aD`i*st`>255e=l~B-a z)cIVh9v~s%fFdNc^~%Klm)ofo@g34Mc~$=JdkfaG7Ib&4jV<3M3zz&$m*4)X`ySjD z0~l7#JU*M)W6MwOl_413l;b;(?-v1Kvdc6-IbGRg@{TPKLK-3rLXzf4=<&{DhNzh8 z01(q*-4o|>;=YX}B>?)UaqE+zQLB3&5LNsy23ll<`~_*$<6y}JkO&ZM zuv4)`!&n6qw7xLT7bEuNoBM?On=b_%4%U6MFCM$(0)sjZkEST@s)QxW8ht*xMK;G8 zQdbp+j`a=mFq7M*f8#|;85QNWHy$VdT=3l%fB%q;wFlBDtPn}r0rcmy=f>tCy53LNv2|J{^Mcd5VU+;2z;6Ul>$~=XeTzU=V5hf= zo9Nc0bJ0B2cG`c`Ldl>&tKhR4t9Z?RO#4u`O^^?;a}W?m3X^QE6_#e+ zbov7vZ@8)Ib9z%G1qwOFGapC_Njw?G(m=EI_#}-e!@y_oOpFBEs*enG>X`Hw(7>QQ z#jIv%Aa3n@rwsFq8)GVV__!Tf=#OtJI$M_gpO-3t7A{T|II#5AKVCcx$ ztY{P+Jut<)=cAUZb*y7qomX%z8ztw~ycjF!U6eR3XEFps6h9a`H%U{g+~enKJh4a5 zIH*lcz3#(=s)-J<#Q5T0$X}o!2fSIkJ`fxpRMH{G`o#!O7HZxvs^a?t_qV^?7$jNS zT}4F@7w=J%G}&9fVuU&kc8O*WZ&x&#`0!j=-^eq*D;eUD4l~MJnp#2i13Sn$&@{x4 zt(ibX1m_^ELQsUp2RUc$`|`RIc}>Xu%@2U>Cu~o@K(-oIXu=$LNW+22v*ZYD8V%e) z6R?3_ntNcoqTjD<9})2gp?tTGL#fd_(z|G`NZs>k#+oOeKp}05xS`!I$YIFVeTKJ# z6tfa-ur~2OZ`Hv0&_IB;#{#9)&uDe>y2PFD`pISWr)r^(GOgdAiA8=t<(@G=8+_CC z>{CZTIX{|^#N<+-&Iodz1(YoeTN|0&W{!S%>6IibHE*?LLFht_V&{sQf!$Xsew-lo z8VKL{sfBlXsFQ_2f#ruay>BES%JDX2LG0dQ?>uRJWG@>No-NCLsM+ZsZj39wkA4T4 z1Y1Gi28*S1$a>H9>;*wzD5>b=HxuKJgUe~jK(Thd*0dwpUfG~tKT)7(6Y(a)kQ4zYfa&(xKQh_7V<=xgLyN{?an=~`t-eAJarX29N zhd*4o{uA266zmnv zyrr?u6%|TQALzW?1j}QU#CHWBZ=-NznURTSbl8kiw^25N9I!k>S^dEMSQtN+cG;#|&VWqa?%CwQ_IkXFl+emz2k=&r zC_*Pqfq7^}CUMHAgrY6RX96uonv(I?&}&UX_g3+RV}YOX2zewjok-BW4~YFac#j`T zF}LcvlcVCZe$B1x3tPV&II@ox)Z@>WxjOr5&8mo#`w(q{o=~^JMYo8jLjJ^85@-@b z92lK4#j(Mef9-n$g(m^`ltKz+Z}-X}u>%oLU#Owzry(ug+@!SX_t*Nu*`dB$es*Nd zONMBsihL)3S?iQzBJN-gFoAWMr?h+MjqXnQ8pQ>7%>ya=k(uEz>8s9~ zcae?Wutt+n#GvjIj`KcP%(VV9*NAn^8G{34mq@UNH@MdiVPs!>Z^M6kl;$97;bNgN zY~~oeuL?(?5+ry>#gVCCjarq~wdhuPqE^O1Ph@~+`v{vEu-NW))e^N`N-r|$N|Ma3yX$B>KOyj0>rW^pv&t5yghc7b?zCul% zC-4wojf~}Gj8$G|2k9=U3k@8q+C2Xnlbm?j;_cN{^JJGf@rtq{edd z`-=cc{y|v1KLN9LFhnDwC{K+^+27R5bN{L4#0n^)s+ZI3Xprh;F$^n0NI?*jWX<)k z2KE*;7sr`2>1-=bgp>WzZ-OTQ(O+Sy+sB0e%YSCs_hZry0)cLnXoK9{ZXxer>Owt1 zxNPzk7U;NyU%x0`^>S@dsE?fvvbknG0ddk!uEg_k>dwufLtGb5?9z*?1IT?pfd<2)?wcA_QL$5LYwJO_mJ8Xks+>`2IlHxTaFl zKyL2l`v3&6=2hbsNdE8(fo;v($>lpvR0MmBN#_V7UKG{C!jA1czIQzW0K*>zq=!^cg0_;QJ^d@}&EC1c>4B&WXrnp4An@>^ z`9j2Zl!~7GSF;j`jWO^3*skV%znkxoyOH&JH?;y7VuouW9@4M*H%r5Bf+>z%@jc6J z`#fg;xil{B(-l;u)0YxA0m}H#gPN#dp>x7zq!p6DN$@%0=_-xZ#v#tYqWp;M5UNRX ziE!K4jLkG^Y{76USqrk)-9D~iYGG^J+${*J>cQ$G+Cd{w4aK<~!Cb9X6uSIa%3N7& zzvd9O{|ty~2XYaG+l|v6-sF9BV?wOMc^(rKWd{;Dz+-T8bG)41B9$SGYEh9eiQ)2D zqF2gtWzh)i&|=#gUaClTSqujjj<5x0(Kjt0l|AW~xZ`7p-WZtu#~7lCUhXFtlOUSs zjQj8lUW@f`SIGQ`DtNNTpx_V|pS$ie($*267t-E5_1Vmvj5zdscLz8|g-;#hv`%v@ zJFni1C^zuJ4h-6GCTKU#;3gcHa>Q;>fe{&>Fo991xGBf19JTK4iX7HnbiCbcMbAM= zZYW_8RmON{xy9YeJ~watA7**2ewj8ngxOw-WphpXKqgbldWZ z%Ef&Rv?UW=elC`GlUoUH+ii$@r**jEL621UN6|;PDNF@O1@!}cL*{w?Mqw*=&5jx7 zkcewOYa9>B(TLNB3bixjBiqdF5TLO{26+Z-*g;s_a>`ZA?zZ_h{+lQ#m+?RMKaKL< zBW)yxJ(zO^Zbo*owpFQ3ZTMJ~y~q#L^pJyS^YtkscY@DU2?1bb-%Ah3 zYfk@t-S&g8Zpfj=sybJt4Ie>zkDow~7T6lfh_|`pR1))7JG~pjhKxJBJLH0#I^PpO zKxq(6-AxOP?UN2@ujjUC2?!y@`wiH*8QzOFJx%=!IWqMl!-{{g9*G=J;Gx7j<{edF zqn2^x$t3<%Ko-0RU=#Q)6?C#e)LYC_q#Gjy1!}`HCVoRg|5JSz)wod3@FR&3@R!kP z3ZCxJWwPe9U(8oCs3XsD>m5W1J?nF$#`Q&HJmqD|yQNP>ZI6Gnd399mdzTALO25PN z?oY{=nT6UpL>8#>`mXZ|#-Nuqu5E;WrC*SxB19CQD9W|8+SR`DtNNc5kVbx2F6O8} z%$-kkweS_j-zDRHK~9&hYfxMYMU09Ybx}On{RNGalQEfeKjCmTj~(}bCr2(N$f!7_ zjo=__h+suZ2N9zUz+m6h;!!gHz_Rep|CZyDG(7%ZNgQ4Vz3kte6#*pMZ2w*KKEWA#>-2#)#@unq5i}AU?nO+a5S$|CRB`sbN^u2x!8e9Fnru+K@+ayP+3QO&88_*VKvEdv4W=?+$CaoP2`&M1QB>EmU}n(Q?c%J z_@rPFLT(7oe{u55@H75Aa;i-TW@sYqRJix-L0mpBIcIcvpP<}1*9&_}%Yi@_HM0RA z`$65YXrj9!Cx6rr6#i-M`|bt&X?#7Kgq73b8|6p}Y9(y7=`D}P!#k0J{+UbR=|T&5 zti;9U$DeP&O_LvZSXpBVXZDye5*5eq$mBYq!{_;8PDrI4JAWtxZUPbrw)XuMI2-i_ z4)5w{c1CXoOCZ)@(gwopa|ahuc2E1^O*eCBvv@7bB|v!O*89FhQwvT}c9z5f3G%aD zk@pDD3u;Q}yfgStiJA}af4>dym=K-$U3?hm6aA**U}PL}Y1vOrM1K$RrqL^GD*+>G zz^1N+d%|M~_E8B6dvHRa+Ei$&_s(MmgL!e0BhZR%OTI~=knDi&e|t85cB$R40^v)E z5}B1MM7YZ(BLyhDq1;ZdEdhl*MGWh)IUhac1kgCBO9DCt0SU;beHBtewLx(}>h<3g zXn|r+_|V4?>ZWO{GC-&FpFgNY4r0Tk0({bMA*=& zur&3sQzL-89-<{U>wZ90FwB1FYK&gwJ15(Irug3FA~yGS-^81>9PIBV3N2xS{3wHZ zy`Y#E;#Q~Qo#Ymtj?fuW=#ORk{!nbvP_T-Dpu3Vp>^ojuZu`{N>lz{d# zjdc$hmH0v`Xn?s`YrBywMIE}a4>r$77nh{jJC)m(VpSH+JBb_6fX)ZO6m-JAFJUgO zj3f+vuNA8n(63=Ig;S%4#w(>n;1$aflk&sGFe%FudVN@qEG0Vx2flDy7d4n-^56TT zoQyAVzH=6(S}g2BD3SU!VOn$Xk zfj#N`nu#1@_rJy8`gwEX_~6(qSw<8+k;1sT*eqJQog`z1PF@ps&;lp{+@$3U^TnkT4>%4~7yC)vzVoRGIP; zF&%XB)P?ew3+MmGn$+mW9kSpeD%jZ#Y+j6+>+dC{}tnE zzGIRhqL*Bm4IJ=^wGz4Rm84Yz>FI@b?LxlwX#J!qyIcI>`^Z% z(p9HcKSnvTv}4F7!Ng05!(skZ2f#XO_jn{)c8LJNY5tE$&|sVk;;uOtnokfVKS#n4 zdidpOEvCsUW%u<+Z&lJC#sUd9WbsH`5c_XRB_=#o45r;h(Dd?SSQIjzDh0#ox9y61 zl-F+R*p8U|d;<$EXJFakZMGRz_G%nXf@2`d%89$vHSol?I&5jX(?x$25{D?#L2SqfThR7Jc#t7y}R zqU15#!5f2k)OU)?x(d8emvb>&6+RXF>FJgaJ2H_LigMj5>jIdi1bn@^;;(Oa4amUH zF3B_X_pQG)Q8ta?tsqsiY=s}Ix)Ca}P6Sa1-BUYv&*8}RK!VP4&hpP(il38!2sSAa zMwi;|BEh6F89WFEjS1RGSA3#g>(DJT7-+eGKF;bp2XNow!n+#c6UxX82qy#BMUK+- zUe7Z#`Y@J_+PIUsuuc0w#0QUEWpOrCEBa4mN@H{ux`}{C+1iu)?~Frt#+#+p?NH7p+}vl-Ja z@$AELPAAaWfh~@3>Q;nKb3ML!qCs<)WU?O*0cR8ppmUwF6HeA$nHo0b2G@Iu2|V{M z*Uf8rK*vSkz`L2y8A_zsLtZsM`rMLa-#xwr&jQgP%uzLDkJ#Uby!I@;!}N{0MV~sW zqm9Rz=y8X>a^Hb3z2NPXqlu!Q;219zX2GX!RL6&-!dm8WyRbHFZc(m|LdL>s@+OPE zw;TKG1JPa;#F4Lth8GNwsLV0ZRmU&yfs_bDsdM0892p z4Q#Pxa260QgJjy*C7+P*)G|M>9U(`90p%WRGh)RGBR|0Pd+SAb+2fsEJ%=G7EOCsa z;V^Uj*sj20_Y|S~8=gjYfkr??F%O=|;NfJi!|3vmRky5V4&kp``70XDEOQG0>`xc< zUN*{X4**gZk~Q-NbXK>KHyU!VU@M%AHA4HJT& zIp6kLu*`W8wv@q`Pbr2^F)e&QlF|_S0FhOB;)zR!4p&4k0zgGb) zp0rH{gn0705_uHbvWN4^fxb)D^~`nQ`>5@)0H@AgbG1&WY$g#>(O!%Uw5OL}$n2E`q9`sTaUy~fn7k6BIH_av& zLs&aUR&SXSz_HMNT>)iX4#0kvzEbJO*8Cz(F-3&9J{y17d?za1HPK@D#)*HqXlu0F z8iN%RJHiOUcV;WH^(|PKvdxgdR^FIvK5V0_@(lu(nThI6HI55PssGVdT!p%f+BR$n zlwD^Fmbkw{mC;zbOI2WjW=$CKbCey5Et%-c?(PZ;Ph|fW!$g@Mo}~NSoT;IQJ&E1b z9y`y7xT(ra7}!7SMbkeES-#Wo!U@qF5KDr$r|r_2&0?nq{H3%l>*njz-5Mv1v_;Q< zQH7vW_V`*!d)?-hnXiQr*m~5gAhi_=9Qo;tfj8{jnqgSP@9KLp5cKrb

Me(xEdH z;kPzu9|G$&{Jsgh(%{eyI8EAiMeDHJB2FlJ8+_ zKnL=ZC=5q~iRbHx>ZRU%NmT^lZUgv10y=}ons;Y%6&5rd;}f=l0XX7qB=lez)Tpp7 zr4?Nsj}A?9K8Cju-vDoCqc{*+oyr?cq(hX}XWL?HSE~AUXAF|iSCS9jGlc2dUAV^K z(;8+*)3j?QWP*bd0$wz8uaJYS+}^IU_|@m8v@swEdG)6eRjy+A?z7Z;{qn}t(IOGK zTT|!*Tj%VHZNW7N4p}S;mCaY8k1U$E>fvD}~wL$vWZB`dXiu zHQ^}cqnR9b&ZkD_w2Z-G!PgVz_OMTUwzEyCPK7OWkAdsjImC#RQhW*ho6Z}xK!pWB z4Ya$I#1c=D`lM1=a#~NEg+>kqmkgB+YhYx?QJmmD^*5Dx|$bT4v z&uB`Q(;|%Xv4%i;UFR!-iu#E-gX4J`W5US@CIrJDW@OewCk9*H{t+2A1Z-M`Rv~P*CRKvnb0YMVYjRNdmc2+ohiE_FFh=?dDhpwT3@Qdk2Bt!4@)Gr?=b)g zK!iK4V-uF>6I0_0)e`{E)N$8BuRHrRztI&7CPPYvK*HqxIT*f?>0&w;O_NuE`3^{H z>Qw}Q>}-b|+iw@k@>?76LkVuPVln&3=Rgc#u9qiy(4bD_ipCT^tfMM~jq9vSDJ=ia zbcop8|7O*)GGVUd+W8dLrg2vQ_kj~hq2Nx#RhfX#B@*To*@c+YryyJCVQL#7an4Da z^XhMLJbIN{A&YDWz!9|Q-<$sx8-sx#o>CQ~3=5ud@dzcT%!X|IJYXtXJeCQoI6%)3R}-$l&xItFJ{gO zT0{$tq_#lS?^VnS6?oKI;?|DXYj^yqLCADkHyl(zDp7AL`d-+GDLAKEEk1zo-ON|J za-JbVF8~1*Vd=QqVJR@`$rnG72{~E&=M?@rCz3*doWxnL>RaWw3CA~2ODT#V6_Ym` zk`?B_`_8lnXnw=<^$Ga9V2NM+(|X*e^Dp}K9DnJkpJMiTy-@$pvQ0IyH#A?`3IYOS~3yOMZxA6k?0Pse_$~u-e+)1Lw zq5#l)<9x70K>RMP@F2y@oFRKvlK;OKxCtS^!b)kNZbR{lg{=5#13~rdj?@bI(nx@1 zpopNCrgK?9Eqfr|CC!!6T9NMCWyh8I9=0^noArX&n@1{?iRb%GS}n7SP0KlSpGf5s zyzMw=JQCP@y61fcbh~-(hbw?G&vV$^mMQ|}E)E}8k8-y-hh3U2s&4u;B`BhDFHk%( zU1O$d`G1Id6_F2%jf5qbtNCJOTYta7N$W0K>|W&Ic}lzor1PA{;~0Io-=*cA7>qEz zWVJd}_yTK)J(Dkz=5XqCGbJmIN5PoHpxWuC=y|{?2e$k@0 z_^mVO$sgv3r$y)&J?g@8g#~a%%xRN5L6Qakiv;UMJn;S)MtoI_w#LK!UMeIFbf{hy zibh?zwkrs&?u-59byLxW)y}wA`fCcgRyPfT8~mo_CDHlZ+q?i3f>{w@v8$MYc5xza9D_6#Kn z>ma3iY*-E{g)F9h3#WH6+(Po4mS4;tW-n)Gdc<=wS;U|CG;uf!N0}$kBEWbzcb@# zSTO!mN!Mt;HL^ZnyH)Z;6Gi{pHR320M z*$sm$=;e#7+=YR&H`#rEn-2#dbGu5bfst(q9yMI7*FERH2O~VYcC94=nS~TKhVwu& zm)Ez2J?&(AZbR|P2eYw!&%9t^b0;ZGs%Q&sk1u4v@u~NbDH_?)GGUTSHYa?MlAw1q zKy0_y*s54{8@ZnHv?A`OpM~B5(E1rZ>pzE~N=3_2Ha_ZXMEF{-QmR8+RlMmu8l#4z zi5DoG4wL^yrfGUw(89Jte2n#3W8P#_+Up9=EKLA5j%jFP?kB1&WuFv?>eb7Sg2#4d z72AwzG2}wmGW34)I8=yE73pOA0Wq%*Cl}0AOX=L-c9h^F1D-S!XNk=Z24o?ZeleOq zZOoUu%XP&_ZrxH#S>p-SG1T};e>2op9wSV#0BdGDPOz7)+h}-5_O+df7tKp4wW^oR zC*Jml57dfl;~*2VVYd~&daeIQkZWTQ6%jQReVpx(Z?5~Cir`r;I0HA)Wo6B1^vnxI z@ucU*BuQlMk3^*5#H@gCsu7qFkra-SDOLCKK0@Dc7NBO$9-|@eSSpahNXYh~Dh~#3o7(0$o zEAGKD6yT&!-)tE5{mJbcY%F|)D*>gD3L|5N95u{Q&*Jz&e+@y7zq9_CtOSvcBT zv~xlZ{9(2ah+=O;mtD=H26@w+~-dJqRBHiKK#8D0v;f5d8QvM_+2@ z?`-{k2E=N@fzSlRM-=aVw%g>v?*jK58wr(^u9^g z1yvYF|5nqAEH^JM!prTfPKihCV8X9Z;k`-#=X${39a6KSN^0Muc%c4QAWwvMFrIbu zk?AD1y3OwcxU@xHQ2cF$m%sll5qZoPnOh~v!_C_qVEqhqEOmXf{|rPd!7`hNZgjuN z46^$IWDB+24v$6z3$+2S0e+ANAn-U(8SkOd8KP12|0i1*P%+p_*!@~|>R)~oAB*(5 zwrh~tn3z$2+}8cfzl7zL(aPG8cQQzvvnybBcwZeEs_cdVlEW}1e=p^nqA}vLUKZPK zz)Y){4%R2_{t4^l=^);7*^q!(4k_pReEh!L=0_;GTQHS0)@|w)JI!FY0n>510vIc{ z(t_in%ZhP6+EFe&A`D6~HM~KBy*CWApT@@bC!pML@D@U}T77--A|wUpd}+ zO5g+B=*8g#@BP((IrP!SSOZ>QUMaJMJ)LwB;X8)7r9<5b=yF%_gtJmbm zz-*eciF{O!s|k9`oj<=~v9NC`l*zYKrpwaS&q|I4zF)Gm942g_9AB21Bv(@ z{x0lOz(3z7nr>oJ=SuV74z^bC3v?C>?g+zOP%dQ97WNaKW{G?VrOfL^hDPV$X-Tr3 zZ`;e>uIFcM>%El6xz*L2$BGXtQux9xhWL4DLJ)_R04IBbByjEC+T9>qYZl0&4XZa< zDohcHb?q4_Fs3=nhfig!1)>)^GjTUBga3`SRw{DY()|lRLKxPbvs3f2NpJl$@(r%a zpN0)GWpr_b(Naj&)F#}k>9t#E|0V>@5l005lagsNJX!XF;$38Kb|WVENmh>HoT&GZ zEmcDxlp%9cAp3tntjdgm*-x_Et#W+<9`!GZy5Rii+1LZxE0}*!O0+t&p~x&EnbH&> zsLVFiIuubbN)#@F0%9jfB8Mv^j;#W|~(Qa276@Q~En=&PexBp}S#y%+k;jrwD-I2>1o*BBdy! zUH4IrSkugeZV{t!CYr?YC`|DCe1uB+$Bzf$jrDfewv5=sLrZV_+9}|R$rtRF*@4Hx z+kFZ7&7RYhU{PhDne3|zqeFx_kSVZ>pX~K8*ZHA|9guPAzgELhf5;uiV}fF0uEOt^ zi4-x+YE``=(u=1Z2BU%D8w{{nXpkqM`N>k*-!@{q9eo3bW#5eV3f|DjO&X0|inE{e z79LN%`^6L}v@|w`+2$2ltJK>>|8Az;{-Vi3NE4oFHK?F{6ZF{j19l1i{}_miI}w(a zgWV^|5RWg@5?-nIZBHX-8ej)4AU}lP(iUG&=3BCz{q!e!0y!+^&%s+C+N|*@XBxO|p0T1x+uziGZ>xAM zKxOteQp0$`zjilYI8_U0jy~(|zGJ zX4RuMlRBl(*sW#JK)Ot%X%RcMNKx5S7f-?M%-&U)Rr)Qj33dLTM$SmkN#G5hFN`PE zD`#SyWlcrE-m}1UGXK}c)S@ackPlI zc`);?N#6cyE;?*(aG*GpQ{%qTZc|+Gn(=$e(VA7}iCZZ~5%LAk%@(v2%?f4VgZN&#?upoVVe(7dYXI^E3_D8VSQUQ&AZ2I&Nu`~lU5j)bLi8du z9g`(#&VZ}gOOQv7*n8RFQ})hrOEMl87|$>XyOFAUr&F;vOJ4{%&+@!w;1=ST4+-(c zlvrTbT{o{WF?MuLZvzW4=|lGRc@dGNv{^#Kf`SlTx*z%b^knlCgGL>H#!uKq0_EUrjH%jQ;?*5qJ(Rfp2N7K>%M>oolxx4)vexrfzIh=^K2k*W_#w=Fm*RW#2Mt13@|$$>s1|2>71_^dTt9|7WrBW>va z4pI-e7{eq*=N0j80sOH#^({!9N@cl^fN4w$WY54R9G~7^6dCRgae0T`8)4{Rv1S%s z)P4suW0)|DrhlnU9>8z%ES1hZ;YZ5DUla}G;qSOcNgLn0ZXkfM&mog-BF1W&@i+sS zDohDu00MD0Y3Di?-0tL9S7Wk4W5Pl4?H^NBgxJhuY>s*L3rC`NI)d^|3$Voc0N~>m z6X39G2tOBdO{~X_9e~$z1LP?zDZQ@~XfE}x4av0TU7=S-!}uh2ZO9bG4n`BxGzL1t zjkia67fYCETr{@Uivui593!@Hq_VTtIOg-xQBb+G^yBDDMK&%sX!w&6S1!pH_lVR#C z#NXUc9XVOY-S$m*7^Hpz>$b=`;mb*C)KaFHa3z^c)-HCJv8F8^0luB^Rd}8QTc1Y_ zS??4$8G*q%BG z9TgyI6{CcK>S<9<9)rxNi&gx40Hm-o!zqXD6XuOK7)5jn`XT`GYtxU8I*k!yT}S4Z zB3AEdDuO>-NyTbI3qtnjBB@ub-#QL4Fq2z_@&^)-)q{R5!!6u%$me{C=dzainO9x+ z8=*nopdq@oh}pzXML-OeJy9wmc}9D8)^3HJTd=+R3LnR&Ki9OowacY9c){uj)*#%{ z*%R3C8W44=pKtNc9<3!;{SEFoPhBH00O9X~`oap2SiRWjl=hkko$`oW6nvM^V`TMU z2@vrCc=MTcU;g$@>ph*M>zt_>D^6_-4*lu`qvu`!SAf^B#eLE{U~~ zzvK5uiNc5DuwMx`%WS9i2w`|yg}t&Pp00n&_nOgZuUfi2LInO9-vx^V07`5aou7AU zu;u{~{VL{wn>mIJ97?!~ghd-MN+RD9XrzQK%w?aK{^E6zArUWWaC~YB44h5jy^aAt zP2cZW3Go0PlThSEll!$T=)GLvqTT@7CCQ;a48g3=@U3Fn(wmBDD^S9VnqE`V;mQ&? zDRHU0zIClkCL}%i=eyf++p-Dki*CvYs6Wvl%%4dv9JK8~ehP(xtJ8 z=GGg{)vtSy`n?$5LtXx({A~l&qPt7a%9n)mknGveQTpdQA(!wg$U_qOi;6`Qbo`6Q zzR;n!{p-=05K5YmbnW0qB1vBB<_#lSEM4C`+MfnyW>ck0^;xNheOj<;&;w;XCU@B_ z@cv#b!G8&y_3Ejl`@!`xUT++TjWOx}v?ttki|~|7=Illh>&v8kkf)!4y4&Uo8ns+5 zDms*NG_2L+RPb0Rdc@W8{SZ{~}Gp2UL&^yQH zM|Q|dzIua1DK|}U8p2oM^H2T&BN+;hrmqV@?;&pSPZ`DnU-&vP2+1FdqY1;#g1j>2 zIU^}?EI{5B^3(R{fhlnKA17+(ZG+^@<1Yj-DmMcdW_@ z*m!!;*tqZ3_mQjs>#uBDB>1T;|7cEljzHfPtw^TqMV_?!C%$&7@2)E>QmZKY*?HC! zAqInZeGQa9;+H5COXJ|xn(#xYpfu1yb*C(Sv$q2_b-^Ld7#&KBTJ$4GTFsO5xpMJL(aHxy} zqTFqIek__hRhyMfGceLn1a!hdXfMfgb;pByT)~VEk_4s-|6njX1KMD%Rga~JA-g5& zmqafya%$>GWrX|vf)`VmUyr%_#)=uQNqf+%TOCBM1Se@9t~oQzGAhr(zCL%NSFMoX z(ifWXvZ>Be9P8O0e%wO@OKTQagd`g4EOP;%3pynX?^mzaSYELd0v)l8rJK^dknN;J zk*jtjIUv>glX&tzyxt0bf}?tfzwc(F*9pYRH3|&@ffJdFv`?UHc*$;=5QgbEy5o2+ zb3CaF)sVd^#&M=3Ca7WEXlb9TQ=Ku~${gl1e)BoE#XbQ7#1QQWX;$`|U*R(P&2^*g zfulj2u~hJq7~r|U?ENs_(83J$;61IMEn!Ru7PoAzVEc5svv7Vdgz)#*OoJg|^{;W_YydA%g+{3u4=+UQG5M1;6*5*eQ>z&J9ZlE@gL|sdWsUb-!bfFd zyRbHwHsuueTv`X`aMJ=iq@|5ijy6y8eo_m+NT2?g$0wHsToiJ(LyZa2X+LqIZ%_X8 z{b+wL32zp#7~`-V01t-BoL<~7dl@rQ)yr@oFK$;9WWH)HM3<3dY(DkSj3Yb_#Bmd3 zX>G0lMwUkK{jN}!8l&ziU$9+jLT`+6y|n=Ys7Bk#{t1o-WNcF(>o--1M>)pu+t{La zI1=P4D9l(vqF1Lw=%~Sc6?xCe9>`S|ovQUsBOq+_Rh)0tJoO9b!MmMtGz8$*UOSpn zTU}ohndVG~Ae~Ss%T_qZ6;`=jKIq=^Nh%5L}E>=Q9 z-f4(aE3ym@B_s19GT)Co-KK25)q1WA`xW;{aS(|ZPm`ol;m%%3)Tc~JrESH>=2pRH zZe(8(F2vLUXd&f>TIH`3b9zBZi|6YPd`c_H7g{Hx?~mg{vV6}nZ%vVXCv4Lf|8f8A zl=NPXht)D$PQ)(!O}(z@O$-9TPhfAQp!S9R$c;Ak-Ze(U&vx!N9X--GaSl*PRv+p^ z2fY5(4s%oO)D44|1d3^PawocQJ#SeAC>u9F(;|xufw@1d;KJP2ALT;rYma}5%9riX zT=!{$%Uew?Y7MQ>;ein8I5Z=l<>p`0{tj^g>G7%oW^l7DoNR{8l;Od{i=^TqXAx?V~B8gB-902c3`CVrI(Z)m}almp_i z5brD*I@^_OGjJNw&Sxpb@daon1k|1`hcDzMSaLlgE>B>2bAI{+?z)|Sjd_3{`qlMM z`CiTdq_c(?!4uzs+uus?!&Pc=^N&{)lHOIc?|tzB^lFhCAWnyMvAUsi8?1EcmT{MD z6+Xb5Z1x=CSPqvZhVksUc4fnPd!q@?*Hk&caL$$zJDAiGu)+IYTBK&hRQhrWVR4;S z)S$S(#6#@&>aw^i$+_V4W7doREHNysmGws~Du)Vv(PpWkBrY;n+N-`1v!W-zffO90 zE+EMtKpzbxTa<6boHqq3Y!Nc=d4;!Y%H(mwt7R?u`WNz=sCF;+lwdtjDo60Z%ahZ`V!^F(kX|+gJPCH`GT+F3f<;@?q z0ce|cwtVNbgiCJBm4l`^@aQ|ZEjeclJQW?jdj;}pFN?~dJOvsjZTJ` zVnH6iWiZN%?tJ-JhZlEyQ-D-)RBtXQ7T$Z+5ie12G@-W^)ga>pOwQq!ncCKHom+*F zEb!^QZ4&Ggv(2`Q;rC&8X$-)mwZusoh6F`?;K37g!p|ln z-4{9Z5Q7mM3W+vGaYKr}r`+UEQZ|Y&2^#E9!$bz3A>;`KauYq{0ZjIl384`ZDRxt4h$>U#(_~4}hJg+h2D8eC8(91W5;1-&XX_HFi9qCOMZyoYED33ptG^i$JcrwV8^5oh1AN1aBAdSd{ z7?-gx4p&HL4?Ear4440%4I-!ls4AN_+>h7@UvhsA#QT`DvY`sw3gSR9%>_In)E2Fn zT`z01-`RWy;zyrdGp_C^l%`Okre(@d_%NcbuD1IPve!7Zr>DQ)ktV6Kkjb~bNo3yhy$=Spa-f(;kc5HDq^o-LhWDC zj}rCO&f|$xcbME>&Xu|u#3k82O;iZl5?vumvj8XOI)A1)+U;-$V{8{J3-p@1B}n`l zMRyzKq4@FeRpUy|>9sg5tL?Cg zm$Ke!TOzNI9mC$)EWq*=u&+18Ts)4Y%)LzsW^*!=7YNCxmsP3U=jl`o+>Lr}=-^nSfk)CeXARc+ zt!}4mAsX5wt+wx}?y>A-dd$}I!Xg(Vd$(G+5he^IrHRV`Xk0L1%IrI6spl`d$@3b( zxjM^u0bDq_Br^MNL*o55<=DB+rzf7^R2EKE&XHWt)ubdw8@))wU-%}vO8P($_Jp_l zRfZ6la!j7G9bO+_%W!DsHxJ3?##gQ7IS(|Z03Zx0D(z85s(--eaWyNYI}JiQupj_n zQ~z>J#_8B>c`!K-GhYl0YaGYEDMv#{p`Yz#Wl+9d|FzIhW7mf*v34cXzb$T){Cb%| zdIG1MKLWeDqE?^67S>C&v0>Me8cA~O^N-$>WdAz42L$PxsPJ#_NNrAtNwF`*pi0e> zKwqWDLKxDm85a4LqpJf63kKN;CAXz&`$@XqGDFBvZLw4y?lT}91YdSo&DF()Gzsw* zc0<)rWV=P;!xHM@c9t^W=oc~?rrww_^F032KKD5FGWFXUyUd;y4^ec@(@VR-)RRmt z;H{;CXEZZhn^;M9#Du*IT#=m^(&?9xD_M>Yk1E#uAf7jr9y=?_JA>CtQ=x&v;OR5ZmBQqv?Tr|q zd<9IQq9YR>LoLrue#q@_6=ojX$^ClC zLxC7qU|5IOQDt%;YT|Q~%r|k-)Hrj48`eP>)2#{q#)FggPT%gUZn!@@Y^;S3 zv4A5l^N6eYdQDaO?NOHubE0&Cf*;unTQF*71ciR|#8@#FnP3ECfytE;PQbiS>Uh$IvZm> zSy`_^$Q(8|;moX?M)^hbV(X}!j@YrO&A?5-<*lF|{s`qfHs(Lhqo7g>4yU{7IFh;3Xup z13I$P>Pa*qPjXQ=g)%BK{OiJ?e=;+2)A+erJnQoeQ8^p64Kk4yVeu~CHO7NU-kif? zrp*lX)T7}x3t+z8)_0_)2z0DBIQe5(asi6fMo0SKXFp~Vr{sRX&&tlScp`~7vHuKK zJ=|g36Wl!$YjXcsAFlVv8EPu|$PVPLH~|eRWTV$t7aeYp2DZ<`V2|y=OL6rUKdJ86 z_($#_XUJ-d9$I@;aWu$xuTS7Dmb5MpOi6A_C%R7XBX_g*Zj{ff{BO@L6&;Q2R^hQ9 z5d8aQR1mfyfxRn+h36EpuG&PV=sPn$?aUNE%Hh3MlqhP|>vjXQJ=bl+r?TtPvPY}A zf71=fO0smCGZUn$5hr7L47^R-9=6p0TiH@t1UeyFOE?-alt`6;sz(OWS?=T&Hj3uq zjpb(E4mE40A%w}6x*r|9ZOdFLV+{upe#&Z_$vzUHnzpai1b6|Ics6s0griM@(nc^| zN>F^IoFnd6?U7`nGMjV>Wl{SzMn8&#%BL9jI6K?=$O{Q#T4CL2^cDkYfd9GkHoq7H zpPcR^?FPug4_P_Lw54=C_pS6yulWh1^wx@@i1*2eDi^*(Zr7dUWx;!Zuvx0II&QX`+YQ86WFT*^>U;jz`LU}kDqEJ#e+E0`N!3H2 z9ozorAZp(l*(cGIpq+gY>RMvd2$Q`4j-V3D3hWKIc|cjaq@Sma@!uNFcM&o@*t>P^ zfQ}}^=wmIIY{{xQUKTO7WDu*!M>NatA)N#y3dDY$W2J+AU#Pw&vbmJYmTu=zg0|al zh+Zw3ZxKggMo*#N3Jwd3GF&km`e#RKaF)8Q>h}0Z-NLWGmJ}hi>D)b{A34yTMA{B{ zW$FLukYCQB!YCt#Fz%Gk*mx~nF6sB+-~CIbU0O}$Y*y2|IH>zj(xPtfYB6i=(v}a7 zlo3Sss>g;ZXRZ=88FXteSPCvzf^QHbjt7FBv3ohs z49<*3YK6Zt7OSU8*!pR%0jeWJKDw8xZS47b&R~xU17va_j<$l3j#!vey$Od@1--bemi(8rB76;>4qDvUyBH=ZGV1oVL9sx3ciM9ufS34!GPWpl5%Nd@$o#PY0naa}I`XA$btmdI1Jd>vGC zh>J^6wP!>Hu@Gk$XxuwbBn6M8ZQKjmj^t%q1v%Su=Pn4i`Z@`P0u=fPXB_@YA1HBW z)2jN0?N^Ef(e$9uyyfe7%m6T)PVj$*iG598LsQt$kW>f!u>6YfZj>C5xcf)LE^;XT z#;;NaJ@4(IpfNr9o7GW{a_wbJ{X_z%yfgEy8_|$=! zFgO@y0)5v#DI`JI*#B=}>fX<=#sxKB22r|6dR7B;g4o1hlxS=>RayUDp^#)QPm8v# z1tb)1B^hn}KFN4hLGJMC=L0Gei4{zhrSC@mNSou@8*N}Vp>YT@v==Is$PwuMeiA(& z6BZ+|Ym!~|h1aWVqsreIuLGvPX7>NnpuB=!$zU;Vp(x=5oz^+Lwo{c#E`p8ezJR!o z@4&Ns1IbzM51kueH_!(4Sv&AoCrhza&9{4IQ;cwA;IwUNA$FFJziFUqF&OILBK|bN!jlv9QVKSy=c#GrzYeqd?RNmA=Q7&lyl(JI6 zooV-o(C}Qf8A`3=eC2hUVo$k&Y%_tbKKX&0`MpbYhGXwGbLK6d=>N#EA z6Nr7S$QH${HgwO;!nkg#gonRf@9#1WswIU&gY+>IiNOl=qIK~=gTC-y^^PiIPk4XB z>dzj7Irnfgy3a~L9Zf@TjBV`n^;NH$G~9Ifm}J-o%TO(+xLjbTp^>-+2yFc>H|Af# zM1^(FySU5oNVY)~+b@z4gI=n>h)X?$!QhS*+prPEoPN22Z3prse_YZMAKJ_l0X zfXD{6punzwY0;#55TI@DN=2-!f!x}E8F5nTk5d4KP%`E9xs}p{pPo=jbML42Zle~_ ztE$CdkVir-#uOTI#LCo+*TbfqJ8uU5M+?i+p1zzA+emp!LU4)EmL!C-W_eHUV=(sr zKa3I@!XOEMyQkDrPuZ&nM-%yflG+AIW+DT?h zE1OtwDfyLN+qA^O<}-pQWEMJ%l-_nEn#mwaFM8AD z)E@s?5`8-9mI(_8WmHv==@r?P>F9WtAqpnPG+h0r=+r=MeBs{7A1@3b*zEwDc)@Lg zk&KqHB)~%Q(|A2NB3HpmBpH!ptqYJ9~84sNrbg~=GtPQluhwvU3if8u~{XI@H)cbYT z`Cc+#Jl?GvL^Z1Co9GPB3ffpN3pyd$$mUL)|7GecPnVkL((bVhdqmWkBp}Jj(P}zl zQl7?s2O&VBA-2ORvV0lmquf}M^@>O!Cqw8utp!YaUMHf|-g3$Yfpki0ebG z<23(YfHZj*2r8gj%Y^nj+Vh>ypNCE$+R=v@DOzIxa!=XPN4l)_~GBT@^6%-RS}5Grr%q8 z|BV;Vrl2-G*-Y&F+9^y|ckIvT*hxXSqI5zM?|B1%05WslgcY2~o^@VP84%vR`;w{W zp0D;88(}fvl+(X<;cW*N0c<3>lPLNfe--KF&g@6O5ADKw2A#gO5RmBmvw+!)RDY>Q z=5NZaQv6K|mxKLKea-r@qU))a6-eeeR>zMpsr8rRM1H}2b@xo|X=d_;?ikF>R;5lgR2AF4 zt{oip%Y&%Zd*dsLPTV(<{l;)Qz#s=gn*GvY!iHwYS;LH#_ZnW`HgZ$W4-pc|eg$$R z-;6TmoQ|`Qp&G}5`7C?u7Aka!ir|5q(lt44Li|d@%Aaq!i@k{3Bqf(1)Nm&`fR6hj zI4V_mj|%f50(aBLHR*iP@z0Ww#a+08Gdl?=4}FDeeD~sSbtm?W!mOR#GC=&v^!G_! zybpBVU}r7Fpp}aaea%VDUV5tORh}^`Q&tPfxwl+fXi$aVQy%tur&;LKBlJy}IKLG}cGKfgE@)QTj_@!RR-2Jt{CYU|~pyq7u$jVNr z8_eGtjz{f5Cpu7(K0PF<5ttLdg;=uQaz3Ta29-E60GfyV%JIqw09C4C9hU>0ML%`QQiBi8diG8!_;dL#U1?SQvG8L10 z;WqFj=B#t0j23VXZ*`l!;V^~C^-Ik8aUD+5TrbPs84gQ{0zrZRTd1-mGM&Od_zhW+fS1<5#$RJa!d+9R4>-)i9zM@Y%V+f6+ zZ*i95qaIHwaU@#_zZVD$smg5o(p?i2${^`BQbf)5$umQ0#(7+|q9fNApw2&*g4 z2LVt26A`Ae_L83k8?8{+xNzZbA9dvkCh#XXMiW87yTp@%IIi)xKK zur{rGwzq5ACbgLIuBs4zIT`H3`$QMHB6aVxI9bsMEcC(Heojbz9B?F@MuAj_AWSOQ z+7gd*Ab^}+TcQvoN4xf+JU@TiqM|xoGfjMl#W{jqaKca1mwQ9Hwf(u<55d!pfHmsF zfh#A1vo?hQt2u8N8uq!DD}p9gnF^)U|7I9blJIxL zfBgo1GRYa{n{DPzI^t&rR(=fLX@m?07ZNU{3- zWT8edVl=q8{)whvTM&>~+bStFxq<#l)d6GE-TNrB6j=tmvQ){aG zLxzCctdMKa)~{oJlhPi`KEk13FW(^6(1MgI;i4aJl1F-?9&yYhd*+c@7&VzU|d$bj#~vLrLf zc5|o|nBI0@DZ@Zm3z@ipE1QN*;}l59hKU!uKt=3wKYd{dK#BPzKovLPbfM6U? z3|Z_keFFs~Pi@GnLBJ?xzmyI%$H=TSdx|lzs$PQMi5+WY`H>28`xqqO4Cgw5G^vif zrYpn%(pc+S!&tH|m?-l&98|_QGK|Z0uC0is|DD7o#m@% zNrg2VN7*u=sHm&@I>QDfmBTS&*+l4k)%#>R}>s@7& zZeL1{wQ>cGS-289_rbx%8`}hdcF-AFT~0xM7vezZ1@YNn9hQ^QZqr?j{PyGyt>^I2 zinV#Ms0Wb(GX~M(Gl!kEI)SjPA z&?fGn{4(%3fE-+s@KFDQNu+OH_{VeIv0OvJ5E* zb6}Y=66keRG*JZzQor9~0$_(LN6tAhpEI(wHKrbwr`@yf7OybDBn#nhmkV;&t>U4b zMWt?KEGJXN^XcH2#!5}}$cV5SK|7XvY*Ab#V>7yArC3rOVMz9w`6#n@zO3Nfv$C0& zGYk-aUqR*I(wdj%ozGmZsE6RHy{h%7$pN(Af71Nod)`PFj!79D^nQZ12LLc&Ypnev zojtSh;ruj!7wuPuvu~`#aZ2C-gS0t}HcTx)yrf~hxoPv!%3`f6%{9 zk{FH;twY!v5~b%tvsH%6{*c?(iK`0K?Bfkke1WUOIeF|I^gVN&A}t;boqC6^cqWjLJqWQZ)j8~&E>YQHE_m@!|#bO|zm#WyKPN9^em zsxjbb$m)0X5j3bo_c}bk-g<|(7{rZeqZc- zO=3*Yf`1^!RQy8-qK!Itm48V%{U_0>q>T$RstY2NeA~}%9L|^mfM^sm7Nb8ducEo& z@ag@@$1l6C6^Yf&uU3hShUX#OM5Us+1y|8?Ca44w8!{W!+V9A3VZyKSSxGHJXZ;UK zHdJgOI@BQ7uJCn^)_egA3ICiA%+B8iY#%O1g&}g8cT1m^l~IHQhKWEmq)1i(IK!&K z`tP*mgc-kRPq{rw1Sb0Pg0P(AN`;i-_4PWJzi9)t0CTmQ@G z(XWSrsj1ifdV{}Fe+zze(y@qPlt2y@ddo9cq|NwjRj$asJ~2Yw za0VIKRj@?ipu($mVQHuqhnxnI@CDmgwEmDz@bv^`?e{dc9&@k`ZjHH^z|2?tkzI<> zH7k|6Ep(zZ&B>w(WUr%YQMyC}Am|N>1pu#I>7$?MU zLoeP6F;*al89$yEIR`Ex;8a-&HPmpDK1OD{(Xg3kEG65EM?l}k`3^br49_~gRxH3- zW>ogrb-O^3NHNE0GipMAUzP#$#e?WFzyZ+oTrMF9H*)=1cxz=_0^xiE+DpR=1`?x&w&3D4c4SJ8eI5m z7lkxGc`nY^x2}`3ig20)5vPHm>TgrUIz2|rh*4lbj5bzib6A5k6whJcn-6nrB4KG* z9j1Bsh=rmgqpV=;vG}&;1TEAU*QdWzU*F!uUcw#kVoOVf(8O!P>j2(8wAf_(YK1B! zSU(h+BO)@ux?H+eJ19}nsuoTwqq%#W$8(;Nje7<08!^^hQ^F|it+7{5st7_onznq( z!rH|JCs+nFseKohsAkdZp<2?^dLU>ELHD`k2(nzeJ5?{gw@dVjMQPW(`lKZaKE_2u zhbWaw#sNskXSG(Mu~) zweE+IB+B1d^_NI>l1UW{>H+cBieK6V6EB6f+WeAKg->NOg~SAXGe@fK6{KGxber|W zz7cg8%DYFmePl5tl%Vi}^au-SP*JLcmij+c^EpPyj?e&h)phH-w0UYi?c1*Ah{`lO zSipeuv4*We{$oWm>O&t_OKRB_CjIKR(dIIF8_J>fTtr&t<1t@PRAPozYupq6cFwIh*x|8ct|nJP zz@9@VjjgFLDM388OXAAal#$QR8pjC7Z3pfC3Vq=m9f;<&J3l;Q?m-}iQ$pfsDQS-^ z;9lX56c<2EVfLxvZkGBfzw5Y5x68A0nk7?0bh|MdLF%YwTq)jydZ<6NTAB#AJk^XS zpy5psFNYBCBR`rh3WeR55QJ+QJ-F_5er!k_UPFWa46@kAC}ZsTGkm&2;w$}0MLz5V zyje+G*kYg3Zj5gJVAH@`3w$+^vcl$WX_J;BsvULg-v0ywqpOR@yE6N`l zoeyj{nEWjS&Dr{?+YQTbj#@%(h{+rdH|X7@23WE5{_FmU({}uA#?ewT2p~nJzeT`h zdx28bu`Wq=m#hZc&7&bS`#~zM!%`=Pw{1U!%XtDaO&m^;BqmFIw&RoL9Q!fj@6 z>2su7LfWR(*s+iA%~>Y*0~>OA;suLyI`wlByk+5UYmRe1^?5n*2XoIsBKb)bal9L! zhWHEOUR)9yy-e`4g^wAk-z@Nc`*?zH{b(%h85U3Fd{gwU39Hc%m7=5HozZ5Ee&z{( zMnz62uS;~#>I<`CQVk}8jT*6&Ry|-a zNDzx-@3&j~*noawMGp$rD;lG&yMq%}C*#cusCF>8=zumKs!`x+qWRj3u@UUP; zwM(hnMgnYaz-N154w4eD(&b;3U%|(`)IRWLo>Sr8`E#bVp%;O4`m&U!M%P()sNjdG zsjWIJ)%Ok7%ALdor+|g7u5St+sw*-rw-h=O8|Z8_OEEGdf271T&!+}TgTdY5QC<6} zLH-M&jUfD8c^FT*x9dPR0@as}RBr`*jZ16$E6#>p_oH*H4wKmXz8fQgnG+krk3t_R z6JmyDz`KZTD2gn)Dvelh9)fWu6e#{HO#osR^E*C`?CkE!JOa{}C99y<<6QMmCM?bi z<50T29s0%T13C$SeL@@l@kQr!A>!}4`<3OGa8ACDSK^le%L#wzBNbqqb8%`3>+!8_ z)9IY(GisoW5S8Peo*D{BmMAahB)1GSXr$+ZJ7p9@W*Dq->cc48u*|$w>B@FQfi;YZ z3$Yd{9LwvG8JQo^6bMvW0b-IlN=#NAly~{2p5C~31-&<|s9CqXxsLX;?rTGsy1^gC zVc_w4m9=1F3vtB-(~;pez_HphU@4|;DZnoBj+hj&0n)i#=)>vZ7YOzo`}T<*GM#P& zx;C{T$dSGk>^1e2t~`FR+WR+1`pdJN`Y-wdPU@&*25!6#Kb$!Txz~O+tjf?4GY#hZ ze-P-gtW7!D6~IZaHTac81r~6q_{WV%5`V@w-{FZWO8W8?l-YPhL`9ssKBn?;8q}(9 z)Cw3N+VHJMTl=b656IWe*3$+iSpQ?h!PdtQ99&3G(W#Z;T3{T<+^G#)p5yUy3RH1t zxWo!_-73(P4{{{2B$(*J9S2m#8O&ekD$*9zi2}>>hC6QqU!1w|sG3+PU1nux3)VG^ znHnk=m^P`?*L*d%fNOxw`D_83JT+?Sjwn?sv1EudWihZOd!EsqOa#(y&GDJ!RnNI@ zHlQDw?gGD#-FNPZha#}`YyKKn8EPmixeT}Jg)EDekrjJaZH_v3Gxh`P#B2ThMk|=y zdV`lmdj8BPZfQNQ%iAkuJe1f1f#T+%V}5^m>;|b?(IW1X#E^lgtz@GhqjJetMox0mGIZJdiQlmmb8w7s`uO+w;@*Z{>r2wX7tdeR?5K zY-^Eyf{#lObSn>(MRGk=8h%Z4yo4ah`8A}F;7(T19p+A>r0#v(G6G`Poy(oC3x~IA zQlvq^TCZ?h_l$0q`bZk#L`2<1jC04d2@PaMh~J(b+#Tz4a?H)8BnTIimlrTJ^Sm^+ zq6u1}tpFXepM>wZ&EXOBXkVA%_KUE&DOcy4~8BRarjEAXs; zbSJHtkgN4200ir{R!hC@wkZJukCJljTv>@gw3+@LB}AFq9wutan(t~*|824^R2Rhc zYH=v;uS&5m;~sOq(*QsKI4tFq!Mjc`Y8xaZjLu?|EL$qXJW86+;mF3jA40jMP9tpF z$J|}o9R+Plz(jSo!7Ku$0z>D#V&7J{12q{^wto^Tl3rx^&=H+PMIiF9{run2?snC_@~@3suy_?f8H`_Ad?S{TBFuk*3!v7-QWuj*`HP=1xZrN#G!YC8jnH{lSV>(4%Ym%Tf;Vxvji7VFx8TPeu`9s=HZ+ zRZ)kJA)3(0g0V!-K$bYhzWJmKs-F(laoYf68F;E#uNel^aWSA!%a&N~C`fX^`q z*E)E?TvSQqNEnU0URg#6W<-E|wkwu<3HlxXyFCA3&9-82GzJ8bV}b}Kp8bX(N&g+v z=v~EWy)*S=@Y*_wRN%uv?xh928=ap7Vc&!`;Ww4hDY-VO^3tjm-83HQ{jH(J7drPJ z*@wYz9=+`P6Se?i{pvXoLapuy=J3Nhk0bwD`Dq9U#jnVnh3g1FMiVm+!Ap90bph+l zwPs@*L?j(aKT{RaZ}HShy|q#vy{dRc70Z0K@s2^V!E1w?>Nw&)LOLsGDjq$2X`6Cs z0*02v0H#TI9JyPf*l}Q=&@t{qwgxV;7s&TKZx@tc0wieobK)g&nGAXT{xyYE;UXCL zr8(r&9%i;vu&jX`wD0XfZ*A3OF!lkG0s6>9iLB0MX0@jtD0--j*M~ITl}D1(qr%6+ z=HR>X=RS65^^J*wgkGqIpZ%VX!{O(!?7@aiFa4Q{(gTjLX8R_e!9dTY@rR0bmbFFP zmoFGS`7o3er>o?1*RSUeX0#JoomKLkRK~(K~ z9yX{vPWXf6|1gVK^}Zh4O5_uPYg9=r=4aG9D#0S&!CkP%t~1R~Wsgd5n; z#f^@8eemDBp?*Mifyky7IKhHb+~B?Z7RSr$_#J(5X<;KGco7uHmVRag zI;M&Ec#?;uJ#45?-IR$T*0m{bm_fsW8t8TYucZP>qzKV8@~vn;qW@2egIlSsSh-xh z=cNyiDxC%gXzM~wX(y7td?BR%!ZaJ#bOnhYR#1h84ln@O6Fv=v=>nrD^Zfr+e z%G0pZKu`MH9pld=iFaAiG$MU)Jb}-q#H|a{G@ssp@;ryq!4Am3XTD^GVbZ!-4FnDJ z5RniStq>v?n?zH&GeDc;&C{eA0CZk4t816-r!{VIk31rd=XGwc4q1Zo0B^BIHNOQ8UMC-3Xb>kl&nCZ|gmgAxVfKM_|a zX)YHI+kiSJZh>ekton>OG)~)K-+v7`GhFI-mV*nty!$M0vUdn1K4@@socy><|j*g&m$-pj;dd2>%y8 zpqu@<($82Rt+@X6Zr}XOT8yq*6(nx zX>mw_`+`sErdGE*`+40TTefS%Kt&T52CG>`7GMSVoBRfGS!bF&)C8#N*{+{S9^h<< z9YSwc#NM{pvXA`C=yY(_E@h7f<^)=l>Zyu!(jA157|8XL|MIVKuR@XO$ipNq7N)}> zoTglkbG1T?Fh{G6n_gLb9YhQbU2U1+>h|reN$nRdG<>%{Vr`%TbBsTi)>pR&G#wd7 z!3VKS^NbeLzH8ej8LJOTg&Ejyb;rE=3(}_~*gc5TyUjDccs0tToe5(WuOalctOi%& z6~ZrRWXF>B@%??&GSpJO)0=r1ZY`wfF?qp4d*}k~Z25GirzTpf=$=rK-G9TK2NHOk zqaCYWx6Vz;2srI8PopJ&koa6y=HHT3Lv1YjVK4PWCxGUzm>5OC8JS@#gXC>MXIFhz z_GokVSz2mF;PenRAv?59$7f#&WaFB8ww5vHb?MVpnO0dnd4a5bTiT+JhYw6-alCt` z!KzIWi;%~?1jcE$iLpXggxaX5ND6lIb=usEljr2)?HUR_dAYPp+Sl2Y9Lw-~lT7m?)%=;U2tt;++%P zAzaCuCcI5kg7JvS?av7D=gxLEwmVBHE`wt^A%?oBj`IJ!SL?ltpJtY1jZoI-4WV$Y z4Z>lcn@{vH&#lVet!eyG#ehFshb4O8oq2P4h7ZO-#1hpmPWODJPWjW+S&FYJLcBs@ zGeFl`hH*BLdH?6Q>U}DvgD_}{kxUvYu51to3N<*-b7`Ur?C2>P3SF+gQmZBxrGQDm zn}UtN+BtatojB{|xO;ciyT;R|a$?tZ%eukdJ9Q;=3ld){$Jf9(;wsX()d2OQ+vifM zvMk6Gw|FC2%uGb8bMcaMJ_CUOc4F!cHV)fwa?v)!3A-LUZ!CD9;H2v*@lEy0cCwv8 zcoXS~R|D5Y$ZBxxPBT*P0;{4`fpG!5ju%9%E_XNZRG3ruRBVKlqme$VFp{)O9`#yL!?U6cj`~12XTPCB9yX`R7iy( zH=Qg@9f-lX=kT11Myo@@%9d04^ul_TRl{sO`%6NY^hy?VUpFwH+>)C zq}%cQ!)rH&@7j{_X!v-z5g#XJ(B+ng|3JYoZ++O)^-)NDqJo5)8dP`RdA@{C#wu*+k#!F z33HY_khz8)Z3@ef)l%*rL^XY$R5Qf>xL|rl2C$~~qI4df7m|!!2jTL}%L^F<5ZR0n z6~Vrp0}_4(0;r}izVMMoW0u7papy0ZbA!)?rM`12iaX z{AhL^;*8=_N$8miu)N%cm3an^;bN5Qh+0<=J9G|%Thiy?QkVKYlGP|w1RUwk;YbfL zhUW!172b;ZUepEwchWArxO>te-Zqa70*H(>GjCc=^Bo>4^5n&Z(Aap7uCoh3ReN!P zpvCQENevrOqBFqyk_Gk8kwTpp5~xZ8;YyItwKMcYT0d(EpDq|L(G2?Njcxn_wc=Ks zydMhs7Ugj07BeK%#h2YOte7)WF`!~xO)W6tX+|4(XuWu zES{-U4a(`fm=Gkpsc9zdh3D;Gik{nTVc|+Xwn-{I5#ytd(MkPah0<4;JA(o*C{v9>|YJzj~ zs@*O$|J0>RGU9r%MdzJZM-jn{VeEz?CqHPZV6(COG-5&Q>bWAqpgVTWk0Qepy{bLu zs}1yGK2-CCtGQW2Hz<0GBOv+PVls_vdOW!W%vL9kmc@-FOTz^hUqo|K+P= zDM>-6?6<;W%J)B3Lb1;|`NsohIgjygqs<3y^K6Sh_i-O2-t5B1O2y^SO-WtTWam;+ zEdUWV+ch1|-ggWSyGI=47PyWtfPs>)Zh>e`)DiVD&r(${=inGr__S?e2f2s9B<5Ja ziOmMT#N$5~?^ZP}u(r6JoswP9k2wo4HU**iqlWxJK`h;J#_MV5TpIuodff>D{9!|#6ABRLaq(tr`h1httV7zuYktdU)O4usOV(*y?B}QOD1V}Lja@SWOddn z`K2q|%_mHyx#mGrofi{%t){)aq3az*WE3ISnd>>#7kSO~zUf`PhmkCKaRyfK3PI`J zQ-qd+t@gIu?Su`O-Z7Wpq6)CyQI0w_gyzs@<`{5>b>rrNhMS!n8*%Ua#Om!>PaLZL zO27IRE7rEY9rD?J*s4gN4`pznUkG9m2+7 z9!a|L>I-M~K_ThSi1i~f1T+;`l#NqF8)`j5nG%+UVTW)jW495V8cKf*y8(-4dY3y25X7QC#kV ztEA?|w0g@4y!)3kb;;Bn+*0s@4+}-_!v9??-Aizw@1WE_ijLDUPlIJwB+N%%SHlqM zz=Wauk7`idjhY@IOI8KK&7G~gtXPTXP!teEb}gMc=-{l}YOw@0v=f`uC9u1=j`6Pw zD(XO8DTIu;1^iHCmGuxxFCStKYAX_;5;$XI&ft#d{ z^V!W2!aDd8;Nu7hgtyy@27+Qi2SSIhZkXyq%1F#F`TT0=#U1BGT?jJ2d5_GwZ01_4 zYt(@Nhl+yqi5321N9?wJR_a>$ub0^^VXdM-5G`6i1#dOFx86ereqWG9hOep z8uH5e)}FS>SAl}Ka0I`iU0nKa%qeC|Q=;N1BrYvbL$#n;bgvy?!j!G>fv zUnuX7W3K70>q8mqd?-)1B-_vx=o{?6T#ydc>*eTRqL(@#eD20qpHp)vDnc*kqmYR% z*ugMrZafO&v~Ae_!nY*x&sQ*by&haLID-rt`b7}^lw|+GOH`AxXobzOoIxSo%_gj4 zVk%@t^`6;RY`zBMTthQFTUw7Z?S{+?@^E(k{tSVy2u9W=(s1!sK4v8C`)77b;S#J2SG~}M= zlxKl^WKVgSH&nysSK<$1}e%&HH8=zxu|bpAGCg|%49--nb3owW8Q9;IOhAw0W^tk!r;PD@@<47N?|W?wjc*4j-hkma z@@d+Wx3D$D(!$KJB&n)M1|QY}>zPksbtSh6)aW)e-wb!3obl5fTnU{r++pUXo8zR* zcFP%m9Bq*RuPEirQPHm{5d$!0^*aaMIu`($fv11UCWX2@-%(>M}dq+{eWQ=M`$|DQrr7H5dh1oA_wK z1o#VG_ZXCeycx1`T#!{aDt_)@UtJl|>K^=|I80-l@UPLa9$}gRi*w!kNXzw6f;xZX zhu`1T1z$_VXS80$Y`%@;vHqUVe@4w!`)vy@b zBQ)k>IgIO&Tk4#Uaq@ZJexRyQzmq%0Lr6bhvIasgS(_f;m)W{Uvu5HhjFYVnd{*54 z666`PNX*;~ZT{Sj0S5tP8}88F{enuQ8n9iIcW<8 ztld+x|8k#K{~c-BB<>IdTe4}fj>pirGGI%R2WpswiQd?PFTWGfkoV)^N(!{9>AW|x zTvVRo(2 z1=?5+;BR@1IOSRx<;m@8aQChD$Vc0Rxy2uwnxjAx?A-hkxPfXwOQPMo1uyZFCMO*>rJFyi0j8 zNSpxZ4!@Pn5CkO5CvQ`U)^HCG8H*>DYx2^ius`RSLfKmIaA@WXz5DAv!KB0JljF;A ztopi&g4f&Bl%MG;WsV0ST=(R&a7FskJL5g5)qF{ws6M?4&z0M(>eg+VVLK|oD#0`P zLrTyloOJ@%Gu+;KdrWfLt>u>_=|M`r9fo(nbcy`6fI*K>C>#qpd)l~zO4=`i3j=_p z;^v0)Ur*Rq&H{5$LK-TE)}@suB|n@QiFmrE*dsCCq3&Uz;rx?naPW=iw&E{dAY&Il z*Vc$#2^l8a_`+X)sYa8WwU#r_6>3$D_f7x`NovDam&*hHqI5!0{VIWWmE7?Myr{nw zNVCqr;>iNFqOed1kf#7cK)k=K=R|L}CppSfPwsAlvC>0cyh`xdN6r`)QH}bq*)vgW zO|KhwyNr)Zs3M^@ma!VI5U;in3EwYO==mFc$j3003MXIZ`k(1m={h!V?K2fqOy1?> zkhSqja=|~d5~3C-?Lw~#qU2r*{h)h(ts=!(60xT5I#hxe;FGaKn zY1&&-H=PgfEyyU5_;16+B`Q`#%q$VtMLdjX(Vn^teXd#qY0KtyVXBVdbBv4fNAkcawB^o^FB3hB|%==mg}}(?8F6T znjrfB6jue(?{YDyP92+uyL(JzqF_mW*F|9#bz1f;>FM(Zxi<$tWS5EABa0=K%Vv{U zE|m`>ncqx}S1%=kic!hqpdh~KLq@@`4BghA4>rFW$YssdF$|uLtZ%6O3dj#-MOL?Q zYLJ<|__2wPQ>AnvJ7$}xKjqZ%`3!b_KV7Vo_R=B9#};n1u>vQy@WHI)+w94ST2Myo z9(Y6KIU8Tdnnqz$KW;>%sN26Obj%D!^WnJW?IX3XFA;t8^gcGG*7D+2^Fy?I-x+Z2 z(xbqQ7f1UBLErL?Hjl($KC?zW$t7XE%Vk1?BB{1@3ldTDm23~{g7x!x&hT24Ctp9~k1(MzKC9GFy--3w1vK-riz+*HMTZ`N zgmI#u&deB#EbaZ<(2YFDL6Tl#8(wRA23E9=i5uK;j%WgVz|V;VjvB z>`*#L2G+RO#KOG1CxQRx3$H;D`Zu=o<&qpCOB+ARLT{0osTG>^JBM;H{ze1>mto4s zr>qRxxg%8{A2S&ye>GJ4N9Y$+Q7;qo*F?JqBvRrSd+l(X;hg3d_9O<+U4kg#eL7P@ zfW62j)W8S6oz6b4=%&s7gf4Q)`^Q!1XfsRyP{Lf1Rdi#I(Ze3)pm_R!j=NLxy)~yRUgoygX;+9l@_f*6N=#qPw8`ehoG@#Aws)`fBCJ$> zg#6dLr0)8`xlCmbM+)VUrRglz0HFLP0e=uTF&8QvI2SGY8n=B3nx8FS*sLJFh{8(S z%6(^9%XtE}90O>6I;)2P#^6uo`qSr#eD~aR)A@~f2>E})q>@-S$Pby&>DtoKAl$1^ zsVk|n?APp&6`d_>oVl)_9Y#d{d( z#Wt{9dzhS>JsA;m4H-tDi+n|)prpa65{-Nv;Gy@-@&cW}&hM=#-{(*r$3Zzq;O+gLR;WF5(mpLh4j4EyaP>>G3)v}9;$0) z$h);{G#lp~ZoN~R^ti2*Ia;Vb1q;!gb`-cib5GFWA3Rr*tGJunuA{?p#FudY2w>-8 zS@J0jt~Y+V+W{0F(;sUQ>lUxxuK^u3gcs^kU_VIn*<%Kk z;I$^-NY!Ep+I=3$ctwTWgHQhoo`% zP4-?seK|hPyHHz)Z>y3%RdHTD|u}K48bDLu-wEQQ0w!62A(&LuY#sE5-9!z z=nr%1k(lkr1K7T%b6{_#qrorqZ(Z#P?vP_zM6g1P0hqLq>?)|g!3Yyn?%R7@6e6po zD~hsCpMuR-V?C&ogt*3 z2*!Ex>qetK1pm%0lLADQggk>-AUB@GbC?+$geHN@GMH2q}oO! ze67SjpFf}3n91yHO0YWxn~iV}@rhIftLmrOl1x_tcbeO7@fhIg=IGHc#6_}?BoLgF zb!A?YTI%csO2B#P3q3gm2Lz8(;T^lC^xw!DH;1%F*?N3Amo8{83>YHOpKBA*I6g#J zG47GctVqvy(S3=lpt{aH%MIpsb+A~}vBBYFQl;fTet$H3st3zM_}f-4LGUkH;#h$I z4?FA_=Yd^3--zNS|8(aA*9l6())rc9B#Ku%6$z)_4`TeP!*b&i5!hZE(nyy8(;YMv zlbAm+U!6A;>@XFjNUgta0<18)E|z|=fKWFlWWb7U(MM1{Ith{x(iR*2?Q)`6|4F0| zrvs-;xY}malVstGspMR{K6uw`#D&*e$raL-c58+Kk2K&YtKCG-g5|SIxv!KPA-h|> z0g8wvuiI10GVs7zp0~oWH;vggwN&xh*YqQGEPzZ!;c5PAR)uA&9Ek2B;0Kx$FF$66 z<#`o?oPe9%to%1ytcRLB_NGGN~h5ZT}0E^E#}w?ETALsj%PJ(qr-peR z=Ciu;enPxQ{puKs7I>aCkYgRlW$?&eyO&B}xpS)4&)`x30qn?{M+&{j9s$Cl>=qOA z2}?qdPYJz7Jifq-+C$OTn*t15ei0J2mqrllym{v^|ME2%%W7Ogya}N{TNoad3M5R|W`Bv={ zIq)5w{&ZQjTG|^TN8Yg7m8H1fP&-M9Lt9fjo^R00=pa$IbOfeRqhWj>HSh*N2$_wT zSCI)4+7dit7%qt`h(26Ci$v$C=GR^qd=KS{vb2acKQFn`mfWDUv=-on53Ky_9XVo=Q-u_q z2li$Bu-}pnRq+EU?fPK~uT)U>MM>{#^B(EIzfdAmWs6w~m+#$pfwDHySKc5#bE0)} z!I+j55L}t>F7ugGSeqi?%v7Wm16g2M9uvNQ=HSv^%|gs%JNhNC%7*Q5*-ODmk4XnZ zoXhbDIS=tSOkpTMOr(rA<5q+-2$8D|dj8fLRnlqUS%W0;6W39zz+}yI0q(pyh>6zk zqh1NvtL`7Z#~F4Sn2*+)GZM<}MK#lTrQQsEvhZX3E_|w-l+OO^yj!leH6}gKm1^K|9nOd8HQ_p2@+&W9iNzwL3C#+XD1!7b8ap)WsxQdCD(dyPcOrE$#-K* zlkJN&aXkTT%h8VL6?A5zzy{Yx=lvFjQe}6xc#9~6*+nMJ<6R~;QR^jxWG`?TE&f}p zJjbQrf(dO9D$mswIcv~ep)Q!`2hRF8Ntr)YazUCATCAg@g-$JI;*y;n0aD(ht#Me1 z=XkPj^W2K{(QDA_cc=Y&jx#+u0EVj)(v3$;OR)281KCnaaks)*D)AFMj$L=Gur2Zw zVcm7#qm$&J;A+_&`Y#bwWF%{Lid=$~xj6y~nn_so9iU@YZbTXs#pSUatTlEA!KRIQ z$rR(YK8eE2+apX8P|OEsN0I(nOZ0pzWTjIjoFKlXVm6g(`5gEfsDu?pHJHXqCP9Fn z`{H7{I?T?C-&Bw`W9o#_8bt@`!45DyhK2esM+v0*xrYf0igTretN|Q}Ea}tsHWv+@ zi%(Xh61qL;nWc+M*GJschNq+tAw{{Z$FSV!Q$o+vlrF-|04i{%hRT0#tm>_t0d`OJ zfl?pm_(EW@j18_ets!3RqMq0qrarJ53NPoX{K1v8;dJ`ja`UUwP2+n-;bgmcU>cjpHsH- z4WL$s5r0HQ4Rn4`f<~J#^yM+kc*G=5J^@xLE2*NkpLpg32!^k}rYqdv;DOD}OSE zsu?P>AI1_ z%m1>(#cw~$#p__3TwVkZECZWclTDS~`?^qWb_4gU_I0hOD;sY^1Q})v2t;of#kx&d zb)*(9EWGiDD<=XSHC6NzB0ScUxn2nMA)TPbu`YA!G@0C~+YU7WXs0&P49jD|A)LRZ zGAcZDpe&g1$k1C>+DGM$O6B2J4f1)mQ=xNd-MCQb)n+W)n+LZLxd2Lb4MrO&4hB12 z>8fs~6fnA>3sNyPjyStFV>yl04*wQG_x4 z>+Fi&ia^^tiWqWQ9T;sk%2RsojKz$b^t}A^*VsMc}REN6c z9P+t7A;d<80BGii%qN^dCRq0qal~HstA-iudyg;@5LYF;;I6zy_$a_eBgam=%CQpL zRAD$#*lFxnuOpLB1W@Tz?4sI&?m4 zVu8-gTyphWN10+iNX+{T&A}*(YCwjAO=$W)DfNB{r8ScY@X|~mLMntM#ZioO$?RbK*qW-2Y5V6FGq#@RLDiFv~T&A9VEHWX(wvOq#;qX*SqHHaP6$%WP<_t!l+ zL|y`~NV>brPiPq_QY9;bBE767;i_j(PMd+|Q8S^|)~l*BFl%Bzt}tmnJcGe+Yqr8y z#Li}Rr-Q&f4%al;1s7UH9AW*e#s}<_QWly&WUYkay~$?Dkj-{7z86E_$Jwh==W2_r z5Cemc1gr{-LPU^!7uu${Y*NJcBRuU@`CN?TAbBrhjhF#@d7ZB+WxHpFh28vg`P1ky zQY?*=c)B|jBj6NQd{V_*B=|7FeKmKVJ;sE%Wlc%-yY_1WtcV9hyk2nOpBIsosmzD+ zaxHOaBM445;!5fFu;)#?HcyU%r^OqM+24$IS`U~Uc-vicFeN0 zH})M+E)%ihyVh5E@v1=@Hz2%Er7XdjRW8FG7R9OExHk7(0Uvjv`U1T_!`=?SvVRbx zmw_6a^BG;#nM}QD@g}}lFmZk~%Cj!#BNyWRe1aA{e{E(tmR>UnknwD;rmGEdKUm9V zGh`A)OzDTi9LLdGFCowsRTOIcOr~5qMzK1!g(Zv_&n#Kdj3{`1wdtSj7)9Qe)4maCwkxff4FtJw z=K|LiAUiGU^sH#T-zDeu%hJQ^Tsy;er>0o_E#T{!3+iaWRY#*#I%!@I8ymCjmE5BN z3pLwj*(6Hnzn&MvO~!cl%5Hr$qn~TO`~$<3Zu8ALsLJqhd z1FV@}VUJw2-Zt@vj%Mx(fe=n9ZP2FKyjaZz#8o;xPv-^hhf$MkM21yXbC3>3oisF6 zyepdUD>Q+@k}4Fm(VZFXAfOIS(O^8vIp&bqX2#fT(<*`z>GY<5YdVEJ4tWoQV`r)J zo;Q?Cj^5!P9P)HtmBVgfxy;9<`X~lZ)u=ThvQegN%w4x)%%wHV&dKo1N_C}}V$>G9 z9f8Qzzt)wWY;29|nuj3G+l^iHJBu+vHV$n;6Y2w?LJvQ@3be$OnsZClbO)2hL-8aT z#pax0L}aXMqD%^<)Kl@4%58||?eW%ofM*R`HOl+sE3P}OlhsXMkXt+FJcA>T)n--X zX)Yf5M>pfgJCbU~-qe>(G!R?Oe9^j$ZvoJ~DGB9%rFar&syH zRJV*u(%!VKG-^E~Z&vU#J#2bSig9{D)W?O@oV5!d?i)-H zMPOIj?I9SJMY%zwhkypW+_C+tQy7r1CL}n6k0su12*9L05ee z()JI5s*OC+crRHQpSdh-S~3X3q46zx({Pxq9OW?G=XeG<>8ICfH%h~1baRGXyK@g& zwL9R<1Uu<36B-#Ex?$3Zo+0=f@|GtsJn`e7e>ECP(CLE{(t&>?V{dz2{zq4Lo84?0 z{)ISPT^!l84EWdoZShHw;T}L6q?(gCN0E?`;BgN`LeJJ2Ej^|V;+oUt<>SuCYP=m; z#kTcCKhB^JwXqpU-Cy{bsw6MHiFM&xP*fF=(DaS~*4&X33q%a+OBL5;rq8OJL5 zgcUokXz);y$+b60;Ki=F>=eGnL6DHc`3aStVP>gavT*vVBt2Xz&9@Q&-h1p0CvG&7 zXT`ny-YqJs@WsZu+|K+=fIYGkAyM41FWbh$6hXRdA2Le4gQuYB{n`19OM{gik*uTT z3HSk)ODkaek(LxLUmx;9SMO0pYka*7&=oxz_l=b4aXwI~LO6e)e{you?RA8K8GYcH zT_AC2?UyT-kQ1>47}zZ~&*4<}>il~l6|*>&C0*i_lhthUqPbIJe%)q! zI0}Q)?)&QWK8MQLlwL~(CLm8B9VJCr)adh2)$IZ0n*5c)O~6L-m6%wgtGh-%sppum z>{e~177h?f3)+jGrL=j}08ZnWrQlqr0F^{`iUF^Q=Ge&W%#rwDAgGF|75ogk6NG2k6;n1-{*2;Go9qH8x$A zd%M@%`>F4qTbvQUtv!R#=}2^l;FAo!z^@#W?qMs>vI@Uk_k^zn=UM7N2l%~tDjg10 zmHkp>1VNJ3J0Vzm=Y-wiS-(Y};H5k%`dJd24ksfCvtI(`H8Fze87|GXi%JSh!b)0S zXi}aL)I%8mbc;#(k6{{ zul_uCN2&lN$|SSa-p(Q9rH-4vuSi4ysM`VlsqekKP@q+8NsZ{@y(MyY@RV1e99b7t zSHo#j#DUrHEJTW(|F;qbO_C5XP(Yv@4#@S9og%DNGoId61GbSUtozmbVCDQ`cdwbl zn}uBYkXrFdSlQc{!}wyqWV4ZdeJ@Z{9nW)m@AEYIU1S+3=CY3xf?#47qOqxL#T|PE z#+)$r+(CFEHH`OSiXDa;M;gWL-X%^y(QGLFi1EBPU6m^b0C^5GqN=_g9 zZ_N`WWhAi{`m#G_H~1U_gdGY%u!z<=M9gEk6BV~gv<->A6M4VHw#wUcb%0pzn61t7a-4ht2DX946-0(3S`{79^f8vGb z#oLnT%=s#jbR-hdiRathX2(X{4g zr&qEGmgixZJ~1)G7s1G6FQ$j?`_QN(8nB;mcz|Bu)YyWY=M{gMFPV@X50Lj>% zk{l*jd7lg3-j>Ic<$V));TS9N!+G#^BQ+X%@~|%8R~k5q@e9V$QQ3i4(uQ>?hldm2 zG*2mnM!pCkmW33kZUDdtxd?z5XdCTe zqU&en3ipKQjL7L(;Re_bV_0Mw)D5tW8BKsEKc69eV0_JFZ}*c4m5kkM%1mw(sL`N@ zf34GKy?T09P<)qHEMjTtkv&I2W7jAPDgqpa{RFr6D`9ewjO_Qp$xXeVTWM1F$U3E- z&yCpy^U3V1z9QVBnLug_&!%yWYAC>JzV<fK8Aw|JN?q6wiP=-9Wi$7M>mKof%bk(ibhI$}ce zKJN}gOG%y^okP(A6maX1uwcUD8Q9P3P8~UAqpAK_rx;$$_oF1{`U!Vh5ZtwUO`Qp0 zn14H3_b3EN?<^a%F7WloCRzL#4-47Gr?LThjP=`nxu_IRo^0!zJ92U8NgCR2&jiIiLjA?P|`Ee5q_&{G)r%_iUaRJe=Va|z{_*N4pI6jR{V z9c=wzI35^wc+f9rX$~dR3CbzRgQrrDflq^0g!zQ)MA$N?&v@5~0Cdmap8jANw3VZ^ znlk7dTXF73&VL%9^ejl9$0pKhlU;#%nh=-)>Rh{^2}CjUUhUDq)+7F6!H1t7_8Oy} zuU^G7{q@>%EzGG*hgnba+vXfz8+bTL7059J6!44mm-}aLKtyeC`#-zomrrC89y`Cp zXsF&xCBa|U31Xn^wAwqXcA`JCjjo3SrQK7*+ofvXJ)nZ3G6o=xjk_iv9MQ|pTz?0Ki$Z<$)HY; zODa$Vy0hOoDlaN?%X^_ee4oNo)h|l_2FeeInNgr#emleQ!ZDAe}LLyLi@>mBr^)RUsA3YcA5pz_-f}j6wvOVF{v_8QzMY( zj7Yv8H;l?#kNZZP-VwKnXQ8RZ(m*y11Bn}Rxst&?ck@&J{Tv>F#p<)NRG0{6o-_Ir zN?)fISh2@nal1lWgbRA)yx6=$p4jnexnu4DCUoYc3q+>j6w5~%0?4s!Uo6j6gT_P? z)T}Lp+-rG-A9~|x8pUFPy~iG}do@L*vP*S5L`YH@O_J0`0!Im`^udPT&hsKiUubV_ zL(PTmW&aWPxeqoBYNy-#cgOc_H5rxBXP*Dt%k)6O$vPM`H&s?6V6pQJo}Efi1f5rX zRIk~VW()XSqXJVL5bz66+QJrA`QlMzf@(X3eo_L0e#ZuXCGAb@zVN3$;kNIC1Q%BO z00?#Ph1Zy>4es?3itXA^ag#P?;SvW%i9wl1?k!%2B6W zKAY)c?Mjk#>hW)Fv}9%^r0_p|4LEaB5Ct{M#ie4?djo9oGkJEI9~RV<;ml;0F>*s2 zvep)a1M7=zM!pga!-T0Yr<_@LI_6fsD)J#)kJF+ZAc9M&=m%$HSuYb^Dyfk}V9#kJ z0DM)T0+Uq^stqU_u+P|P?24JW-{QRo*$^p0R38^dVTGNj$MB^3jlgtYX?eY>23fRo z((u>?74WA8y{4O5hZW^!p7-CeQY08DmJoo^=cP@DgU~1;vVGjyV0;&DYKA(p(>c!j z&x7c40{uHip`<{91uFYy=$ICWaRkr^yDI^_*rHz0x*wL+)opa&+lpaQ*YeZ7!9xXg}&ov7zkfCffls&f-P|{MJzC3qQ zrUPPoGlPQie5{odn6b~t#)@#r|IhFUGeWyWM{Mf=R1hLLvRMFBCqqMaHve7ZF9xd} zd(^NC4|wS}J8bMcccxo;SaEPt#Mg#ncuu7{G`}>H8> zdN0aFib*y^JMX@t_AQK?;JL_-L#(VxuNPc0#?z{%u!3(y-rGSanEaVp=_t(@aA33~ zrw}c{_(^4owsVXiT{m0WEf!ec-0LB3UT(X~z1v$MR95z9Wh-6cM{lbK98@S7bSqd|t5dvoWiVOQJhQ5mI zhA)iWpg?Lo3vO={&MVnZwAq^2H&xAY9wjocc8LO~_Uo{u3p_i>o-+i{oKn_;Tecq? zuz}U80LgWHm9j?$V_Lp43mbMYu0M@U10f7g3#=>hskZnuWS!WuKOP>62h*6CqTk|Y z@ipU}w*_E_E5=N2U*F1UOafY816ebPTI8XeyHUW;+m6^T_-bO~sN8l_BE5d!U3;#d z`5+h!_?Oy-PpzudK*i5f2@c{ZGv8GOM!TPgrtVZM7CD?v~s#$VQIPlp|wak zah-In=IPq^@#g?yS4j}D{9t4bE>NTqt6lQ>6$=UN>&$KRe%vWP_Pe41kQPzNH~bU% zOqxx&P~6=Gx3{*Vo-yhZv$kST-6mad^`C?J1ED?QhzJ{8$SeSq2!&lXDM+}2xWE_F zG`8yRrOses4e+4bPzu_2Ys6w(hhcKKl2N-OLcmp^o}t0#-NF(oy7|;+Kb;K6qmJ4a zsrIu^Gap4sE0STv8_iog^$`84{MN*HuMhc1K8|ABOtbPM{XI1bO}xR~EAR^FTv z1ZQNAwh73~EWF5h9TR66fs)A_pvkI1%GOm(^H0QeF}mbnpWvRAixDQuD>shDxGYa@ zP^;wxJ+SLSVHb1A_In@?Y;5Wu{PdZ=4oyGcRqDgqsj|U<)kZ+gYW{(Y`}<);gXFd) zS<8O(K1IkU(hD7kP?qw6ixy9S4LAmYq8$JWyv_HT@j>{q6r*gZXbI^*k_Y2xtZnh%gnyve@7X_hn6*rd

3V7tBa zg~85J?T0tc`!36sysI9-HA47ILkqYM>EOn0R>y0CT8pcQrd#)y%73R}9&CzhY*=m{ z+d#(r40vU(2@aVsA%uU>z5)EL5bwBF#I&>br*O0KDufB{1>3b4=GU&`SdMZWSw!QF zCmK^E(TXZ=@I~_mS4rwgE*y0jcFFjbxrEY= z`&ABPs_o@xN=Tmi^_AglIh4iXD85~wSQ(t641^Roz#g1i9b9`!VID?0wkb)=8iR6g zRB_yCv3>xD=-i__}YU!@XvR{AXgb2c$K(3ms@O9JQ)K zMiG$BQ|past(`UY%bA*!=pDChnNwYy$omkMxoQ7wt0D0F@7<*TTb{UZrkWXgpF9x# zap?s`;Cb}SOSpDUp2Mui{THWJ{){BY(s*2yL~DTA{JWoiTc+AKnQh%C7n=<=o%of1Nm% z2m=miWIV2aaTNk<9G#^pOzyU@h3FA*65o-F_!A{k>;f+z5swcAE zVRky5B3Mr~up68euZkPhO~o`hr@kHOX>sB;_WrUH#^V^hj0J_1HsC*nrFCvzoVI)d zG(t*KBU+la^m#%Q!A@sqie!L?K;+M`yS3{gekf)6G_6 z$E1iCj`2;T)!Fe|v;+g3q4kP8(dxz?6fvp&tV3eLu^JH2py5GL67f^*!(Z?k?Q%&` zT7~^)T19k+rMFB^Jx}~eeJD|Qu5i_ka|3J-tJ$X4;6fpgjEjpIFJnOckx*zz zv9t|uB0Uk(A`@OkltoJzFar}1-V+ect;fMGa1OkfnT;R)R9un7QiYS%`kfvi&tXoU z*BD@+)xBH`D*qt)AAbG&z`$lR1F9AfJi4-Vb9}MUmS&v;u`V*<2XhZs=Xw2y9730-Q-j4J>s0 zg%PvAiJDaoU9lVEKufLEM`*DIJ?i$Ak1>E9L3e%BzN?C44;MG_)-)|A11M)10L1_~ z%7g!9)=Pn34s_3e>^p6Ak=KpdXDgYb+{#~7U8a{=7K;{HSA6ktRqW8@`_CCkxN-ux zT-{+U>V4=UP8~$?v5M#NEji?G!pFb=m|HJ~@WctI9*{Ultp4Tlr?oAGs}Oj)MBJEr z_SUYY7{UQ39j6q7RHJrI!`(z90m*##66}z%I#URD4GU)eTRu zAD2^VW<0KKCw?4L3R{WTX=z=Lju;kXY>nkD%Zs3nV zlf$tbmipiEn1H30>_|?XT6Up!ILy83L)j=m#wuIM?`yXwSIw1nBAxXU8Y1q*=osd! z>P%Jkz9AGo03>t~**SW$yhhM6dych7Ek`0iBc|jabpYzrPsCLNf?K@m)o0dRw7B_C z_@0H>;%9fR63CAI&P(!C^VbsiQe7W6e~(QSbilWL+C0c_8|d}gX$(!pI} zkfNoU&dAQCR-m})_r#>_Gh&CLAOE1Kgu;9Oq&3tw7O*XV|Eyo|Zgs(cWk3wr#y=GF zk(>YqbWZM}>T@W|w~z|FlP~3JD}#x?Imt;pj)yG;P(*untJ;axu(lM}6q+!BS(5m7 zJW8Xut|8kkkAgFq2hH2T{#EJGI+})AmhHObpAHj+nIX?b&b+s@?CCb8`BNUU#CbUD zNIogwkY~<=YX32%17<*g4Hl#>^ZJ7Ttpt7F+1swaeuv*Lp948WLW^CE$jQgww2{^& z7O1OAFVCtMe2|)kZ5|yg2BJQH^On@p$aqB*o8mUzW}y18!@!EqxZ4(ZEykZr57^z3 z#`Uq4l!5gJ6hLy?!~54#l$~pV^9jYnV1h|46$sm-dI=r3?}tKA%PshSp}t*86M*$! z%q4x8sJ^%Z@u?d?JwJ|dFA#O*a37X>m)jm-mvNe|NHk545kS(<;8VwG9^=ey{>6_P z(V`vPta!JFN?jRPF_TxNeMs|SyB@<&cyY|SCG$}hmYc4}F zy|8_ReVvTI(1?Qt*5ocW>!7m$X-){hppeQv&UG1VQc7hPy#Z1rfob*cvPnPU z^R9nr##HR-v6Isv>3A~#=D%+3h{igPO7P1^2x(0PkYGjej$tY(GDO0_0*(={@RDeX z9MXr&_w#)RtVyq>Bnq5(U2}`*opc*bomMGjx!``LC!r2hNISzR47;edN|z$e5pmGB zp>}t0JHLT8^wUc}>MUS$l}|uD;2-iWwa02O8!~+5$)BlPod$%5<)<;?4CR2t!z-3qz;p>%uLP>FsN;>8S~2;4_!}RM&W<$tB^M?4UsmSpS@wQg3}rS+9#< zV_(C)5VQSR^P)F9yj4>ITxQ53)gkG((!re08tdW3qwXyO3qfHS^?(UV_!AS0ZV`o1D zQo?^J@yLGlmJLSmO1wU;72|J2h)Osza=>iI`*}V4O2S?ApFzv0E-VBayJ=Y0dzXGb zk%v3U-(yfjXXeEgdmDl35?Wh9+plV`w^-k39jb9~#~vX8ko2LB|0x$Zox+hitTK-n zkFi6SLX9x7=Ks>GZ`)z5?8356oB27pnydr{(WPIOT82n(?8vB9m3q|XLa7@V{U0>C zp16Fhv6yU=hGs?~tB&EEe|zYGHeDZMY4wEKpD$uUVX*w77RgsCjlZJRJ(Z@>7!`wc zkNX48<20Oft-$o!-?Ui)L+lB8c%aWXkg1>R9w*(LfX?kE%><<&R(?no@3g5=2Sk_;6XIk+5U z7|w!>3Ob0V%fY0yM)Fev0niY(@%~^qd`|inV6b60y+|MkR^Gu7fQVX0JekE4mTf2b zvR>?gxxyv#*i%P=S4_b-mRS~pN0Vqhyf?df%2uAD%%ffX8P;(<|mqgDz)jF)^g zSaM4sbJ$tL@SWYQbtQYdZML9u*#I;;&~*u7{~&JG!t|>Bz#nUHD(b)_qtEXyb0e$z z*&cG-OA}{leh@}B{tHh3mt)2+H$v02CvPX(`m3)5r59e=rHQ23H7daP6E`4A)?+$e zzHjfn5=y}FcK6f^s(pPPp5@QnTjq`m?ZLC=vT!J(JdYPoQtInESS&F2>`o`4rQsmw z;L%h5@{n8K12GrzOWCOKaXdicbXM;@VM}g>gc?~@T>FPU-kx?FsM7azUXhFYMXbne z?apcJdodZ-RxkCyxQWd&N;@-=RGX7{F2pYhYBOZYbixL`C{Gs4F9oii`b9Cy9sNO* zzo!V%W6;aCT@>dxzE(S8V=uOzR>>``evicFf!iv>y&Af(=A#gLO%V|!L9`z_|GbP~ z+Lk=Y_sVGQD8VyDvug3(ZU{MQL$=UB`oK}caEf5^Rq@Y4OA*F*Zv__ar8s&x&~QHr zKUlf6?~AjE?R%EaL+D-YE)m3xx$7L5Om{%8*h?^NgFa69N4(XXS<|qeU8t zTSE=O;27?R*wvYv^8t1ultPlHT3gv8Bb}67528i$!RO4Sw4ZveZ$+!o%cMc5@t}<4 zEm;L^l5<6gmKZ?(P0VaaSwU$be4f4#2G1$d1<95supuRmaNOdJ6rMUj#8NI(3JElB zZ?efyRAG=cuL>Y3d|O9@uYKsLBG_+%ULagpv4b4e zKy;gENrwvF+TB-h=r|48a=mY=sCFC9w|W;@YmR72qDLZClz2|sn4;;95^kwng)2hb zEnG5_zAVO^aC>6M`RoPn2z*$n_z184vVczH*`H{?v&}*ee6K*I!99?WhyF`~cT*AiR+&*tO|H@3ng$SY9p6jewAbrV-Ce|&CtBJGG=!tF1#2DBKSX#f< z#yemyt?&jx(|hK22*`Y&?ng5)x(c6{OvJf>tzu>2mnXpfn8`xI(W z@T$99;vKNQ{;qP-0^MJB@)SX-`Q=0gSHWDR_nqBS#Ac{r$p|7!uXF^zXl4ZIo?L+C zg}BwjE*y%HM#=eIC>?bHngv{mXx}KcEiT`ka==M7R9=PQr*;>XgGpIYzA@`A?hPe< zX6p6^{X>kG?W(oo&_qGNNaC}W>geR+(2~X>Q+UR&0GK_Iq<8rTgVu(j~A!8dknX~8=@R~=6c=)ByL@SJ@&35gAv0fC{+mmki z!ile>GA&?=OFI%=$@N7!{mi{zc6<1>#gpo4y6$V1RLiHI=O_U2s#GHLh~98~eGCO# zc|%O8qW}Y1Ch)EE&e$r+ll8WhkKg7RLLj7xP4( z0tSIt55EWi#(xjJmEn=qRmAzkeX%$O#)yMG$;~^O&InY3v29ORrC!i-(T>htv#~PtAWa2z=7w4DwUlDzx$+6LoUs{vG!*>Q=Pt#gbW*kzSir@k&M%Q+$ z(fOG~Ui>(pP@M20A7m$DkZLXMMFLTtN8}T8KMQW~*7sfq?99N*{DYe&S&%8Vhdio7 zXft4=-P| zjp~tRFyn!pv~45emCC- zs^&yCUJteVCOG=#$WZrq5LatK(Ic8~3s~y;Ws>F(t`Nvf_dpRYldIh8O;AhNZQGaI zlJI5h!nVykJll(78ygMGE1MTX(N&L7g4*0+VwYXxof6*0DI;i?tlrK9LwsJp>AiMT zQ9t!e5n`g8pyV~F>8F&lq~?HjLyQ|;DR?;+Tvaz)a#zYzS}Mv7M3Bmb2>N0BJu|mm zJL3{ox#PP^f?(IAy2Tkymp*g~x$VAuLJIePs=F8B_RXE?tRwS6e8NXoR1+(Mjc!v? z!wWbfY<7;pwMb{)w2YaTrVUhk)qilai-SJm=h6&P?I0PCfJK<*Uh8UhH$$shCwRPm zBi<3xVt%w|ws-XUX0yvQU{D)(_i0jL_W#PIBrHqS93)W0h40R^lX*(M+AyUdwYaeP z-YYySG`ykqluVqHw@>y<-VA*g4nK_z#$=YBXQOt9=;!QfH0s)QG3GW0z45RGi8pj$ z9rS|-4@Rjt1#m#b7vp9ptM(f+V4nj zQ{xRGj!JMwV;yJx98FObpNNFaLvGlfl3sLPc$JmuhU}R{_j)0$PHNR-4Ket}fQg&e z!OvAzI&}BvsYA`^@UL@x|0P+!1e{TZ(tTfWb3A4#35VctlULvevjm{tKe*zOekR*& zSTXh)$KozmW|e32ga=_)oPAI8KPAegyXVIvRL)oeW{g7SXSxVNLDv*O9UBF)+XWJ= zM>HL}ov&-G?+e%Bvh@=zQ=mVSBLW2n;v%JBm5aylQKCEr(R6m9E>?G*T?ka2ZO2>x z!VNBG{_t`P!xpDSJ&>%ev9^FfMh9emB-kEiTvD7n6bW11&U(0n$ryf zc+d=O&i2()(^({>R3Uj?!p+?E_g+Abi$>48x-l0HLSRAMst(4OG3QodJYS=+8N0?A z8ausQodf9N0coq9m|;#6fkb?o73zZ-DlB&j)E81?&A6QB>!ScOK+M0V_siF27mk)y zDAndEtxfihWfVaWR0lp}tK|O7h4Jp_6L_7AtVvKOJvKXQEh3e~9~6>UH31F_$PlDy zjA0|{=E4}a7fsQ|Ht;O$wFPNRSJ8|1%Qf%$Y4t?>NJ$f?$vVI& z?tbN}sIUMr0()Rzq>db|A95tz--Dub~yh9F)TNg8{xu(>#tj18yQ$850PufAXw5z>q935@fF;5ogTARjdxwSqmQo5iFG3=gK?9? zq5x2{UGLKVKc@qalvR4y%&>xG@a-W~L?YrpLT`w@im>`|46{061QMW#hnrTc=}lHa zR5raqw_xYF@|XRNXpu4yxsy4#Rk4|z$(r*ai(qP#9qw)8&12U@bPH?VGFK%C6NPIQ5V5)GQjJ|#r>D`gP@drU2$vWlKOJ`wVUt?gDmD{N5+PK?kByN zfFC%VR0I?dcoo?W`d@8H#{w?`6)b|)v48FY?l8t?bb$hzUSW&v>c{i|C;ol{q6efM z_I12vAM@tykxdP*ypzb!6mu`iPjYb6*nIlxtA{&2rL_HYqfo_>e*5Wab0NPk8Cofc9(zE#PH`NP5ZEjaaX5?9(fyb+_wmu|lE%ai9r41CFP zsdSTpKM!u6DSC>x;N~RnNp70F)@_tmF6XzBG=r3SV5QlB_w*b)-7PuIlEj?8R%RgS*t}s1nH}gBUhO z>VSGxYyeLmKs%cMa->)QPzII*(!W^3f@Fgws0y^_!b8YKiq~rL(;S8T?leTr-Df5t z?--v?k3g-ph&Ch)pEf+ZJbib`F$y=m=|v%nw5pW$8<;^2bR55F`OG_;Zyc3RIRVFx zm4E!w_FyJaaG7oHQO16=l@-!yYLj<5zPay$QvYqy>ipv>1kWd>#4!VsB#>1_q1S>g92Mk8KQdT`r&=}t zEiG}4_EpS}+5?KM7^{!AAEx3_$FR!KHE#3WLfr_R5_5Z_VBp_|-F@3KKsCh1r~f%g z;&?FWAB2;j>oIUS^>iXDfJ~9eDv9sP7y@t};Z*LsU>b3YmI^FA5kRe?l7z)8y3iq> zeBOY#wh8`7gk}COK;YDaOe*8UbKjt3OEVd-e_M(edCq~w*T~Ig^DNIfk<6W3ceX7z z4HLDBNR$+RgTeMtz-X_S55ue>q!cT=KyV?-W8u6ozTVfxDzrsQ*i)cD{sJMaJI_>h zWtrRGk1Z51*bz@lQDfcV6F)YN?s+EY*j!hHn2TvZnfP!Q%>h@;Oz4s*&W-Y)!N_{4M3l)9+0kMBl0_Rsf6R6di}na$ySpyti`>!C29jb z{m74W9o}c9;`qx29QbcR05DHyKdp_3aS;c-My(or(2X1MB-J3CW9P#dGh`YJ8H_jg zoijrpFbeCD4+k8lclBFJ=hQMAyN@__RT)gZ8xfhxV~fa;#kyr=W8pYx)(ec56xAoG z8@DewM_33D(iR`c_-wylHv+F6`{XR>&NWO#E=+X#CYt{JBMx&Z z5JtnbxW40=m9Xa^5O0%6TuScQfSX5-%%FCez{*0Hip)-Nm(DqB%Qtnp;d5@Jcl>W8 z;+V4x`(XOnd5P*Y0FM9^{>xi_v2*IeH-rp`X>#%=Jzt22&^Bc2@V~+uw4S!WE_4Q_ zq})Ap@C7|p%6*6u0w3kZhRv`vm1P-DS+YUpdUUME1w}pL1<9#q#7p$ZY6Vp8vYI|5 zO=gS)bAg+^pTOHRq;^z`oz!nXb<@zP=5F`7x6*TD!QHCU@~Rm0X|vwPTuA4EDbNWb z@GO8BHVN{HjEyda*io6>mQG*ka7<;|6juy*!Z{VsrOk+L#2l}m47 zo?!qNbgxL%yaO3ULHMN}UvoSh(eo59=@sImxYY)mCDsAu$`@#qe^j+@(}dEd#>C=g zF5(uLB9563i+s|-9pG3ewCtCjdhDMV(wu5amtuGAGHNs?3)#a>QMej#>r!j z?{Ud#KzDj_PM0T!KMXc1Ro>U&*?Vrtwt*p_Nb_JeTx~*aBBGrmmAzx(RkVPLO)Ylw z-M}oLwNh-T|EaGp-UCRQs`vrf;EvVhKnELADX=e0T;)r~F$fqRRd<^8U_1}P7J7Rj z^^f$OFh|To12B`p1$HYG=i!cT4(=(REp&=DFL=xcyDYg!z2X>j zOVmR$aIw8*i2}4v#}JAr7t_EUUOovIkh*bdtn)m5IKE3&L_%HAlUfv4%l?;`lOim$1Z4S4HKlovk;8J>U2_IM$!_j&cGK6jYWh%~lC z0ju{Je34|WT7ic$OO}X^UI2uFy(FA1cqrImXLn>h{Dn52+2SapBZ1>{l3GB?HRH#FM7Jw<(YRfFQkD3lT56VJ(H)5- zkjIi`9v`F}`z}mK#pz0+B}}Nt#rWqT7hcq6e>SBJJ-D)~iCC&;a%r}P zZS&PEoa8(KT)q7=DsRR~I|4=hX}jIe4ugB0oVzDg6v9=2TtWTG@Psr3NFhsmH-=8| z*T>#=m0RAGM}A`+NFlUQDGTuh^ZrUt;kpZbj0L_`%&G`_4tAjbc?|Lklmj@-6b z<>ekcs$zB6Z->qwQ`na3jNAf=u{W1p_4`~auL2(lDZ17r!;|N$P?#z8Pzok=o|1!a zKJ6j{F?|T7)_glduEeU|1Y%N#A_90duS_qN#Mx#E@kb0@{B!LX7jEC$Fz)w)?oM>~ z4nT#5GREV6kw`^QrOOuqc^JgmwnsCVk!NJcZkm6FgMDZsY`C;%MmIfO+|h*>OheVz zmnzuhoc${+5_tE&!xfD$GYJ3NwQ;Eh>?JdqZ?Xpq@Fgx-Ex4xOONATyq*9=-6*hkw z!!(ywVWzb7Z=)q7DhB%HyjBShl&kf<>?AdlSZXvY{($lheOM_ zXha9ngdqoB<)p3Ua(U9C8NNOE30CIANei2ro~ZMm`65RWtKjS3yfNXV$6AvIsF$Rp z!33_?^pm5z`19bUn%OXE0T?6B4HMXN!LXL#Yh#r`j9!D6-=$heTqe=>y=J>6C@sQtRh0zWaW9*3n7Zm$Mzi?1Wi0%@O zeww>FHQy7Du-3Ru80QJ>ED3o{<15xp+jdFXWVa%&3uB!1KSH0XsW5c0RE9zxNfh|7 z(>RWpzcP5Vz5#B`S#^+(ei#Lvn{mvWU3Y_>`)6Fcp!=+A;>I7IwAaEdFJz+t8Rjt~ z!@jBs967R>@&pY#aR??LKYU}60soiu(UV<>yq9^^m9IUroYwNj&&R6{g0 zCl|8RSwh?3T!}K!Mj26oPz3GxJ+;>2I5oL)L-Y40YR}35bWt7B+9jM{ zZx+y1U$f*^i-Acmx?;0)CIT70dDu0|+*$D)m(_NY);UL8_18w z1NgM?BIPV1?Vua2@s_idx$S2^Lu|EwY*LFn_Q#_)h-C2pU#|t#Gr7%BnanVYqG5js zbRCzju~hfGwa-Lce==9X^D`GhAg=ey+DTS^j{1{pbM?E^ALgEIbCdw179E zPmfy#I~jA5@2p?)d5pJ}Q9JCUwZ9UYl9$WtSwh{4ruqTu1sPMzU$3ZQvpqBAl`?9F zp2ZTWvzy0jJ2`fhJ3r`to)B76uq{AC*bj1DaVopjCElnW3IE9mv~<|^21Y?TOq~Rj z4U3T0#_2xmFnn1(59jyOLE+gPw9o)sJJ0un*xoym&-7Vm&-RV;FdVJ9?|))Ng2=Ud z@+!=PCrVunmErFd?mkhLWAfUWHiCpJf(h|o{ZFd2D)HBgK_F#*UEKa_200#}&(Y$K z8n#HqQzh!js*mxt-h>sUyXps|0GW0h1BOhSKDVT_rF_LLlU`6hycoO3NJJD>+IA(@s)+e~N%l3J~5_p(+{f&CiP zK<$ECT7IQj-P58?{rnckL=v(a?36y;i5r zn4XdH;1vKx13Im&B9H#sT(M6fUD}2SmPf#j%wegXl1(O4^FN8ZSNLVW;>o44$&pHH z9z`Y(VbL~hkj9)Nr4kLm^2u^{mc+uiUv2y91 zy(;NI_z-talZYGare;2nJtrovzM2r}a!~7>{76h!>Q&_v4U>CSq54H{>PJ_FUWn=O zYM&(es=C!Z>BLU(5hi}&(=|(*t%F<&1)p8{MczKvfvHeyY{&~5{*KmOo`vBJG^TZm z4cz3rW$_z(>b;0+9x{J%#-`B*ue2G&w1R1gf*s`vQh>1`tF%tS@-23%f`|LJjh3<* zVUUH3P(I$3^!N_L2cAd>)z+8e{AfXHNFr?wG4re62!pjJ%ClL4Ud9;X-hw&42{FJr zuu#$>+@=DI_bzjC-rxo*bkYY*0{jrmRz}A$P=qfuj<_D!Uh>v*Cp#%BGe^&ZJa#0C z4)Nh3SC#DMNJe@D=gD*lC21+0Gf2O(P%G3Jbm-FIho{^Pci6~~MA;KEx=%jQ{r!a* zaa5gGpIxx#eRdNiTIX){4>5YB)4bFspWXz!Vmijx89h;g@0#V`BJpcjTCBUV@&ebu zxdDs>VF=^a<_ajC4q&cJ=iP;*3Ax#K_I-k@1N5@%_%PT0tAShK_}RGH&C0);N$}q; zMKgW`*AzuV?3n&Awl9rYgkzl8Dl~ZLzbTtC`{Q zN}^f~`K!>wd)^(NPvbLR$jknpP)~1=oTiR|0Tgp^C!uOzHgI*L7Ks`bcf7_4FcsTs z<>|1WCu=VB*T$agngM^;U(Dl15@|2=dmGyJ)Y7>!?dYWNKuXlBTbY;1T`JD4T1@s( zd%fjZFw;oG^>DFBXAnaH<)t5T2bKt$6Dz#V<%=ElLP%n7nBd%Q|`C5XvDb?%7Q6zEdL z+HpacM0zY=Lb#w!|q^#yXfm~vp&+WBLJneAXZFo{0Eu_D+a3%Ymu z^f2yH@WINie)jWMB5ry_Y2bbdq1LPi?^pMB7zj>q)Uq3Se?3qqW^$D>HqS@tZ2C*~_W;%|=3ulW zW&bw^4cdrZ?6KIJ#a{M2?O}BE_l#A{{R1dYop98X|FwbCyAy;SFH`1)Eft4*hPQwH zmtCZ7UodQ_sQBlk$%57o359@{ILZov^>qCu@QxyXbSS(Kz?SOzpGq$EjILcF_5*_f z#Dw5ox9yV8R07{^WgWVbe9)4}e5ou7Q4L+n*`?>p4j~31Vf`{9WvqM{&Ag7T) zVR7K$B@CY6Z@#)fL{w7ym4^hbu`h3DtFW_s=qP%9kE~>6W=Je}omRFwwaec8Ic9Fq zh)8v>@;>xZj^t(eyWVCaeMJGNbG*z2nR@Yu!0#v%QpE!yUr#B`TCfGXiHYssi1cs}mYPyK zi{?zzN?fkGG=&s>id3IP*PH#90jPoWrU-^T!m*%TRteo1BF!+&xzUUAkDKbrnGz1r$#yyR89SErxMeT;fd&0&?)5yac%3%wbBuaxRd}8jikiq zPv^!IrMu(3zmW}>^X3eLcq4v7ao_!Wy!v|8Tk)&g9ZL_T_1U^Mnw5w}bef2dAIiuV z&2t6``n2eHF8dO);Pibbi)82(;s`qISO|g=U+db}Mvz=Q;x*w#=B*ZfH_yp6b^4`_ z`;j~?xeO`324Z_3h!539tnO*c1@L>u3I=j%N!jQ%Xai^W`8!`% zolp$;!;kCB}d01({O=8z!=b*@C?ClV4Pf(k8XgG6xi&VwBazg4Wd3FX$9ElAA{B zz|uzD$e71K(ur{XIYfbn3-@?tYb~!Eav>NNCRieFtPesyYDPQBH~URGTP*}19}&PQ zLyo3k9FhvMQGmNR8>tVeuK0>Onh_u|bb1i4Of8&F9Vyf;UdAh8Sr{1ajsJTN)LwP~ z89m5|SPRbyP!gAD6jLhwsJ)zQo|Xll%`biuc{rof0E#@Tb>+m(V96c)Qe}702~VL| zQ`XBj2Sp>32_f}u6Mo2OhG7X+#&_xVu|G)Q3j(o#GCBX1iHISGgL_EYuJ$MOM4xl~ z%dww31;wk0lKdT`{&>r>$X#0CJpE@wVh-i}Sk$*?)SR$#$aX%@3F_(?annxLS9i5f zQxAwj5Wh^r!c^Uz9PKIL#>7f*Q*?4F3OQE#ltQ^ZT02X{s^$Zh1594Jg;N(DZZY|{N%9~%gdpO8riHGE&C>$S-L(Uy@pbBO zr42{6f`-@;C4&U6~<_|@!^HX+D6$(&Q_ z#ck_?im5*#BX4yP1}Ak;UIGjjNUM5f-PD<3xlHNbGk(jr_JP39<@v7^1dPx#H3`T_ z{&dxO67IFM4vD?p^Lt%{PKN=U+n1rohAjpF))atZ`D87lmxj^sxmHXWshcP=mL0dZ z3wVia^3!9fqq4!n3i=i_&A-4iR9nqwC-!cgz0>feu<2m)%kMVr1P3!7cqXt1GPwa* zbu`WMN*~vpAkn%GN+4;syED{;H>1>TtayC26x0Iu1*x*Hr93L%TU5CmoimsqiDq<&>!?fYrRV~-I(T1N1EnXZcCW@kSeCS%JMlyRSwXOSQKmE)Q;-m{@0+j z_GM1|UqvxU*1JfmvAgtyKa^yF(YRdvONdI6XBKvthiUmiv()MceQKNfc4I&P-q9OOliRxNsUp#S^C-EiAm(8DEKkW}g zceCM}17|FsI`RRRQ$h2czu}Q>(?(Fzlm1pOom2FX zciVpASHZ#UrUx`4188YLu%33<+1W^Pb!Fq2cy{0qVXc_4f>{#gL1a6aHq!nSs6uew zi+d)9!&&ew=ZMHo8Q>kCYp-0_2Jsb3$y>PWv2x=AvySAhj_s#N=KQ7yqm1?6S0mRR z;5`35&(9ZfyTB|lh$0fl7&Iv145f8PVSWqcU;5q@3wsALyrZT*hmG^HZ5U{|87|@K zJ#?lN9X$1MPFf?JF*>G30k_W0SN!LGPp#|uis$u;F)m0UrWP`o$oxw9KvP6?mGZAn z|5Ud{ZgE(yZ*Y0)&-XmC(4aFf*DPMV64esKjAq+btmLBidu}uOS&?P79pQFYnWdvc z*O%S(GkC*N7PtbbZ3aGw59wkSIUZ`a2-{$AX#oTJ9isnI_rI1Lu`J{ zE!R<&LO8lB1%M84&QCc+Gl0fU9nYfYJn7O4!JluYT-9j*MLkfO=&O#fYZl7l5Vcqe zV9tR=XAGxro7fq*K5Fl-+(el3zynrE>I@lphh%spq7tgeeNYwjmHY4m77;sh$~dJb zSC^wJreUd}o^MsQfNR~@s5F-DT zMGxUB?!RPD)ju7kaFPhgo(lx`RR9UJlY?CfT4Bikv{q@u6P<>lx#fq#|2JgW1>`eX zJ+XcF;v;7FZ|*r>Y$B7*_ytXoTH@FfCV&5H`Me;wU1I{m(-N1C_tt^p2lWl$H=V_p zK`=n9m!;zXoPJdV#V2FEv5v;GzZg8U4}phV~=0|SIdI?0ArJoY2TfSBt3 zT%F|Iy1DVeD&qFJ-Q7*vI6eq{3VuNqL$7H0oE1h~zZ17I;-aA$A#Uv@F~nkws9=&7 z!Q2jnhUV``{f|mzJ+@`4%s*BHr&n%T_Oo+1GntA6QtXj}^10PMm-THm>U_;ZJfW=t zTx_pmko$dX(!%QAe#+W#w+R#+U}vP$N`@ccDNx6oh;4QYqki6BDT^G^#(+4wCrpNV~dtpZ$C=cBh zNi@u#sa}rd>G&n?!0$*hzDL8H%`|lih3rN=?MBBOBJ?OCgynwxnBnje-~DDPnk(L+Fvg5>JH09d&p8-yI`vuzRO7L!-uv46E7|ESBjt zL<##7c$Uw-!ZnItAl-Et@gp}^V>krcR3;MSDxt~N{fGX59l=eA>*@zNOiQj7DKKAZ zUcb;8V(O{`jGnwo58!Cdg;Ua2n7T0(bpsfd&oKHCr{HS~+O8R88T)Dxmw(8(JCb3W zPm?OcmcsqC*?H{SG!frQV7c+C_H{8Cfc^5L0C9KN| za|JtqVvH!)`!p&2C)1;TK@)nLUi2&`VaxpH|Af$X<)%Y_8P{R_H0W31<4{CL-*;`* zH38A}49!~%wHr2bH{UfxO2`&^eE!`45_l5KsP6yW0_*sFf>hWwz$~_oqV%&)diPZ! zLsg0YBcyg%gmIf2-_T`BU@=-yKv*&ZR!BIly^@z`5q>kgn! zV>)=oD3b*9`Zu*V?00>5dPZiqXIG=)38^Z1kFcE{VDN>_o*DiGv%OJm`5LcgrV2&6 zDUA)^7rU#BeH!TNd6d=!Tix^8?Ta+i3~^n(La(9p4K+e3i(>5?9M1LG}#j6_Xy{-dQy?lB4nDHk*@OK5##u>u75A97oET*;Wy_?h8b)}%04BmdlWL<#A z+$nKeVK7$E(}v$Y@UmxdYIs7JK9Dw@VLh>TexO{l8k^H~Gqm@3=^bLiO~I_S!;c9IES zQToE^CcKmkAz$he>jNRl z$0rz5Qwl(k>#E?#G%XXW2zkpD$U;bz5DlZrI|PLCuqlp3mZ}WWgNH}SlutOJz&BSq zmx74V-mVPDZdHw1k;Q$dGQg`>%!us@6M~VKY|@TQc}|B-?Y1-Ft*n)QSp5JK@%(hW zeP^@06dbrN1yg3-<{^jgbPV1J7ZWgO@yT5DgqBY!V2gbdwT6Zs<3CsK-Q+W^A>PI` z01$iRh)4)Q=-7)bqs*Ob%dHmm?ct|18~Mv$f4?j)hVD`r38I^ZHjF}WoB67YPwbG! z#4@+aqgA2OG`?hywyjLqD6`-aFFQ_+;JP{LS2ejj2(rNy7pjIxtcI*Ocqz>0zTN2E zMyw5eE!7|U8|rjU49^`x57abI1RNalQ9W$(%qPboIIMkx{DZ&5NY86gQ& z=jWq|!{6Jya0p^y{oVDFrXcgg2>r8DET6t}8%CBR+xrSXdS2B9uOM*%#Rq8HJ-+P7 zuScWx_XLR~REbzir&Bq)xnbFBvxI)9d=W-_WOx?oh6RiIJ_^a=xvRXS zq;_9V2pIUAqVmpu9!fnmk+9R$rEzPsP50>-w~(}3)Yc%`G#1VD zC((e5H(#4caAZu-0GQJkk1b^ksl!{$o~)NK-1p`rMD=t>c^AdUV6Uy8+I6j@&RH}L zK}5v_)(_B7U;F38;*czWMgr2H@TWYdB``wxckLxC%Y_vPWSq79?w^%1K0G9N+EB8o zpo$i*$EEGUX`oopAzXb`-3qX{T^uHylzfT^v&;Nht%teCnbflR^4?ts?Q<;{l7?_n z9hE-hIXCO&E7)hclwC`;^bLPvP$4b*;6a{B&I2$>NnpZi%n-5~~m+ zuzN6aE_aQ@n@2rtI$rUiPYgIPpgWm$T_J98Y7;rlK^BVh; zAl(cseLv;2RGs9ur(_3in!Xn|=TVF?2DR}Va-MFFWsB=yMe+I8yrm<>sVXvUo!<~P zpE>h%P>gm6sm5K_dy#E8BoD`zT<9$3avlATQ<3!ZsQfuhMtYleuSQo8(i2fJYQSg0 z<|S;9XFgf{zk4go2pgfh!`ASDR!`y`8c;wUW4Dz`7yK@dYAsxChTP>1iEht9L(;dY z01;E{)Ax%G&jp@{gv3Iv$!H5*dot^A0N6i6mvSq!YHhOaV{g_V2`?Mg=-O8fA^`(e z9~8}Uo!XBi@jME9;ovtfpQpliqWHg9azH-`m^5>(soRQf)mFwG&8P`F*Kz^KPjPJ^ znLXuGWBk>)zP@rVg4<)B#_{sJ@qv0D$4>A}o*Nep=v3$>))z!1aBecEaou2sNU~%s znR+xP&rm>zrsqN@FqJvToosSx3tf4;ICMr4t73V6=-UjuL8XN`MM9ve8|*+DD{F1W zbbUJTzUa?g+u`^r`V4Y*WAL@H%d_z|bzpxHoJ6a}DR8E)vtcebnr6C|TTT^`E=|YIhBaE)OCj0ooS0`7V&FwhmpG+OOC@ z?q5II^Lz9eHgvI21i=#OdYUP+0Zlj|uDx|~3zOY<-k*9l>tcu!Bq68+8ODTT7??_6 zhzoFzaf}XE6=D&Et$VCM2IOLEM%uK>FHG2+I^`3ilvWhR%l%2F4cQI7vV7>cgKRMG z0KJz5+Z^3D)J%LFN3J|hl+G?C&Xb1<^x_-)DN*d{$;kF;D3|yL&*R<;%$g|zd*p$X z3Z1o;Odp-XatRVa)bvkK8v3EO4NBGOO4N_enebHuN5lx~8lZ!N1A3UfRSd#K@Ls6&Td^;}p?3HmjPv{50O}(0C(;{9x2uYz$M(EJbiRKnIl2twfN2n}pnjBY`q82Y3&)E>_5EkmIXC zL+Ktkni5xv^An_Q{MpPx0?YCtxc-)8f)W95U&ghdx)#6$l85N@`lx8>o%<`0h>FnE zy7x_%nTD{sr()=oy0$x-FzundkPm=?^wH}*TB2TRv^=AwbRKsyVn_KF<8|aIR1bk)fkvc zlJ3q^>~~`mkG7q_d<&x29Kca(q(`je3cd&~`#{|3v78a_G=sxi8UzM@8xcr(=-9B)gs=Ay06#o#DpEG-0;hOMhf5~1;$#s^NaoCXMn*}UI zPsn8ESJbP*@haxPQL|i<8zj&KT8(7885Hh;_FV~CiYXi5iHAH{JZ_vw+(9igSk3Hj zm(GjyNaa+86a>T-0Y+U-U@(@Y)LY1NsqN7}Ni*l;KBUbS#N-4Wg4{htuW*ns+8c(> zYv<97Bk-Bx`#&3vWh!6?=2!R02T(|R<5k+rX8Z2!y>lcY#$c;m39Xc9wzj=H- zVSYN-T%i#`50&*RBN$^Ds*IcI{A^1(+59ZPO)rO%-Gr)#fn9KejVjUKO5@rhq;?StZ#pPz1$-Sd?~&8F0_D^WWM2D;&|(L-9hs=dvso9-XC z2D6Fg#p6WW*g*oFWZS8lY3befa6~)6rXT2jctuw){&XUreT0F64&Ts(!7IZkUJe!d6S(({cNLa$TuA}&dwGnp zDQrF9b-+X0yX$xTn83Bu{y#hr(!fgC=tVdrp^G8&~)t;r#A|j_4}8D;0D_ecIy5XX{gjOpljcu!j;0BaEv2V6<02#~*`6`6GX{$uO|YIdh(u@4Noom>mek91!x1~xybdzq zm!6}PO8@wfS2`Ue5%s5et-srg%%dhcvszWq_6tIrZledldD`2Bo8FIo&KK6OR#qvX^sTP1L;6tplq%1 z0Nws@bn@p@Tk>dl)p4||;<^wch%0&(5rm=Ll>95#pMx&O!3 zkago~2ib+3S|U0^8!um{Bw-RWS(HEzFut$M>{gscapAwL3p z^UDuZ&(+YFB`gG9BdLf%uRIeKu*=~MMa)jQOus7?nJ)89ED&jyVT(rAu%rb2iV@x( zvk5k?ld>Yg-$$kDpd*)SRtQ@*_w#7gogOE15F?=5Sx?L`xa);gIQb}ym74Y-B+(27 z$D1m?aJV1P-6l7E_y0P8PU+>Pn2D2Xw&#d{9ruwV;-g&Wh&M6AP>xM-42JPE4a#Uz zpv7myod82$xa9t{E;FA#9H%51nsM!`aOvUT#Bpt3I{1uzHQ8VBBrB6zDrI{!RY4Jr zTMaEC9b1h&xBuMrdHc{#Inu_wcAfXL#nV5RHGIfJsRxjICFEve7QY;f==)+ zB7?blH&Rj5EP8vwJeINgVw(GK!OSD!E0?oWZSb;!0_5F=afJe6RHKSvxow|txgg(u zaF>V%x7(q{bmkaCa^Zc__e$Ja`hay?0&2!KfU~;e=3M#D6!qc0Ua4$pHyTUiC|o~M zo@fVW z>f!w!BHnc8nNJ=@sMQ%akG?lVqhAGfoQq^KU|tp|n`T1HmH?|C_coT@dr?e!4 zAKy?+Z!ncJE7`7)7_SbrN-czCTM_+yiC9v{L(lz0reZXVr!1qxd>}dowyE5O8faFW&-53qCgz%eAByLJPhY}-p8rAIhfVNkZ08_RQdU7jDcEIyoQ_W2s^;*j2PV!Cif_f zUZF@?7H;CLf?IV%TXVVP>)M(Nk&NY%ISmkm(Jb{EnD%p{R=o)G4lqsQsQV3X#7 zBcAzmk>jFvJa1X-vVgys$$0#r&e%t#O7@~7T%-=1B2S<7K#Fv|fvjmj)`bSF0daD` ztj3AI1Y-Ol+LrzVxA=6oESnlz!@)P3^a)Ay)q3miCk}7qMz0Y96X2Y$OyE@*M4e@{ zf-eI63Rj{%l1yNhB?#%_#Nr>2aY!t8wfp;%u*zqpLh3-iR54L3-)j zU>cI6MrvIc`g>8jjzkp!fk?e^UYzv+~Ig+ z($laowP!iTr{W}9R|>pLvk!<6W%Z&r zuGl3;B{`vn%I z1b2(yBo5q*0I-|-IYJUwOzuz5j|UR!Q^E}fbfPjq0D+&<69j&)i(fAjkUkKoM#%+y z$ptqMXt53vz@x>rvJZ=I=FhLPcQB3$bX-a5bmc1Y*1k(#L3XDDM#r-rNa%|PM0(6A zcV5|v9}>}gNEffZ#I~fKg@VmS2yEl1`KtT+b|%IGh>N!k^al_&Lf=>bb0shyW5V>= zu3G4b%F(eNiYk1LCTX2B2HR6|tS%N1>yUZiW?hi(Hr`eCTTZ_OEA*oMfSk2vZFq9! zDW{NvxdXJAiq+*Y{3FKoRThJ{p-KHfV?OUCPX z{_^TdcF&{vi4)C_k{Z-tyRTKJl(Tkc0MEG?&*=Ob4(RwRl!S4}-wHnRKr9ZIo+u54 z&QOAk&;?wkr`HaG6U6I-?k6KT{ckD&O!)GyqLUoxquFGy`)*kSj4GJm^$RL|NtU38 z-Vt!(_OlUGu;uvQ>{jf#I23i;Q4=t=IZ`+7;!Kz5(I(8bSXFo9WIXFE!ssWNlL8BT z=FJPExsc%aT#A6e$fmyT?HKzaanQBownKValqb@Itu`VY^!t-qGIcU4?I}0^KO~sH zNDEKErvarlq-S!+3tTI@1K$6N%zO&z%EMX@Udl#9xbIBw5x*y-#O7|_e-)0xl!AE? zD1<k0VeEBJ#0d%x(x5D^xtm7^b1ETrk{qU4@ zDPqGqD7Bi1AjXc!{r0yz%8Y99v@f7brH<#-X5x}@czscIWnHIr(a4jXyt5NN=Fay8 zbroXAUO8cDnOK+f(~A5)NI=NLBd%yGH+J*JIN-~{X?6j^AmIf|O#o9zZC4nZgK~5V z>7UCZp@DZT!PAYOz-;sx48;j4jpi~D5lSkB6(GCUG}_Q!Rjzp->8ZFgJ`xFdb%5uJ z%QE}aDI3i~(0x=D;L;S~uX9r}Eej}j&nIDq6N{Q4eW0%hm*BZ&h}HiF6JzxZ77_6O zjX*gjZFY-mjClBtby*VT@0SCqw6|`}kJec7>iwnh8;n!ddCEvSTRTanouW}u4?NhP zG0hV#)cs~$J7bW#Yjxc*!&zZ$|NVV(8^&&L;rJO%=ztjz*cp+kp_3=-LzE(I+h*C? zm+nW`wqUj!c%@EnQ|F)cxs=A)Z>zrP$8Jydt&7b!7(G;=zn`CGhGDeDW?r=@q!wZ= z2z~(Ws)10Ryb9VSUh$lRB6vLBh#PaWSC@M;Y)mH)aOM)3e04k#JJG44dD0r^;%48_ z!@X>rW;TdY^>BUo;ydX=ot|7b%3m!n zvF4vu(6rPo*D)ZUm5!zAoT85GhUwpwu^J2i54{?+FrRFxR*abElTmJD7(Vh)tD30` z!Q>ReGHeO_lazBuU5`j62~tr^dYh}rd_w`-5upUP;W-JviciC6H14B-o``qv|IrJU zhb8?RTVWh3sud47#mBz_Qclrr>yeI6{$*A2g$l?|+`g`>4;ImT=9zB%r$-_T3FkfO zgNK)YzskHFh9^zwLfCgoK(r~KcstImW&-1UXcjwSNb`N><01gjb3S#x4&)BLqniIA zp@)9{N<;*E3^!g|7rccN#&Z24LS9HOkBVudRLGzGeHa-+`#{MEQS)^1-;2DUf11da zD_1IP=Q`1Aw6h}aCCw@VmCcLp56s}b#_wzFa!7XufFk_^(2(wG5edJ-8oK=<{hU-( ze08=LI~o7>8&h@>vQ{YT`rx#C&~Cj=K*(NU>p~aEbR60~=aD4iqjlFXcJk{dv}0@> z2)@PKWe$?pdEI%;2{yA!qmBMFk95B1joCXL)bii0_5L_J`_y4P`<>^Cog#3V5{EcY zdWfBPll!?{98``MI9;`AVK|#9n%FFuGUlK0#>%a3s zig3+y-I@?0Gk4TYz?<312g$b|N>26jrd>Z4Pq8%|=pvzI&ujI3Q0{AS%rTeBW-zc$ z+&dPoF={$~q?`AyA7Y$$2tOC@RY0l65=`I(die`!H#F~^zSmf5w89=NCA}c+tG{}y zwPJ(%er7SId&)vG#EXa+bQtbm_9>adk>-4ac4pNmU>hMV>-@EA<03U6g^lw%o&Fg@8NpCcC^7x@jIbaS!wN&00`gRJ3RBoZ)0@fR zL1&^ApE%7HlK_wgZUXB#Jp0l!P=kKo>Qa`wE+ZP-5?UB0XpJ?wEKj~<`!*4o9>~*9 z+$)Sxmon66W1m0L^=H&B@IHK5V%OPLD*DKeRprYD2_A1dgZm7~E#U2@zBxN5om+vD zlpITEZgIn#m~aWe-WqiXK$=3o>>%-K)1*}JZl*P8hT=YV-QO0-*QurFqZpw8CCmvZ zGB3~_Ed{95I;eLPX%#uR>XUT_2xeslEax~$Be6kbw7jlMOtx`w^?(au8%2?R)C5r0 zasB%Kpkz37%K5nPDn8F08B{q;7WBwKs!|ZRX_a+9QAvWt)za~K>Zy&$5|v+Opd_^Wi5(C~o(znA(IE$il?oT+9^Mt(j!-Gdwt3P&=K&mayYD-qY58dbEYWp8mBM`C9$MTmjF%YQNs?uba z;cf@XT@6wUw^cb9dovRLN->G?Da#SoL4hv>=2i2$V?MHW%L)yU4@hH%S^o+ zY5y9AdY!-0Jm}16)FGsY07Nu&c+;~d%V{f3M;FuGUVL96XRQT}o24w}d^XOov~9Zd zF}*NCioNes>AUxaxfwaNVOcZj5nqsKP%%Z1T9W(Fu1zn=T)Jf98!RSIwIvx}b9$>p zJlb%2z5(>R_sm}C-b+Ry50C$>X2tOt6sbW0R^QOk@JU|fmR5Z(|05TFzHZnFZFGjt zgdpP@MLxJY8s1BFnOP{WSySnJVHRq7-uslsqrc2E#gY6Ft8|BE{a8rud=3bSTs#Pt zPy)`KHy8H^6qLp_kU_ElBAE6o!ohmXjz+rYoO(Yn3R$?IipP-qE?<1jQWa2}46oTP zg>a8I@k>w|v1sjN$4efqYo0>pcvWSzd4*K_G=B^Kop*`mCL92NpqBM{>TE8*dB53a z*sP@9Zv+OFL`5vfT@umx2707%>{kY+P58?`0-p{HjFFJsNvDmcK_XQg z-*uEz=OlUgy$R5)o_&@ooBgK^?2MdmUuNI5Mzi&Px^9uzAKsQy+TwUcWw~Bb-1v4+X9BnzAYf#ls|^-r_hc95EfV#xMDB-a zwp(%#r%m!M1Hanx>Zux|XP59s%BH)xwD*m9X1g-_5_u7@ySgrHnmu*>rv$KX9OJhl zW;mfl94kH()hY}WAu1>0BZy%sT%#8LUoxaEBOm7CC5(J0@!@r@glyya3MOs6QU<{n znGM9UH64o#ai78edgL#1h4nB^XZ@$1cxYx*-rb4=Yb-An=WW%j;o?>Ohq~~9Kkwan?eJRZ(`U>r7$?xBgvPn~nWP`oNyNIO0Sbd-B@zOFE*n0}XL%B-ddm=$n=s zunCF#4hNCiN+{X~!2d7>N9~*{??PkPfvMq6UCEafrgxF7-2yXO+D4^sLi1vq_IxB|37RxYevhrR{r#x$7GNIjK`KuVdcQUU2ug!6 z6px#UEI)Y`s?G7pv-HKpb?-V3=ajse#QWXXB+P<(WKqk$X4!Mf?xfUya9tHjOo-J< ztKrcHW$`d9aJ2PF1XLoFk5>8&=U3532z#<^e8z4e*w${0gEmx;zc$COKJk6Q5uu|CWH3Y`Koj za9|atDO-JVHKw2xIQ#zKPXW0!i+kR*HKOoj?~=%{s%*3+6Mu_Ju%8uyR(GS~e4%nM zr|Z$wVHNwjcSL#~AdT$75}!{v2V=N~=rq9%M8{{7bB0X8KBK5Y_N@<>UW~?uid65f zN@Rw>Nja?GB_@R!lD8NS^oXDH2qW<;ztjAZ@M%Qkf2PM}^1{{u()Mi+sc(p9PYQpZ z&x?jlg~mrSqj1X3sR!}0)LZhXEjIH3@GN~23J2baQ!18$DjZqeCSoOPT}3p8SZV=M zUInozo4YK`3J&In30qb@1cYV2uK(8_rO8`TjH8@V!D_S826WYe7*~JooCHc>d@6BZ z#?+&o6wHMG$$UkmU~yp`YUJT9#=x9cs0s)P!5Sqt>G4L0jDTW2dpNe^NLk3Eb{YUZ zz&mhgzZAkBI`3QCatXtWo0#=wlL%(aSAO}W06MSw6aCM;MV45uQ9C~`9wXbH-CoCg zc(F+)RFGlH8Kx&`yJ%if5y8OM1eQCYCD%0qH@s~?dod!<7@n-aQ$Z#Wi0_YhaBqLvUIONh7i{yyF{81`in0#9 zK=EGqKoHuDf1VOOlk(bFhUg;Cu+ZtA45q}Fjfgn*ST?tbv?ro}ilb-*MK(6)>sg}t z0zql5gd0f8?IKLy&X}!KT6!8%?Z(;}+8HEY*c-;`{zG_?)v1*)v5UOD-i&Wvqmw{e zi!KnrOSdZ!xm6zQ8dxtzZw|tjx}o=>!>=L%ufs-fpZ@K2i1NKZ*2H_}oXTYYAF+Nzpb#eRJdw1~ZY3-56nB0Kw1gPhRo-iUbw>ubMGhuvgepc_KT zp!Z(einnDiXnR`=a`JKwe^BR^SAqt&kK4M5xuyJR(hX{STE*ez2H_}XaNm~CfiZ9X zziDo5#Rf!M!SMw)C>){cpZIF%g3T9oBQ><$*U5`kXeSjn>bCRkk;!`n}~#V z#XXGgC!4r+Fb#9{Az~q#ECojSpGOjC-pt?c>y+uk`N0a$7OP}@H0YT0XDhft^6*0= zKr|@foR!&@cGAHM@A-xVPIw0X#om9pkTEu^C^N|*R_BWD6q%fJsfDy5sY-=)C4y~(j~*pn=io*|D#%G2X8m} zf_2JfKNhGm`t*UgSBu_^V~4wk+inL+BodfSZ)m)N;6NtI2(Db&$r@HMD+M4tPg9ID z9?(M)wShl4PauOIeeCl%E z!B(wga?`Er5m1n`a|!T^0t6w8ihk{KKerkjLF!@Fm$HgJJ_(TfH_{3R*jvgBYUub1 zc^RG4A>$^R3?ggd!vL{kJsQF5XE^tHJK_}qX3+;3+QL|P$ZyCXoi9}(&R(li5#3Nn z7TT*v_>LyL*C+sWoISI;9q)=yH%*3lucs*S+>DVBzmP_mkLx#dG9Saho;6s|gUzN3 z>pCQn29vI+%UA1nzQM86Cd?RckeG4EtK=?pD{5X=h>{b3>`eR87`Nt%^IYnq@?q_- zit;|G|E=kz{Jb|$tjy@QgZ0}kMb5czosQJ%xaOA(9&pxWouo6$t`C|*ofSGjydbvD z#Qmmyx|oyWn^tfBgB&JwvlQUtb~WXU^P5Uur8NIPyV6vutUDZ18H4E4+|y@IQFIR- zCp60L?n@jP>mmKhXD%<0kFA<@Ma*LiKwHjCr92fplbTHfH&-D4<@OqhGj=rIufg!j zu$}!PO?TV~jzZ$Y8B$!`vSBAq#csm|~!FF3bZefnN+1=w_yf2oH7;~LjI9u}t zbdcxKo(9{LG}qjjb=GC{&0LvvHTU%cltr1Pcij|Wwc2C(lVWBxi9!!vq`0gRuU0DA z_Rg0>15?6_zlK>yf#XjR77?l4yLZSsf;?+*lNJpM7XL>ejbsx!9?cHt4nb6}BpNM9 zY89{-1|=5}!4u5o2fkGKjtJ`%tVeHKQJx%~X+a2Py=P`I9mCKNS<|8;<+c6Rj!w=v z-)zG3q~foVlj@YFf(M^0x5uH%GcoXFT0o88iV_*g^^P5YKeK>ZyLCX`8+_S+RDCT* zirMrDt(rhF77&&K8W*_yxX6AuBwq8{paGcvB;wZ>h%Z8O-#POaNi?tk=Sb;#1MA{Q z50fA*kx!rcZW-sB^Ck+U4w*q5! zf9^j93L{xAQz%g6W-%5Mz3IgOt}UQLWML_m4;>%Sp@@>>PUm*^r2V>CB4U=|4MrRs z@fEfXX$#s&W(Z=^RhUB+%8UU2c9c2i+0OFfyfwX}2668E%_UE{xbzABfQ~K~Q1Kn^#RRXQXf`A0si?CZf|}nx|C?piIgZgFFMgx8 znF_cX&c;C}c)7%r4Fv((MaT-B=RizNlj0c!Cj=$U8_5~T$7;1vmrcF%oWN6Ky!=|h zjt_03O1^9$4)TU^%WB;G=zk~9AxGCWsd zSH!q)X{@8opJOrR^Wr_yjy(5S2HPK z;>gptOmsfJns=*@TAepRgTHXH+avnXn(yp@wHl5Mztj6C3y6Il<=d`KGn}6<7M6;z zD|q5HTUb>w-2rQ_a$dfNoGyANW8T2fGSp{#)vNv$G&Z}?@^6isZK<`*xa>+Y7=$iW_Pp=7yAf` zK-uzMP1-O);LiM^9%IuAKt8V=eixr6&y8rBGLIIENkqjt$=}A&)F-${X$1BhtfB>i ziXl%aocx^Asg3ri=9anX&cAdJ6(NM`rV+Vu^M>Azg3&qB=siyRGqI2lV+?V=c|nBE^b|< zmLc#1Gp6*aK_w@U(S6uqpl}W6PXXr!Db;N-?QjU|x6}7(gI2FbRI5nBKZ1Ni0}xd1 zYnwSvQ;b7kQ24YjK^u6ELhp=8*jk54a@{6yqrNc+J~8PVxVCCt0TMNMjBMRkqc1HX z6q{PbsI3=ol!f*^oUTR5%ggcL>9F04?owAHGb^b*> z8xb0VDKh4EdHb)J<5dC>?kyLF-$6au#no}DAtuwIrQTC2Qgps@6_mY|P4y8ck8y5s zBq4Tofrd>%EE-LhEP;xv31g#&>HPci*g7h(_Kk2va{aNf3_B9P{r{u-2O{zNkef`@ zw58SnceE4l@xbzMV#MF4%lTtUZcDaHRs*@Tbw;Ns-jH4pHIR|YQU_q(d52(k7J2NjZGdHb9b9aV7_{RHQ~kWnAkLtZ40XT>?=A1urd zJ651t zo0Cx+$r*Eha}X5Fg?0e-_urfvJd9%wUyzHA%2rH&)L?+EH+qYPS|9k7S2{-xGw`&# zCtaur4i#n^!EeH3Z%kVThLd%1T`JsaL18*RZMYo~6k>=I76IRz#Lf9}Iv78J$0v_* zaCL$`yjbgb$nY?q$&Re5ri1J#xpE~agC@ljbeTP)-?G@>$?I3wc8m`zA0GStAGHMx z?Y08|KNDxt42v|%eru6N&qmsSWDZbdhxdvjai2nTo$~DY;n=~L+mdLJwZv6!C;$68 zsoPlJ<0CM^blO9Tms@!%2TyNhWKp0Pxn$O>zl}X3$UT(%5}z4?S|*c-A5{Ix8OHP6k3~M zfIXcNqLJ_9rl|8w1xk^Dchh&vF_DkSCjTyV9O|*KY1waya6-0kkjc#1yKE-irmrYO!pqm zKE2nVy(NI}mEL@xM2ws_Kj&Ss?PM_25t&{nfi4xHCp;9Y7#A)2i{fiH84pB^e8 z4a+E&x^~m3QkKMo^?Cc(Ht1d?*)YRTUU1er@4p3yh3mB~D^oIw3_t`WqA@LHkkH;^ zhN7{+h@n-ZnaI}-$4=G_YC{B0?= zAn8mr?8PQ{{%N8b*f=%hGvPG7FeSn3WE0fDD7xj@!nQjMZ+KF~#2CYSTd&5D(R;OgOq24$;0$F0Xfkj&pku@G&sd7YLalE^(r%Q@k`*;{$$y`PwfHKOXLXE1Qv!P z5GWTF3>1c0I$8LVCI#S!CEieJs}A;H z)<+9QnB;3>)Lvr>`q|yIN1E{k`rMf?2$wWJzVb*2vHn>Ui>XcU+SD3A&()%$l*#Wc zeEhV68x{Q3Oo6~$ z;TQENL@}8X+O)LA()Rk2RIe7}<#pH|YiF=nCw%4Yl4xkBIwh?mP=( zz=pdl$I}^jswor^^iTyFm;A8w9#82nX}t)~4-1~*d#k6Xguy&V#Y^G5ELjrWX=bnq zGV^u09)q%@M9^<>6j4V15R5MdLP?b3+j8WsB76ywd@7eP=_KVp{L7>R@`zc?+t=e9 zP4oWbo6aKaq|u?!MBrc}4D+&$d=CR1=48_GTo`Jv(p)yzi2FfXc21WBp$YM=R4j2A zx}yS`$Q!_%ikv@Sz+XhW!>?7!l@oXp{UpuhL*=-g9ei@0zExs7eHR*)z0_Hmf){-7ixH6nO?0ZUFcTzs$PZ{vQ6E`@A6pr@wR zw2&hR*GWyN)kYoDoV_T~*pm+KwFMaN>vV2MCn3Km7wA$(&5*8JO?g zT~aH{PtY8i+S_8EjK|#dajXM2oiI)Eg<1-; z`YIE}dQFyJSYS?afHv3^|IFp~pub5LSx_=tgf2-dVQPHsC<9%y*(ufY+9k7X9h<0m z8G)ctdKlk7jRVmps1lag7J6X=gUz~dEwd~@FgH4nb1HLIWOLx){p%b`J($NK1iLC= z^xI=Ujm1i4CU%cQ(|&fY_d~~BW`5K^izUkMim+e5&cJLi*;Su@&qBkJw*)&-CxKc0 z=k;=qUKpJh2yq=zdq3^`_4(NmN5ZB=%wK$nIKQN=Ue z1camz?BK@SG3`8R4;_{@)L`D#PhEBtr-e&+B!J3+k@`jpS7yeV0tJDO3lrj(S!Kk*1w=c19WxDP|sGm*J>yF6BRGt7O-Nk$hC)p9=419;kofTwT8 z%I2yJYbk2BB0DsGEITQO4iQEBgPI!#GJALr2HnJgYNI9y?ies#51xnfMeQv^_OpPq zBL&hC9(@@DcT^wkk{>R72(!~BK zj2b5ua=BRu;9$%Zg{Cp9S(xJDyOE}T)@$XFZsqzL?4|oO!89SiARWf}vhV&lfL?}Y z50sb*rJ27E#q&Vr>hRt%CUNIvm}M4`?l^fSlQE~EdGm$jt+E@(T@FmIT60rC9E|%% zcLD~T4Gx86e=-U7&@KitoHxWwCyxbs3uiusYmgH{F~6TEyLW7G{9^Zgt4e`8U*_cu z=&#|-jvnqO&(R6#bv&KYJhFS+3U4F6NL@n|BSPhiD^R}GMjxKb;*m8#>ZjZWx8UeB zsG_&|$oK2`HIH2OJwpu;n(dIH7t?u{0u4kLVdISjR14(a@xr!Rrc`NZmtocj-l|GN z#XSIrV}$JHUQ(>ylB7*WY)@Ne{0>0C#1uxuHuL3&cI(M8V8S)$+>0_IS2JwoI6I&) zV~|=kNxx6N+{;MXyhM!NMk*sJ81;)v%c_lB(dSq@dXzAgl++ad_^p=gz9N+_4d{UY znA{nQrcAQVR5BX0t<;bt=wJeJc=$s%iV0+%clsqA>x%F&0)+?}drBq~hMKS!1;uRo z8vudP)51RC^VH9->K>CP7mv$9+K%)fr_av_J?EYqeX(i|3Lm)c;4dZmJZ0^7^FCwj z6-(>WnG&!49LL-rER9V9 zeJOUjaxz_Dll+2 zY(0qap-%35M6`(Xpa)0C4J>@(eR2dqeW0N&&OVGFTds>uNj*u%{|b;}CA4@R`L2>! z&+YPy0-eu(vf##EYg~4cW_Ma=rYz;)L6u%b3Mi;~be{vX{ZSfYVq6GU&wC@q^7EL} zh*+!!Qc=abcth=)#4JaXmOHRkY@4IY~10R?S~NoWRV%#i_A zw!kpE$i_*tu*SdQ0(92tW@m)2NZ0cnJVYcR(!r>u$;<&>SX&Lyt0yY!A(->$X`cs{ z1}5iSbu^Q*8$Rq)tJ4R8aM)VHeRA2E@}Dqe(oEiv0lh?8?!f-``HNX$Ynod9)?!F~ za~J?E(jYM^3#ud`k|v~zf_5X95MGm3f=AlaeyY2j?T*r^zdypZ0v-osDb-~mB?{6R zNAl~^e0wwuRBt7H_?UU(IbUh&Q*O<$ra2#P|J2O2-A}I4&|cI^AEz=!OXhWz>7k}U zQSqJ-9B%smApU3-*o(3xD%~Z0Tj91;A!((G#<2S;UPd;iHH%~B`@@Hh-TQV+3x2LW zNDj&FLQX!2yU8I}#ep*KQM^K>ky3413J95bQFYa@+or~V@ewG`J=T9%f`jVRX#>{P zC=bVL1JHQCJ6F}rFOO5JDRt(UY*qo}CM-uVQ-c46O6EM_A&cKF?2F{6>7+OEbBq-V z5WBv_&oDnDrwW4oXUIG^iSK6E={bdUkigyL6#}*sqpt}tQi=Z zd$hJI*<~y1UW@N6&A#7Ls&QW)4hbMX&Xwt3aCeF>Qtw^tMY4|Sf zSLk;2`%QyFzgB*|Ru+p^H`xsjQ^`MQNPP@Yb2bj;^zRb9L{zYmo)T+Nmj9;g;f@9O zM>h5O5);3xJg#zXDXFbqz?79)j6#YMUHGjfib) znvC8rAMf3kJ~o*-=>Cg&W+Dk}67{X7fAH(8CL4=jUhAr%@Ph6W(SKT)%SNYDdN;+= z#2Q@iuv_RCBHScq!MZ4P^of@n1=TOJdHii;Zpv&?nSM^?Ry43aemNGcJ3;7ipQ*M+ zwWQwlPuoqW1qItQ^<|j;KfSe>{wsmwjsLxo7zG!H!MN(P=ze2l80G)rp7lk17a`i z7#WbuBd;3uqtR}pOC=&=$D|+1I~nJvp9^!I0|et{$uNtENN?*_@ukj9Zc^{MN9aM} zs#YXhHzRI@fF4`RMHFg=KT}4X-2ZyE$rOt@UKJmN6v=4~obeW+RFs_*V$#soB^ebF zJ9CV)}aXg+Yr=lKMGAd8X!47(3X+oVpEP!~&KZ(R6 z!Y!R&zf*AGPAY@aLlmU)V4mR;J?zYz7P30%%XmWlTM1>oNN5iff}G}-jbRz%d|AHv zk2kS?g;9JU`FIym*iEvn-p2P^s5ZP(K1hZE-IQb3yZK3Kx)SvL+SCAJ>q@Q#&Eoec z_D~$uD$UBR6cpzPZ4Wv|oq98qkdN}f?lGx6w^HfqZiW@o;LXQ1m{dYC%6?#tJkjhh zmn7Qz6e4ZCmC6;hKp%K%8!b^ZVr*~%tpKJ=%|=Vyh_*1vb>Ea3N4l^n#FMzBP`|YR zf(7~%trb;3QJ*+ycjLXBm{--rdQ#Ky9eognHJ%Va+q?yfDi}kWY+5tYka1!-*5 zzHo=12C~Gb!iWdql=(D}4rzh8`AcW)m2JSQ{emO6J70L#7txE|$uK%sx7t3j4K4^x zKh<;!&B`)Z4~^|=!TDjvIb|7fiRG^wq=HJ*f29RA6ow*F&dBoZs-y>pw+owB9LY@V5i&Ph zn(1A6)t!t#(loK)I>&?fGkfM{uj0@y2Uu;ps&qIajoROlS8eRKG3j#2psIa_JN`pU zA(}t1*i9{;l_j0lQ2C?X;qp>HHE1nq0e$TxrdbvM{2a)o><9Z>O9L$ao{>w!boc_$ zE|Ff7BjaVbue0WS$(%t;1sAEAZuoVVP#yTSpK3jp!6j&bbWgd7f-44*qbdQ5e%*6i zxn4?mvF%UVA31O!og^Rwe%t36uI{^_09@NTDhcsJ;@-k;Q zYR+)wn#WhH%=xDrb8s&6X6MIwz;dn0F+_jTZDRA#)&dW{riN zrrjfdcLEr(%p;#^Zq6dM5oU`5T3aefnKEJ7F+GEOSDZ)@#*|8rT^SV&-sA)qL8#ax zg-c+@yX$`5*x{O)gMZffVklpAfPwNom{DET=!(+vpmpz#2h*csuc?H?3{v==pE&$J z#+g36v8a$W42PBaIsgo^q@lVj8;9P7%ZT995bV#2X(tTxaoRHYy@3`(52`x@WE+U0(?7 zEJAk2G2xT2IEHR*ucFomUyHU@1?V{2?ne9Q+fNWwmv(7auM7!^6UxjAJoG9I_s zR@9Y7@IceMW$J`@%Ky&_icv1S&}=l&i|o6d_T{z&l@1sLyoPDh-a#WvDJ!(Q_KOq8 zdE2HeA~U9U zy?P24!@oPFMs#e2K+X7v_N~Lyl{+OCq)8}Da9zb_*~`%zfQT>Gkmib0Ze@yNo#suN63B+B)YwZ9=#nNEmCk%(_v~$}OZJym} z6pC?91WZjqQ$1ry!1LE-u8gEwu4LsXCR6l`Vok8;(HD9%+ilBeaPk9!uajCGQKJf% z?Z*p0AxitxPKO&D2OX&U>1B*Y!HI`|P|wT9HIVVWp+Z)Ej_SwMIK|@?JpMU9)|J>K zjfh&=)796cDJ*Os((zF{iF$d4a_BY)mfVT21u-;%*hS~Y<%r6(()=?QW4r<9mdk~6 zmE1qSZ$I?P5l55d2g(mr13`*N5H)1)adS!oQ+csiis9ZqdqIH_#V8$!1CfA-n5L^r?uR4v#yC0; zP_$)RyZZWnLUpYk(gbEOf;z&a4zz+FAPB!Lh84fX^Euzr=UV~7&=Sga&Da%ZwQPu4 z+TC`%v;DVA-?Y`0XJud`SK!=8oyVwbp17iu%KdT&8iyx6iN|M3!{&Z|s%28+u&bs4 z9ffA?0TY!Zf$L-cnKN*J(jlAdy6r*un9w2RnzbM$9J=pfYU{ zhG;R1MgMNet1ugBG`E*3G4e=sJAOBGrjIJC*4>(rc%}oKdQh}7nD*AvRjCZpp1;z- zG8P~@ux_TwFCvqKfYgO|moB_cBeX_~S5qxh0AWmW_p20eSjW6oB5&E+S8}jM%C?02i6pG*l`+~JV)}Q??XuLhK#1_F0NG;-um8D#v zgOJzybj&vrPMB0AM5L^*O3WIvJ+Go~oOjWCB+(lPUy@M3n%v)b{%xbz zvy08N`R_FU>y1D5BjRitg-Vyh0R-cYGhS3s^U%?U$}5j29G5%L=;}aoqhiB$!!Y<4 z11-VN{81XJ)qs#Za)gLArLy2CD$YAz=59g^EQ0l$oC60jv8y`NpRA%V&P z80UsK=uCq^#lN(FFf-n(sW&FK!s4IbZtfI<;G%2|iTr=~s?`()JY?mQR6pR`Q=Fg- z;qt$v&N{RNRboaVwDIh<5m4V&L%5j>Tu2A_;~;PkRE%^LCPBPw`966AQW%??%fB@T z_#1}t+-_;&W~UM^hZ>q=cZX9GC7bq$sKepJr}Lu`@P`Aq|2TI}QukMEH1h-RgnziX z-$%`!T0wtG%zd9&i=`Zf`$WAF6O#g4_CYYH-U6^r2k=Uj(F0Hn4gI3~a){xQ(q`ZP z{krm4<8pry2C{=!JD&%kFck5E@bZG{B-j|PbCV;gUU)WVDoaDHHwS0&&BG&*Iymb? z=B+Jd6A8!BE#bXUu?js>>BDVA_Kj%D6Y1KjuHG^B{g3@&Cj$gL<;(2SKyix(E6!N(>z=~H!U?=bjB-MJdjdIP>TT|V^{O*Q38E1Z0s!;2qKf)^H zDPrrE>kpBY?c;-b86An61RvB3FrgNhd~+Nf)0_TwFKGyf(H5?}o&ij`wFHY2J$~U> z0if@hzwB{WMyga|vunq|Mw9!cb_rd#-eFz2|LvHARX_+48Asmu;IKm%d?#BmF+bbjZBo3j+Wufd-XvjP&&;Is|Haj_U(M+QM=Afm!%S z|7{r?Fygh;c{hENz93k{{>6(p`*Zqk>saSK>Qvbl1q$ zRAkLbnQVSFMW|jq0P6m&v|9OLNL}8t_RM_E^k>pT;TyGG%@{#IqE24NxN#6CsjrQ==Q459TK> zED*haRvy7^Svv5E$Q3Oo8TCt+1JV;RP-RiGjH8=I)~xm#ui>vG{?l z9M^1UC=ZThGog$+G%w_sA2OLOZSD-+A4&!0@L zKtUKd%Z|g#r`Z>M7Jf|nkYFT{{H-2x@EJspg=kTXa0Aot7tS(fDb|J-Nq(nOn60X0 z<})^DyFiwGDP-DN-j!$g`9;#+Y2Eqpe+?oy5!p26_Q|RGi>G!Pz-~VeoXqMdeG@o+ z7w?Iq0nu-BFL;gnBG_MUB^SmB=J$-A`<5MeLp01Vcy1agIyjqVr(wY=?gZ>u#aqkU zs7^I&d8l>CqFsu2Zy4YeUzGjRV^=~SSkeR)x%mhoOxvTicMXB2T_(T@7Z}{+Wsq76e}ctUHgy<# zQ4akz%j8dth}&AGyLI4kF=A)RAp!(ap>dLettl(h43FT8xQYx~eP+I-I#_YO8EJOS zBq;Yi1&5I3mY30FwWWRg%|crzRTP^)h3GwcjoUlQVzC9qlx5)?{qYB6j))@;z?rs} z=WGM4x0vXJkr5Xj(_s<*@-vc30#SvIv(}}pD+W3y*IF3P;I|&Io8?#UGeh`_d$T#! zf={4#V#Cn)g)-C3lv93F58M{|(0c<*PkiN5D!O)}b6jn!`rGlZoG@1K17l9ik^P_f zybZox0ZxUDjq(SVGq$`j>LNa6jmG>C1}B4BAS(C)9~fc#wT8Zti3*5HbwF@{ER5pR zCDUGQ|0!H~JO=^gHweN07XE$zwd~w!>`?Z@-;a7CUov^NuI1W1HOVR z;NMq)G?~O=u4o*l?L0VdnTi$Fd1;d1HL)MyHR0BZ<5qwGAZa^umCB+c?(3W6;I%JB z3WQj);TiggS^j6jCBDek(Ee9lgp4B5GCjlF7^(KOH{xJD?xT*(U%(VV7RtH-;y!79*u3Vk&B`bnZ zJL6qm@zy=`saRms;{WZEEKeiUdk&=yQ?T72di+&3>+*+~UKAlS6>N)Nyo06Lh&8}P zqEq*?c|RyLtE>dAL`P$!NEm;j+^kgU-qOBX)a~RuCc^8sI*r+WEd$KtTS5#ShDY-4 z=_P)|%hFzrN-+6i;8=rAb@lxVr5z3Iawd9mLSowfMv5~w&M`L3xubU)yT~njhVoB> zVYiE^nSewvR!Cn?H-W<$Z2Rc_&5arTck*y2^V@+@)JW^X9c_?7h9^EregTKfv-6A=O8OX6%eRy_VI==S)SB5pj84d}6tEhi7F|BIvufpstmZvpF z(Fh3WhNoE#C$^l|G$;K#t;W!#*D6t`U3acNT5nuZ(@cke=F8gp>g-y4{c~g&HAQmM z-(01aoe3Atz)KOy0A1Y@R0R^{c`yka@ABYP9vbDqPem~KJU7mdKFq`^wRfYYq?F?p z6b)I2s@BRP@7V($=Bx5fwB!5$r>aOP6eS*+lYQ7(TN&oZI?qm~@JHyGpDP9Dp~nDZ z1Fh|OlU2^*P0l2lBy3Qmp!!n6qI!mSHR^vzc_`N?2g^TIC(ont*Ixl(w~j*qS#RPH zsZ>^NR^mj|3$_DMl4(gU14k?#M5nj+^3~zlj+St)j2eXc^#h8t{~kqMbkVpOB!Srs z!ZMc3_d)1fv`q;zdeY3Sz*<%sR}EiPy(9-zwOd`4)l*yc)Fn6k1Wpfvv49t(PhCb6 zg;{z1%EKm_ig%5-)I6U{3YxzmXH8x0%D^KutaeHLj)EwYYJ*?(EESV;Y+jN&3Wk}e zx~)##qwpR0=*X!wn|WVfii-0RXM$h<7_t+24rhv=Xl1sPiC*4QKAS9-E%K)QK7GB% z@mSS}tH}%Rv3;K18FYwXB#1)LMAO6M`PkuDh}ZW-T?>BO&pF1!+)p@30zdx*a^g2t zXPG}6c*n0R0g5?C+k`RLD!_afbwOLlb^?|dSUAiMcEqdwG^&7n^{}(4k$pi23B8uN ztDU8am%(Ss^k~yMv?fP(3_yiL7n4-)f0|ndO+dp*98>qBepx5Anoo<4O-tCpcmSf2 z;-6>Mp`F3e4KHb@E$N%jvd*gwch|9H-%7pUd!A$g1j-8(NyL6zfz|Vc=fmaQlb#b( z(s;;8XZBrSCHzT?8m#It7*CJTQgC9WKI{P#W4w>awPBn#@swQb`pP)h#7l42K{mNA zb*devFVqp+0kjbze;o~R?ojtTiPZ*T?q7~wE$2^a?oaqJV)0vN>fn&>c9E7kBN6$2iz6RHlEt4L~+~$(a?I*)> z__%OvSmG1tUsjV)e|U&PiYK48lts){U=-|22$cEeXSX&$eu$Km-l^Op3Z?>|Pg`ua z1znkJO>1bSsEj+d=q%}SYSWaYWLc(876_^t01ig@=A~)>lz7rFfA^}n6^G_-tJO9i zK{cfN;!vE=O=Jr#Cd-So1*ZJm%JkBOkJCULn%UDJ<6;>aesC-ipQV!T8Z!p*Jy0L2 zD&X+?E~qN+1hav`Oz{jb`4b$-TaL#8KmcCD-$s+CA*HD+Y+aY<3v{RxIkf^Few=1S zj3hZ_=soxKoJ{{l@~@@F$Kh$1op9)Y9{9WvB=kt^aRpc*Nh0y!Y@*MUl<}&%%P6B( zPmyp}Bg)t9YCQyq6~Y+-N0p9KF;9o37Vy$?>sVAEA zF?ii8>6$wUdw4!t^@k*s=?I&u10++PtfYjR;Y8rI9bn$I8j1ED_A2*081To1-3eFy zkCW^rnq_fkcm-r+GRZ)}^>;|_<#;&ePwt*mF^!{78`_Vw27V=^CqAe3{qZvL!ZAwiz0fRYtNyYwR=qb=rl79B? zVdbUbMizrzMV6*MRKKWiPsQe88rFje_YPTFW}#nCJ5(7QRP15I1aIw~CAWK_OI>c~&W4C_I#p z|B5aa+cUT~Hn5B4vMbcGv^wfMQON&x;(uUbq37xPpTk09BoFSi6u&P?K;esKdsvG- zA?Lfacl|vIbE2Xp^*6Zu1?r6h-j!h}K(eDzX4RTJEFQK`3Ud50nAk+73Tvr&RkI1N z&8U@4K{OMYY|IfMbgnR0m8*~Rt>r`HQo>PxWhk@<{-0F$4MV2W=HGj@`TeB&wTU39 z9Kr%^qh?DH0~fg~dEE-9f;7&K-tx86zqI|u_oa8)qx0BPl(|m1Kri`?5l%dok4|Rt z?=g0EjNFiquzl-rj84>dmxZtlEUt5vTR|mf@)dN|L37@t$Wi{}y4B3t5q4XE^+2-7T-BTXu0nQ+_5BLf|xk!z3C%B z7H4j%7qNm(XZ7Ck7}u++qi2IR!WCb@hHUWE0_YX%05IxQPb>RbfBU?Cq<2G{qMkFgJ z`54KrM9Y>tq`#qB(9M~>mbMjk(p)+@I1hrJ9u>bd6+tZ72EqNOzb2P`nIEIjTMT%D zzAwSqeOk(BQwCz9Vj?^eKQvv2aeDM;5F%G~rMI@#`AoJ>U<(a0M%7iTx1T0R#2&&W z4YJH;>w~^;l?4=HTgz$Y@+OQu9pXQ^J* zS9Y5b;*vscoPa2hZ|F0xc(Y6QNA;1>iwV(uV@fhr*M1TCHa8f*_#@TIdEUABz6*gv z3>au6e9GZOVe}eVDr+$>0=qLYpe&5w;AXpp$_N&_p5Oi>Iml@ zy<86iSYS=cprj8Xo@3Ncp^{!wr&~7XIVmj;`)l@JOb>?E@4@_HqUgHq%hv#5vzUK`rg<&{+zooW6P(qVm-lr(27&* zS(J(WeUqO!`o22Y-UODQCLX@3ma#WezD~3ZfMUnz$+oWnuzHT4eLzh&Z%@yNvmIVu zdrEfTB~MtFyTAaxH2!unI9iTDRjqTw4WMDOOv;_aXoyUA9Z2f7jFJK!Z2hn1z)lC- zxqyVBoxeTVl0FJYZj@Spql+^Y;(8ey5oZ%gc8(hz4#}-mt~C?qZD5}6y{NilUni

CsX|xK%6T)53$-sW0pjsR`X+v;wBa8fEXINU_cuGIQIHD@Z^7azfLDcnJAQP} z%O7v3dtO7v%y)}AMYB3dt|xxB+rxaZ8Y$Qrg&&DjwFvYGqpP4g{;KU=u=2hGWka`b z2HPDOe>hg^dQ_kaGmi&+MGOkuZ}OP6fXe-M{rGys967*5l#)iLagbPXJ$%O`*)^9E zX|H79qG<@F9?Vv`pF4=9&@TvWv3LH2n%wuIZ}o3;QN>VJa4*+s{ECxmxFxAhyVn~F zRaXzKaN^LoCbMR5wH$3^62aGqDkSyQw)*z((skCutO)ZMFYumXjthD_jR%u@xOso8 z4#WWQ{CGmev@w)*EhXx=>APztmvLBqt&~tiCv7YLu~oI>b_f; z&?Saq&!4yG?sL~hzNi~hq6ksBv+8(kmBJRkM#{l1`r2SId~F!_lV^(RT-=GowAKH$4$k^u9Vo zhuv$5Qlvd8Ec%)L?B+d92C;O{v^X{qk|E@r5W*s0oYR427FPLEvKe;YsVMc@zMh|T zE|CyT?DNgd`84z3koM*skC>6InI2ubhh@M()#Va+Y|j7mA@r{IfxSbXH~a|A4aa{A zCMw2_hIB1+f9UlI?%ZIg)x5~23$Zsuk3)H0Bxx0J5=0XRH;w;J)L`0mVvv1yI}XWr zN9^}v_RVXl*nC2D?Z=Kc0(kFFN2}Zimq6Bqk8rdK=brN_CctHpO= z1<-WZ;}TF*L(Z&4}p>+Af2fAXQry?~!h>7T6gVO9}ZVaPH5Jild`_Zw`ss z{8mSxr;r3jW^UT{H@{&rBL4V&ZVb525>cU_+V~-u1`U{uGbx9^w_H#D+2sV0w|-l8 zK@aYM7WKv<_3NZRZxC#TXmd4wHZ=?ffl7rFb>hGA6bkPz8$G4Kj}tOLRUe40)~J1x zn74jm%U=axJ%}g{JY_Y6!kY5HP}=vz*k1A$3GL1w{VR>Nlf=Izbx+R9p7GsFvf9IN zyo7)RcGsZi6Hhho0BO$}5UjW7mxXjj9|^$V-yiYctmqB-65-pjAJH^Vq_(4d`g10y z+xbT`27@y5FQL>c|I=xj*-Ueg#r-)`Seyk~N~@v~vhIBX6SjHuV+^8k)i1 zk~xW9u0PX}3KtucvU07&M9H6ErkV)XhYM}xv z89E?8^Cv%v{nLKKJzbJ08rY*_loftaDJI*(XZvUB7&C6&>RYv<^$CT?Q6bH8ljT(s z?ytbgN+yzh`yM__#!@m=>t2#=a?N|9+5W#_@5g`2{HCnXv0yoQr+85B4ujB+UQoi~ zn;eNl$t)jf*Pk09e;u@-TIDl_ZQmrHY@+7E6YWHh9z|HNEK046|K7aHij@5JPm$TL zar_Dpk&5wzqqL>?Yf_M+t3I?h(a{;H2MFq@gnze7m%q#H>c0I>Nr)E1#sDbBv3Sef+2wsSYd(LId7798-7_lzm#WUY2*31V%H)FbmMp0HAA)ji=t{%jP`Z*yg* zaH!s3H|I0{_DvLcn=gwYpX_#;~jdt1!4(Az9atBFdNqANGLDw zUiKZ91OXfV-ViLD;AYCR{Z}fONZ1T`fXiDTy~GcrRcy%RP#)Xd&~BS|^{!XYilVdV zEOlGgXV)l+wjj@#RmaUuO^|1HG|wMClAxUd2W1tFZVK>opAJs_cPJR;s@>9s0W_Ht z({_G!{_oi0(x@guv;+H5N>(b{xl^-=R_-mAdo6y-1xArU4!$PeIm){x?KEKSyR)f5 z*l7ks{3`6RpLhaP=|c0#80V4zuUjf)C4Y6{3gq@85yPHVp3@UtqSR_?Gm?Bx`H{JZ4rooZ2}!Un1+SVZGlZ^6fXdIR}TRZ~lM zM~$FgVK=o210HAc!FliWlqVd@&!2NU>m+S)E~L8idHJlgfQXGzsv^natci9 zy@q0{TXYWbo4GZAYO8kSE?uL?)Xq)2x_aR!6}SWUGbW16)|);gbrnO^-sNVe&V#Q# zS6Fs64M6{xP)2Eo@~@5koyOrie>L3K0rYsNPKm4;SQezzcY;Q*2-0tt48)5xBmd&K zf;X*~o!{#~Q@-=H3@=>Xk2;zm+{Ebo0uRhy&AzI!8IQXMh&371kMP)jmch;e&j2I^ zjzrGz0m!j_zLAkjq5Z=2A&;~SVjPkHZUJ>6&K#@|(dlKj+sZM%-s&2a`YhN$mJ@Bu zEc$O7XnHzwhq#YI<;3P-WW0S5UEB;6jfaH;=jTFD-}^{ z;@;d>HLKHcA zpd}rn_W>FZ4^#U`Dsp+(;D4az_zdGjt^9m3ctjG%!*K^m%KtR!VP!MmH*%|xyt4@$ z%Jl&;M?(WToF6U7V2^mGSIXNIovj%?-|=s-TxeLJ+;_?>9_CWa5mwOWF2OD_z<&$A z-O7_OLzsENPyBajYr`YUY5<{{Rq%BAk$ftfC@l4vW^A{(>;YncSCL@lFp8Ax-sd>7 zL&{V8A~#z$Gx(BP&iHz`E6b2m1MJyJ?bkj+aGp8ofyS<^_CLER01%|T<5anG?ZYse z2;U}FYArVK%M}7V0q6wfAhR%iz1B^JrWzi0II2t8H4Kv#h9lMu9sGWJ#1*^f&uFM* zk`mCUJ`@2jTm!}<#x8bqfS%3BfBM-wA=Jhm!r5b5{gt`2%=lDdFm_#p5%n$Bs8@W~gye;t2l2Y@S<@fq>Hjfdl9IiyF z06P|Eh#!LmP&8Wk(N*EmMUs{Fe!RnLQRx&>#{8I8yMZ8f)ppb{4?qd(UO6oJ9-63yrwt%PnV?e&gaWLZ!ur5^zrbxEM3R4UFH)r!gQE|NmJ zqH^C-IbX__I8t9pyS72TH=~m)5V=#~ZqB6n>xuqZDbK?`xw1BVL@~aBIVH>Xme5() ziwhGZ24ax1*KrJ&5*L66_qQfS3y0gh}Ea6$`*QuI`xgOU6t78ta*Ur z<^xy;N2jW8udS(c_EGilSAysUH2I2NJ4R7si@r8_bADbk!NMzcH~*VxTmxeY{a#OC ziS+X1Rq~WO!t)PhIcGl8Y&=$>+>f0`xNV4A)aHZqP`eBsA_`GG6hrkg=XH@k49eqh zgZQr(CO3ZXsM|Hk)Ys2&GHprWS3S{giIXdMkFo$2>Two0mv6ZnDn&k%T(Puv1$J(7 zoDp;@5<`1nIb04B6ISBl6AjppUOPy5fS!iSkYrkJ#gZEmoDJ?r)&IcPO@~Af)Q@QjlUx6sxsc-czpI;E8)=a^k){`d35Y-u`+&O<= zhN$31#B+)>iqUE%MR@}%x=db?Ik1wvHJM899z^dGA(qa5PGbL6*|tyQ+iexk;nh|r zo2y!=A6ZSq20WAt;-p#uS2&-lOnf+t>ztrCQ5 z&59KZw@$viaeY^;TBxc$WKuw2XLy#%m>aN39P@}Q6<5~4gE9;+0Oc13eWvJgeFK!@ zMP3&kGe)F&Y-vPTux?mjyJTQW6sg^#YSE51e1gX83Ue8h_@6*hO4r-@hq_h2gpL^#U;M?RK5H$BxU8|(aZ-0^&GH1 zBJ%QfqUPb&Vq5~kLQ^Wm)a##+w386^8T15G4!4*q*3drW6|*BtfnGDmCb4hK`P$(! z%J2o~fG4`=6XrV2Uw{T(?CGu>e_d+~>f3*N`8wnZEN}O7Mf@h2Iy3OUlSCP|(Wt$s zhHCGPtGoc2t zWvCAoQMDy2{#?Dc<)<$(9~Ai z{J2=S*AJ%!N%oFfV-k|sE1#iS;>KkZ@7C?fNe0W_52}BE_(ma)p*96#b*GWN1Nw

o^y3OGdW9*G;Z& zc&yO01NRZDzyu57BaNucylNSWLKqya6nmpHYnKyKFS0n}45I78^k42Uq!8<;lG@m` z3Uq+5_gGG$^QQe_@K4erIg4N!sDH|JJ7r!@1P%c4Wb1>L%Mx0fhkIagLpd zSWXd5WGy=2F^4BzDuMP~%#e8cOGPiZkgiJIPmI2b@{%%8;d8Nhm(_oZaG{{==K5g{ z0Ak{+y1V0Y0Awg69uFQ+2mpn=I1Ybzad&WLKg~Dn{*^y;XBFJvF^cc*s9=Mp@lh_& zUsZcY7iZMjm)I-*wjS=2oH$%o@p^c?)#+x$OOo)t20??2udrdcpg7P-Xy$tknU^^M z8&(A(2_X`R7w#!bWVZTpXZ8`)WEey29qd`w^lC~L+q~5_k$JGJCxZkg_sygQ zHyxCpQu$+ZN06nK*VJr4(59te3!Hdrw&Ohuj7;pydz(zmD^u(jwM(_Zu)8#yebdp{ zTN|&~F72$plPomZtS0IJHsyo;NM~{{6g?pec5N+BjJm+BT6dbKu}V45^~$j4I)ZMn!3Kcjl*t#Jq~q*Ow+jtBr<866HU$nU#{%X_T7V4>) zvKoe(ft(WF6r5}cGl3a0A1H$!FT4(8R8q|G!u4HQ?O-zB`kwege3J?vt>>VKBXL0A zA^m3CT;4U&`&@)L$1x{cX*8b<+Oti!MPmdcFvG+Jn1|Q)s4Was3|ba;FlWnYfMqXJ zZDT6f$$R8mn-ggbPI2mlGw6ZO2G>kju;{kFkqclr1Sg5w=}}zghtUWO!!F%mUc8WS zB8ey;dDW$TS4Fl2Vy~Bq=8#=vrX1}HL8J#&-42ZDAz^cAD}>#SapPsKe_r*@|Dn$3SU;QDyJN27EobI~$m(4%{i`U=$9 zf7Hy(5oj$wcBG055IPHfCqsXxw&ULj6ZDE=r60Hq?!y@p2qNku$s(E&RsGZXWX1A- zj;YGWFh6!Ht?+vm-`S=ACeQ?Xj}y5Wf7l8P=@=I8pPDV1vxmk5oa?Nk{;$QvF~KUj zT2g|)a7M?H%l`9XgH#w|V%PWiG45$L+g1j#7$IDGV3L@MC@2V_=DunOO&Jj6qm@8p zfu#wq16t2*h;Qw}liS z9KhkHS|oD0SiiYta}F4nnMq5r45Qfz=|h?%0*Z{I_4Sv=0l%wuXimmhhD;ps^JxY- z{^rxHEBDUgv3gquba}~uunSQKwJUe&2dUjP#iG0_nR0_h*9MCuFeWF2DISgw%ywF} z_wBnAr!!F~C_K-$pc8;VfR}(^86f`151C)6g6Xi^|Hta}1Rv{E{qRH}qb1PN*{=$i z_4+}hGQkr>=6L>cHOIWK$+(UGYSqlxQ&W)j!5MDhMmbnsVU;@NL@!vhEcgL1&03zm z$5tNDv@C&xDQ6q9jBpG3YauqPiWy2~==OFcYGa&#>CDpY3z`yc;}WyRsnUtyu=w4s z;twPatkMs{Q-#b=W#2#O#Kc{TgHSjkm%fUd1m{6$_Cjhk0okCd+K*mF-QjVgN#Upp zdZDf$%N?X=gsx$v5JIu9 zkEKOuVyB75GsmATCot6SgrY(wYIl!UQ!@1vfV_N@f{4{&LxiXHfAV zeA=v;0WVpg!luqB+TgBSwz}(Y+cwKrpZ?#`$;17_M7)16?BzAfmX*vR5fWqkcsLQb_gqpLZ1sf=$&?6uhR2jO;PvKY=8&2T zx|<8ml0Je|>JLhfH8aNzRmMCbBIz|dDLVG2(D!hOO65Fa{SPZXi*3T(_}jaDaxB1qQepo!;w_n=I+ae?`s^GHQf$8e-? zxs^2*KO~hf%Q-_SjFjI@H0OVZp&mZeKwuUq*gU9sry7N>^1=YwZS7K?%a2V-&gHsp z)k{w~eoGS{q)3S5#I_Qh4x6qDZi%ONs*09!%W}D%nDp@G&6UJSHRzNznv$N zRvc^%0H!o4*D)3b{1=`>K>+m1V3 z;N8j*gBdg$n8JUVPrRMF+)FtA@W2!2Kr_$W{g)LR)%gn!z~F4`ClJ)Ci3Uq0ER%Z? zTCm&r5(D7IR7buv>vPaSvI*yg+d$T@R@sAi!^M^_?^P)L^v()19Ke0$%xbZx9+6TBKLjkXSKA`_xl zsv@X5c#0Cs3w;ag9S#`xo@2WVKtw-J%Emf2n2DnI$e4_*L@=AV;+a&)T4P=zoyfopaCK$(QL@Z6aw zQwq4FeoH)El*u z1Z@?V5g8}F9oC)!HR8hgD@#B3KWr)Z-->iR8a}C z{FFdWGr%k7$Ng_y8ReM2807SRHN~f;s?o0f@}(H~`5Kkm2_M5z$|)F_Gpj_Go5jdO zT-&Qg8SOJdp{4le0^NrYjgiwdIM9S52y7opd%e~_f75a}w>P0E#Crdd&L}N_xSW8D zsj5G7&ZSr7T{MvH{5tYI~nDHV> zOHJ3|N&L4E+xMK78nKAU^3TWuy0(N}iU&$0fGO@A`#ebMFMwS#R;bOC)miXN_Op`; zYqb~(^iEDlR10xgqjj}Ih)b#sGxp9q3q zSWo9JuE2&@9>=N%xCq)>lp89#>GFHl$2&XhA{@r-O+pZK-&FXmG3ekCi%MfQvu0po z_1Kou1oF^w=8S-Rd43=9C+abu1sLOUy}e|7b-U>i^?_Rhyne?4Ie!>g0W%nT$ppkO zG0i;?m4#VoGq=KMrM7GK)Rth5?Nz!6ixy-V)8!cK!HE<7qB(INwrjvYuWw~|F$8wt zx;JgPgJ!6GBo?8xg~kxQ|B)UR_ds`A9ion&P9&n4)(Hni;@rPyjUn;t(8cQo!DJpR zPuk33KGLkSuK=m~{FUTu)+|)V{-{Y7lJC%%YGYDTji!Oh>-$TI52C>i z^6gxG-hy4RS>dC|RFlV-bQUR@YoEhLzgA~2tcfPZ1t&3b5=(t$+|(8F1SfEGDi<6>Dfbs9B_i`#H!I-)=S1!`ns!pE&Ldmw;5=yEe{C)4?9kVKf|jXm(Hp332} zS(r!l*r9Ix@!OXIjoalq_SYAyQw^r7Ox9WqWF37PXnPzeIYB*#?TSYSxBf8uDUoGB zrdZ}{E_|lWaO^!gICXU*&4@YM@wogAad8Klwv!_6om~F#q6$VoR=Pfb?cdN{AY}S` zREsVkFrc(dE9@EkBW6Dlp7OUUcJ*$!7ti}QXOprrQdjmO&@7wF=dpfA&NIXBaBk7c z1-oQmZee4~!W2ykzF5X94ts>hhy@zjDP)7-ys<_* z5tk7oZf?{X`Eo-3KUO*0gED&ETI;-=(SBTNsvVBB2V6V8{5Vlk4*G{k!j14e+pF7b zU0247mVzJ&HsjWOwom$xJY#}!-ATMNywkK#Wn&y#c# zb{j{Hg%`#Xc|kVHt|2H|pp3mblf*kGeT>#)`7$$a+wMeg!~{1{{imnAx#a*pc5Glx-m)Q z-?<2)Z7o)J(-_;w)>A%k?(P& zh@T!TR!wvme0?FAd#{O-VupP-Y-`3&X3V2+N*m*<=ZH2A&6QKp>@mdoy3CSbuNNhb zbqsaQ*Y4jQcbk_>>0i+>p(lK|EYZbsvqYb8@Y3GMf7a;gZ}Y!~GQymmd!+sWYHpL8 z4g7M0&XBCMZ6rS}OnR1g#}WHqHe4NRfqgb$v%!qc)3F{)O%zY2Uuk9*{%_;&C*~{r zgu@gtHbei0a@Iu}nB|J!4y56p6jzMhl(Ev#q%O?tA%qxYx}~LjRRVF&>di>itBwON z8|>i>sabW}FY8;R$Bc!fgv*P@%fdk*)n+|c86E2E^Ttn0{{tnC?D4m*eyWcA)- z+3rb}*m-Kl0{9vz7}C!a_rRit?;6m-t;S8B|A@?@4T4P(%l?r6@Om9@HxYJHvf=db zR9EvJYUF-4|G?^UgEjhluU`{zeLZxgNE?YDTA*VOaL;3?QC7#w_i=aG8V$#! z-(c7w8UBb@+mU*>BeBS@-g5FEcSl7jVn=yzMHMR;_)QnGhnaxlt zxiM|~y@Yst4-dPiTq9xyNXYa!NL$L>elAv>VQ^8A|F_FZi0wm-j&O9T6h{?@x2b#z zwu1$7&%Cw_+ysCJVFOiNy$e>F1ZSGmk$606|nR z@Ax(94`?6sG!OkGGZ3O)INrxiLEoDW`d))f>HcPAVggwEKxW_9|5*6$`1=kmHq`d+ zcK|U|*?E0?<_HePo0&ixU`p{E2%;`%`1`#hkcz1w1Oxj;zRdRYdMclY#K#`~27k`C z()$}ZoiFK0PbUq9jUWHdT`$n!z3CRi<46emV~mIe_nxm12qOgo-|E|zj<{)zN+<9J z>LOLG<|!96bLeZRc@MV=ox)EhJOjvvDA3K`gU4j_SeHBi*5ka-d)}?UCIbB)PR{34Q*I}u? z#JNKz1;^dd^vs(+#RJHRK-Nvm%NPW6v?w* z6=qCmgnYu#rQv+mu!jt(Vd_zY;_6*7dz)HR|1soUE@kqg1q%teQOoP4B z`@IvVR^vz?1or}4z6!g}hKJLXZb#nx?>bns4;QK;ZHxmL)VM!w99 z?${CIuyN_&G(6Nf&QyAe9HjrSNq^JPYSW_&8>fJ>fGal&QDJ&a$`)W>m>7IR&Hqq9 z;In4|=<`H=B?1?-=bE2V(k<|584xETUjBLxO|YU)RQPp)H(*o9v8ue)vLUXZ2S>4J zK%vC8^Ed47NR|Qo0f!uQFjX8qU5h!u51irHGNT{&pv^9@OJoX|9TvO5JQz{hFSXru z&L}jupE%X&p(2y#`sZ<4;+&br5^Oz^`ded6vv7I4USZ)LW!ANkIC_C@iK$7YLupwO z>Wdb(rs2em;jS|7BiOHZYj-ZYcej4yKfZ$2Ex+wB7`WS~gF+CH~`k-vYWw zhBeSN(ABTu$_Jp1CJ$A{OWk?Q(?IDnEGWdpsN=dv67nb5lxc^2OQ2tmU&vmkV8yu3 zQziCNvzR~u7%b>jJNnXz(mnt>A0TVkD7e9SNeVot1gk9px{oZ}`T=Mm?}Vj39eYw3 z5C%*>#@6U32m*B7=xR`oHceqj);l_5%e*h!j5W(rY87puQ7DZidD^yIl-Ot;cgO*& z&Ekqjo}S%tB;h=i<|pKWa2*$%ZsV;I^&MKvgO8eodnSJ8&oT&Sw3US-4}kDlB@}JL zxn%uW+rfj}AnT~L!s%7@f^`Dzwo^GU4J`cr{K!(SK-cMgid%`B=mAx=UoQHe#=Ys( zv4Yby*Ku&fu*AxDymz?-V(Q(q3NvG%w{0mZ>%&&Iz`e9yi2<=`{d6K$ zOg4Mhz?`W80N(k^d#xY>)l7OnvZnJ(1XeBql7A3)306n+dg*78%4q|=qTbr&jo5!6 z(9R9VS>)Npoc;K8th&YW~G5V7R+m5K98eO)`luR`2yOBx+MPp zyJNX(M|JX`%zxHW^6mv1vi^r0-{%flmzDVk8b+O$Gu*o;*M39+NcG~6O$&jYY>+tH1M+aM#Cwv$c zAOpM$omwhG>O0|(T)saP%%a8W!_|FNH9K$Z^Fqv|cICF;771zjnotM0<`IY)T^3?o zYIJR^f9NV&50%_mihL_1z6R!Io$6%N<@s|8O_Bb|=bH***>^GgjEV0v{R&u%Cr~@~ z`vUIKtjl?;A+V0j3WT9v1{wwpZB>cIMX`qNY(r2lbB>k$rHM>Lm$yRJ&%QjIKME!# z5N`TGa*AO=mDXn-aWhv>TTuwiO%P+p;viG4-64<~5V_FLD(x)C<;)75pIv6j?y=EWrNxohG)s7Z37d~ zn}O#r2zd|Cospb@OF{*SXr%~n^Z-T-g96n#PIBHARh1dQ2+e#Pha}F1$GV9qnz*C3 z<_`J>?oCFJ@c$Zoo%5D51*j)RASxQA{zizjdJgOr14tBkXnGepi{fvHV(g%jh(p1j z&^^N0H+Rq5E7*{(KkVp==H(LFVtiC=0a64)z z=C%j^PX*Ql!8{?YcfGSMweK!Z@B1DJA?oB%E}>$=GohtX9z#mTakjy)*No5lx&cRe z(j|~q6SXJV3H?!7JaM=Y(Mfhz*Q4{HeN&ZT7cYPKUJvUwf?t)?6)HSBdSZu@GrBQJ zM16&!FRAmb5BLA{!Cq@7>i>bRXmmr}_R;=^20=jugDw^#^m>%3XpK0Knluj3d&Of7 z?wH4*FQ+Nj)Q>xQ%Nu!t?XpCxs6(0Nz+XG#2%;vVZNQkkd|FDp=vhRnVy{cR*$UJ; zy~`wi0hx&C)PC#kc9`*1pAJ!VR{2g6QpdWX>19S_5{y(L?RbV&GpM;~G)j4o3%+#m z=H}iuBnuK+WW@RT__7<`(v0fnx^OT z?)(CS3twbnM5K6;ua9LWqs%%lQ)^bAr~76|zWIn&-9XXX+{tHO!{;^0zW$`3*TJj* z%9j#D*?PDRkRgf@I=Rcja>}A{&h+Z9Vt=O1wH|)aGvIu|K51^R`vqFu2=hk9D~XH> z3w?*qDI-BFj_qC8 zqUOM45Xr|sk81+dBb>Pn|ou zd-a$p1|meQjX(G33)_Mi#21GwRz&%yHEA(IKgrlqe$+)Hr~~#lauQ8yQnDO+1fTwQ zm{A0$ImN<(V+cCOmv<-PANI-E=S`)89K=*<4lld$7x+OO4)UyrOgBFb>vdT4(bablR=6L{#n$3=|D4S;gKGrS7$9E{ zA44+|rHjWO-$dv!N(sS$6U-Sfkc2Mu`k_zst{9J6F)P_zP0=w-%=yBP;9Y(9^>|6N-A=4rH)F|v?AOxLj@L~Jumb_?laTCBEo1n{1h2*`PlY9Z z&cq_`Nlbr3K>f>&*AX|v4vPSHS z%_L+0Me!rU*ZmO8I>Pv}xu_=#1Cq3lG_qt60^7Tsx)~`DPOk@kYn=%r@D6Mj$*L61 zPL&!*o28=7;34GKHuzOzKj;yExp$bo-1I>(0l>xx`=-wtar6S#vNJ8FV44EzmHNODUkAZ!UnRJzJ!{Vle~#HKI>@}HYt1&jjdD@%+zyF$c}-~s}R7aG_(71 zCeli~iCI~s2url*p@x=Vx($t#zu!ZoMKtM4C^X??m#D(-i#R8 z(!PH=7iPIya!o2dgQ7H~t#8g6hq5LBq`D^FLMNk4Q04B9Y~lZp`)(aaj%ChS8T=sJ9SEM?{txs8T;q^^Na5E|=U zT&+-`y!caD`*8=C!GeY6?Yk$8o4-lfaQw+bsTW()xYKRR2d@&(qxSa)LmSh(O+Qb3 z99k(<(&3oE)gKewXxmFB7mJi~xHWZZ!PAyD`5R+Yn8-~Ljk#%d-suIF5CiR_3(r;? zvi^l>VG0fodJOt7kXkFI=*80*)%APFXkYSRZd{k2t%@gh>?8?anbg|ck1qZGfDSwf z%y9~Fgy%nz2DBYwQz{j-UjV{G$1kvFt8|?a0}SvOA<~OYSzLf1!sM*KC#hc)F$fnas6Nw8}qv+_Lm_v=@@tcw98##{&Z|Prqntn31l3($*ODB?v zI}Vd9?8)!tcz#W3<&y5SlSHqd`^`~9(_RjL`t7R?Zv|O1hUO+I0BmmnQ&$iO=?v-h zDuZ?1deqeyT|r49n9Lm56@6-nUZe zuQ9W@Wucem5l__xxvMT1>)TXKK8H_^ujSh!ck`3{=58EsT671 ztc&-kT}Kzn5mHFzK_d97k3f=?A-7 zgLJc@q}U)_erlHWs3;ZbF<8WwyykH}za zhBPpZBOsK?`o%V?QNGhlU=z?JA9J4kTYw#~p%gJQLBQ0r0RJ)2-ZLeYc0R$)Yuu~5 zwoO+F{vrxweBvuFSHTkelTBLRD02~5a$rJ&^_HA!if`{K{wqvCr53vP{(Xf`{w?FK zsfDlzTEVne-(SQR>gefm%#j}>xMnzaf!&Qj@nsUO=kP zF_Sef92tCR&fML79bjccY-7 zZdEsb%@j{~Vo2tR1*0Sy8N%Sl>K3)Ls?hpaw_ zQM*a7Qz*oZkC%>;2xy>JWCwVqK%?^IJXjFte!a{%iaQQ%7IXGhQ}=D3wRD-ou*mb> z(ux{#3t41ycyCZOo*O!pCftamJ+QyT`)upQG4yS^ir5dy(%0DfTV zjEB+V?fjU!Z~McFRn|qc98y^@oUCN<$Ikc5X9q*ROBve9D8r%IXmDMHqv*2Ns*)Al zhrq*KAANM5>RItc|?^Of|i2N-mgY`>*jAl=IqJD*r zcW=hdDTvaex=<|LHnV;;kE|WVs9-A)+q-xSvYdey5{zTG{N5w7fnjw#%mc=p)W$90 zc%m|-v$Z4E!D~i3^Dl%EbG3vrPvN|Ej-yl=`f!s}Da_1W$ZN6iE$4tO`EFBfE|dXQ z$cfj+(PN|)MWvbq-qz$wzqWV(QN!wIMbZmMsonI9TJ9oy8+iq`IW~h1%MAtgEi|iW zbTZr9QJt_J>K6BMRUeHbw+6`av9kJ3eFJ(VStTq^xx$obeOw_-)eVSM5N$BU@+fX` zrG)+>KH<(t#=d9s1HYsWNx5*V(9{LWK*Q#-_9cCF`5pUXPM%RoSPE`bBhb*tsA zl^X94{^rypX&b`Ah$uz!N@Zn7VFTjTReV17+C<@=wZb%%Xf?=h*Psa(St97Mt=`M{ zykzTFuL|9h++D(y$V`wup7y7QY?4pS*p2?3B%_ABhHPfNX_gz4!&Z`@|JrSdYbhcI z)oxT*e6@;fY>i#HtySR(01ay6>KJM*o}bY?#GP{2_ux!WhfhYFD90Gg4|G^Ya$fln zK~K9xb-~hsA>-!*+16^Y9276Wd-Oq$qQo={va#43Clh)lqPgiy1V|pl>XIRU#^VW% z()cZ5XARVw?a(>jqq$XbE?-?S}%y3vYIk~J0!KBoY!rdi>`OerlhOKD3%ASzmVFN*lJ`MJSt1(n#1y3FqH1_zo)+8C&{+)k44DM*%)g&r3 z|0DIrtxf$Evz$fxCJ$rB$MPv7zLI_6n!0ShnknTBsK@Cy{*cJ{S&E#8jeK;*|2{CV z)C#+BD9M%A92>QunfQ6Vkw-CQHFm0y zLJ|59dycCDl@A6lZW9=gKMdm0#o^ezC-I1sx+~et1%lL^>_Y+bHe3fBrz@;(PG}UI z)EjwdxpWpNHOxfNs|^#j(H*H+PU=$&7MDnZ3_G+^*+KG@VoRfe+#t2g;PZptPBP%$ zMZMTh;GjmM`+gr0vY;*Mu_=>}Csy+eyC`i;?=KfMR1&?1<=OC35%FvE8tQM}L z%$}saOBq;fLTQMq`m+nb$#h$F+t0OD8o;it`F% zIHc6!+UYgCA{OKZ(R=)-@E)Qme(f&9d-{+W{<#L`XNKacgH0>`M%dCA<-03Gu&Sbu zR=s2@)xbz4VIf$WBHyzmSWHUh^Yc`s{YG!3!|Tj%JuveIbiVv7^0lt4XoB!b0R7|S z+viNF3y7aNOD2$#S`PcEmJ4}`x$x_WVKJ?HjW1F006-M_ixrk^Odg8T65(Vk#$!dr^%!D$H7tsbfQeJlM%M1z=F< zJZTY0Y{Veu0ad6$4@Ntk(`noP-7G>P7drXU(tr>=FE%ip{?sj z7>m=Ulx{yi;nbhQpENeHU|Gd;yIP)T5UUVoU7xnZ(-dzkqB8#&9VFos=`3hmD<1Qq zN`1_Tc=8I7dX(8TrG_y;Dlo-V{9BEeAf?2_AQlyu-}DWOH&@!VfDWIZCD&!aU>5^g zR)>Tz6&-${%W&pjK64@$WS8o1`m$pL0ExEYzSCIzmOVN7S38T@sf^)iM4=sP$vppa zc6sJ>95nKG9KpKZ^2(j|zo&#?ND=cgUfm=h)Vg7!nD8ORMJcs3a$-~Q=#30GTScty zMgkezcmV*{PL!H*2Jzh?sqcBt*sWivKTjb!O}$793Te69gt+AJ0hHBbU}O$pZlYTZ zR=wMEhjs1zk;=ZbQIBx&c4n&T7(RNyfYv>_^zJnN+81%wtL#N-&Cr%V0DAhej|6{A_V4q}O=OhZCD?Mk2cMhRj*F@;P2 z`K2tPqqnr~KAuJK6(%NGJW$!|xdoBa`FlG`&vdTrj98UeQ0;V zb1Ln5(UR(0zTn$sR}G^s*&G5BL3tg*w8iX0Z8CvEP_XjD1K&eR zSvP-eHYEj`6V-)I1B>l%10C7<&Q=s%aG*ojPX>M1Nxhm-z|NJZMs0Q&!<{4qLI zOhxNMq5eo)i;}#PxcKXm3VK4OsML?O=8nueghGdm`^B+8)D61=eRL5eIGdcUrKcHG znM8+&5|~T&wGbPg)I&n48$eRJ)GvKb(KN2@R2}qRein?PoUg*_tWWhlyMOCF2%%ve z#EZ}U-z88jwuL9?0;dLg2$0zZn^!aaU;=<;{6xsqf9WA%2uza*QcdZ_m!y+BjLCUL zZ8D1QX^gRJPAL9P44|C&*P{0cAe04 z#K&|-jqCjJF?rYFCOnt@Hj%NITRL|lp6@Ssf(H@KNC}gG_L-ju*}vnz7{1{tggpwF zn@Iv|`G8Dr57UiE5ET@>N1z2a+PMMbUrH@iyZ_D6X<)5q29Lx1Iw{BD+~IER>u0Dd@xzy zfhPf7E#}jyok?s_=8yDm$wtb$M4$jYuf)Liq;G~5%u8_@3p0;LX$Kt>n^1Mp4`z4) zmtex2U%4gR{dVGsKqB(5<1HecbH!WCK6ej}4g|ETNnzvP`h^vE3n!apR>Q7>QVYtL zypjFt6Yi%HOE|Dipv35!+0@1_Uo<_mJ(~2681?fBwT9qQYjjF2-BLQp*_|ryrQ8k2 zPowmWKR}XSmcwN~SzO&beHTy}SUrR}Q3QOT%1Oe*TO3?|%Cna#@G+25Re_=sD7mT0 zrbIqtCEbKAg@Y(jS2EE+@ka_y4HV(r79hhMqUj50XM@GonLJwZ6? zL>nT^Dl$mae;MQ^WGB|O5i5uY|Wl`-S>Y@7-@wI623{V$Gi&aSV zjh`frp$N0T$TxZ))-H%)LW96Ms4N7ty7i6*bNc({v_qJE(Avu%@xN7M1HqPM!#wqP zeGRO!TuX2o|E;YH*KW!`VqK6#rAe#w+Tn2~K4?4L9fwxQAe=&jRlEE?7&bZ7y^Aaz zZF!g)Pys}fV~pMxzL~4q20Z<_+Ys6G-TxaD*09V)%i|8aox%v-^MCA%+K68o3Kycy z`dIdCNsSY0yROmlJj)ZR=tej{!?P0`z(FCGNYjV9F5#PNBAC^b5uA@Mm%RR^JGcmtKK zeDCjw5cNC}0rCBTKb%K&yjnQj2^-`%*=1nFQ-a~@4mZ~GHS-5XJ>cLDvw)SUv%N2c zd(i+$DGnY*^dw(no3YRLEdh173)Wi_qoos1-}`^TjNgb{3*O{q6R;Fz9lK=bs@}mOvuIe479bYUss68I zTXmk#=6&w*%=Uo;hGQd1NEck5X zKYB7$4xR_aofhe{?v-5N2M$-~fXqF(qzLm!uf7A;UJ|q}Zv2FmYEqfo);Fxp*wtHt zA#Euvmjp>by$RCO=F$yP!j^9HwN9v0v>t(!Ah6|yEhMN3@mZoC4y42&KKDpnr%|Vj z{bjVW;Q&wOPXJ2{Nm7n~1&0gw>W_1KNGYP@DDef1G%1pXx0o$PAjon*9w1mj$4fOD zBT(GY(a+f+UL?l&Jh=)#F{`}#da#RvKu;+y@3mN!D#1p9W}YyO)$GAs+k(;rDH8%c z)+!9CP@QWPjRs#5|su35pGr z!50n3^VUV;fGLT6@wM$gjvLV35p`lu(6Etx?h3Y6KW6!3oO*XIlnaZM9+x^ebR!Qy zL2^yhz-WjRHpSg$0EwAypm21fQ{nb0N0@i8$JZ$kW|mUm_}$GJ$4h>P*ZR>>g zG&8R(`{TGx_bXDw5Ov@mZ&M{`VMBQ(n|#4qZ{>c`-H8`t7XOxEj$fnyLMex_T3MO# zR*1XV;X6E99M(iXhLS0+?M6}0(6zr?4PA*SuQXd9Kpqwwr5X*nwrQwFtk zql97FcRmT-47wld>{R}sA_BlT^zG}|4n<^@5xYwY(=~2Un=H*>j=K6&L#e!oa=Y&Y z#+G>;{YElPm7l-XZT*ytbtzn8Dz^bw;a>}u3-lbF5M~Zm8`~p43dVvi>Li&%3bTy~ zYzCcuUn<}tb*~-99r4z0%Z75PYR$g)Q>E9~D4KciQol$Q z>z0eI*6tE|<&(K%DAG+|A?5yplQdWDUiIi(qgQoHx4yHkytVK6p zd9rbbslOr6_JQX4*Uaw~a3dmE@xQQ$SqPEq(uT&=vG2VUoutQ>Ep4iB|M$@?zKV`v z7(G&ofM!I3^+$7Lu;@7O_S14~JlIH2KVr*L9n6$OiJ%i?J00w4g}%$uaaVlHT8_O+ zXbt`ie$M3N(o@|#mLxI)z&d{AV=cU5P{J$nAm#|#8x9b4A}C?A%o)k*Z@_qsu@{n7 zZRcB(y`Hwgb_`y{u0?1uxs$rVQTPK0#UAS+{Xda1kf3t>e9DRCIi3Kd58k_^b^MPpq)v&Tp*E+`N9; zf}_6iavf%_hrX1H@xwUud>j1860cV06|e0tOnURx%=3K9+9R#)i+D$TQZr9#S+V5^ z+k&+?bkcIlNgn452XP<~*y&8jH*0;P5BaUcZ#@bDz?d7*>3>fXqE;|#sGG-z`6ALI zulizomQvB56dTn}YY=PDe(EgGWQJ{qSl(_p3~&=^Ex5s0gCVhbM` zYi;qj&`JlKnvr(2PJ0OAzMJ_869^sub$T|+!vvXA-@a_Zn1vl;nEHSDh9O4=pk_|B z7XuVFu{fHo2A^*@aioAjdA-uHn!;U!b3LOhjQ(#%8_^3xTnb5&gWBL-UIB#`71VM0 zuSF-u=WIk^4$PISngnf7j0O*e+F(LmMD{;~C zszQ${5<+`Fg!3IV(t0k@L00ohwt~v#Yqd-M`N-SzbAR4#A*$9b0Go(17 zd~I3XE2?&*HSSW~Ge0IkM;e3{2tP4R*;%)((7p*NHruMxR$L!KSB^an8Xs+a3 zf?sm?v3@^QB*Hi4XGg=?r@g_~bqVw28YWjIN4*_OYI6WXG|~Pq%~J&&iRGMk?L#0m z6l3<7C(`?120e&5ji$&x7kEV#AG01i^#yC@myus)mM1Q z`4`>m!1E!F0Feuw2GwxyCsIc1rWaH=MPEkeo54~UrDM@fjQ`}yT_v?a?398*W#Ia) zvb|idOtUqEgCspOzMSR=Va?qvPsROGHC=`fa`YPg?-I^jW3@5Qs)-NaRzwu{B&n5N&*qrJ3mGwUaWp?X`5Fk4*W$GdT$c#X@GV8dyZ74N<42pXxRP-{Vet& zMtY9TMs$RDi0H(yz1k?rf3q)63SC00BDLcT>l8DLmg1LtQqgnwu8s{DYrDkGim7ed zhellNe3MHj>d)Vt_rb~P;CVhB2Hb1JL*EKH7>}K;C2aLacH#8XJYxw zo|A$eLf&BnrY@{>H!&>yAE1}rAh5g1eR}}+Tud%-AP(<6=&LERg201=o(;_c}xHEuRO~@{B&+i@0wz)9j87WM?7#~r1kZyi2$OwT z-clAxqZ+>8{1@B$+D4A@!u-NUW0U0};*7Oo{82a@hL}LF5(77op8X7_3flg27WPKl zIj(HFI)<04o;1rq@xk8rRLs?nZ&dQc^W3#Xk*o75{~xAIxdOc>_tm34<@$vkJ}iZ( z#NpV-42{tH;5U%-=oDzVYejbi9wwF*i3eTo+cKGv_D=;CLLpssNvN2t&V@FyP}D0C z_jqHronV$5#SS4mIUEsCps)Gt@+uVm+6AU>#|(KUcdOfd-%KpfQC7X+ zFE)t@!|d#jL}_P~H9&r2u)myPSGlt_O@>Nq)=_$ilIL2kth-a*Ew&#qF%RgQ_?$MZ z`4pM%_hp14v5Rz5EPpZgWm)|pJ8S&%Rs5fXz{1WOv}LplgG>B*BKClIHKrD{;46>g z3P!({C(*p{`Q7k(*1`4yvuKq2rrADV|5BFgJ8d2) zIK*{TifAD%wqgSqYM*>$@z+SNs{)kqXYk^PA%DS?G%S_aq(r$zDn3%^&Pt%3nINCv z9_8A8bUYtiDmjLXibq00)x+$Cc(ti&y-7pC-E&D1pT?LLnN{W8rjKbAeM<MBHDkmP{2Wfn2i+s$la^^p?L#jNVw?~Kjdgip)sEuoLqW#;tO>gYGtxQm zZ3kXPfxI6>uU&8c*ez2NicPA>#J$=@sPtmMBPy~vsKEoeUE<#7&Z6FMRG@;$Gx(DK zvF9+S6@;b(*K0AM`HfP?JfGZH8Q)HS9z$LKQNzL7UK_o38xq9mH_E%~-wW87r zx*sCzNzQzY8=NX#=quz=?WXCZmc2o84B{N>EiX>R)e%{^5Ce8XT@G1cp(-ra8^r`QRKD8n^e~#Q-6|duo)UVI zve8!8p=Qgr_WB6>qhx!>2{SE`D?g^}RsmB0IR7s*Z+pfZ9L@ua!2_3m^k~zN%2xjU zwDS0s+522f-P?#p|KT{?_wL5o#O`tI{VIXUJGzH|`#;qw)16{eW=rb0X8-nBrF3Kv zv6WN&I1>{nsoRf8!zQWi?PDgUOCZG)w~!5qmHna6lJ90puiQiFep%vamV{eD?@`WJ zj$c~*_PkplF8Ctoh(2j2kIZ{IHD1Mq34NIgXt$2Jj)htM5bYf5mmW;f6C~*UvMb;g zTkR)G3h7M~7=Pv;8i?lj5}oxs0* zF}c%(C_(Pe);;-JKb?tmqZYjdqG>LYLMPYK2BQk^7M8D5&Pe|}9l}KK1PtN9>g#3E z1_X{(GQ$Tbh}8h0Ne=ZshG~%Z)EL3aBNSg>K8? zV{+jkXiKdq$p>pUq28{}fdCqQKs&YgQJ78kRGm-?Qc?St2xy=Nd>X??Ia&H6W?HpU z%)T^=0CLEh^H)g7v3`)dzvZ05Qpmy=noi%VVyDTGnw!6cliTd*s~pVWNa2JW zCaCu`jnp>SohXeip5@e87}|lkj?&7KiS(*N%bX`NG^K7Fh-dLamGLF509;X$uJMUn z-2&ly(_!*_ZCvgfW!$Rwtn!?Uv@x+WJQUTap}SHD$v=D=xd#_uHFqjpoQC2vp?oNm zG?wGkpV)HK;bm)tW=&*DEKL_fH@&{yGJbPBgWJ+Lab`VxxQEBV=$OpwJ{zfgdnHiqliOeHZcx5ZN zaKb`U&fcwG$xsFz-@|8#Zf@=Nc>Y(c%?=_~-}@!IY3bc+83gs&KZd_zB~zfSfVDlX zoohrtwbbWAv&G?pAei8sI6?k*qW^69e_yR8f*u8Yb_0XcM9CgN7k%0Ke+YmH7}#NR zb!Hl(2Arc5rPmPO+iJw_{p?lSFVt z*n6Sg`Oz)+&=Y?2Y*7u|e*25J4}X^H;JoW@SYOsZBD-PM&ixvq3RC$x8H32{&FiYT zakdUIDZkyhEA5M_(~(Pc9Oi_x`11(j<-7Ql=N{F1eX%SI6j$BM^-SlI?mG3g5%nqK zbW9q{_~Z0kzr5LI1UALcP`v3Wq+p+uX5}41Hj8G(Kx%Q`)iik^3;#!INRvZ?R5kVs zAJzO5BzM=RUUg#kyhqjQWfmuhCuUSWUl?hRW;*+%f>F7R0vmlW^GyrkqoVDaiB}M* z7*|)^A0SdNk$EusiVw+P;BKME_cUma++EQFXcO~cOgAU2h4qb`fORI2tVMBM+bL3Vg`D>HP-!i;0x18$OO+UZKLY`RW_0k#-z!5BSrxuAjb0 z?pYkF-jf%d;W(UwNAKNZR>=L=jNe$a2~6UqGaI!ZLyym=VA!zj-s43M<*khJ*~E?M zug_lUj^*72(%mnH^A3OvAkaFpQAL8w8|Q%GOcOk05NZ4~C><*jNH>Y8E&KL8DpeH! z?Su+<`}qis($m<@nhuBG!kRioR$i*d>KMK29oGJcc_8W|f3yMU(I|=6C8}S&55Ap4 zlfG0+o%6r&n(I}>^Lm)vBxp?S^uLiwGMR5Iqwwz}qo5p1Uu!gGexC`8`gh8<6Iy%E zQf6BO6&T@`uK5d|6B^t(0j_)OWe2*DIXP1J@G!xU4^YbWO~xvd z?lw;sTmw>=2%O~5@0@7y-*&Sosi-4%5lpzGju4$pL~Op_D7<9{m0Y6ixBMe!x|wlNOkmb7_Zece}d=63dM)R)GSFZsNs?y9vV53wdxf)+YItKhCB;uXwy>xYXuNVbMX9en_?Z-0|hYkDjxmx1~9Yi<(vu44(4v5&LHa&0?3J|Nc& zmm2}(;WRXgXa}@O*yFT=bA?8frjWl~f&8Ni2ao5%rUh)Lix|K8ue6VFhGG=HW;uEY z4w11W^BMmhX!z_T!2^I6gR%5Q>*CC$n`V@4U;UOw!tHY0)V{k;R8I{{#HY^p5#_I1 zQ8{s%uA&hGmIcQrel9`&@X2K2)jk|jkUTd!QO_qdpEv;l(jyXs^k`GnXmX&s_$y)e zA0s(PfzXr08dfEbW-2_HRG+M^&}7eDH522Gm9g1hEa%*=gJ1FV=3bDt{=xtT$Kz&= zmK1a+e~JH&YkBU}mdo5R0hc)jT`IDg8`1#{T&4hQZWl9P&cG$MaijFilUj=D&S*oi<5y)rT^#1H?oPcnAc_jrw`J1buQuGez>7*b?5Zyr zkJo-QPDB6sHeo>A$~W9%CrTj%lv+;;;g6i-G8s4L`X8>?nGxT;ux)5<6$kMesX(JI z>F)<}wWzKG$rRNF1_CTeL(b2;2{POpze3je|JV8ZjK_q-G#G6>~NC2hxjI%%0Kv>JR$2(t?^qIU}*jQU0C zA+dpTtSL>|!0Hy{*A4^^?5DB@u~dp?Hej6%YKn^tIDWV`B8H4Q5_%M@Zo>{3j-}!L zQy;lwm9=u9U~+)5h?J`4WDmeSb^WvopPz`Fmt>WrvyHkr49rCbbDMU2@M8lHY2 z=Z`oU=vB;NTIsd1?a6i3nS|!Ptjm~Jm1-gX&}BWgwv{Xb#J9xSZCvCbt%}$~{SC0c z0ysao#UT_OIdo*aFF{gYkx`h8b~Wpd6&<`|T}&{e?XT5bX#EcaKKg%X0cYoXT&R0H3vcZU%C}Z5l^S;RMD#f}J>h|7 zmlcGT`>_?9BAj_qucvm=ZyN8KWz>-zIQhf2Qym9-u7;o`8FLQ1_Xhn3w43p zVB|G~{jYO}Q3x!F9up6?$i_s_-F7!$|3``R_oT#06b$X;*PO`kvwXYpM<=jnv2W*) z!H*7)NrH{noVf+;cfk2wkzF}lB}vjf!xL{Wv|U({h+rHWDdvZCvFTlD&oY!dhq|#6 zh3U#MnKL)Km*&s!{&db`a_9s6)jY>lvTv&zN1KA~W|k;AiDf&jmwG%`<&Nq}6Q%ff z=Q?$6lp<~nay5KInuSbqN}|Ak6e(2y%(!1O94~Rby>NF$8LiJcJYybd@7z3xL;T2c zKIr!Uw@de!!vgGsO%=@d7`!|gw{vpzM~+B6I$+QnQ5;f|CN})(?4g5%5ojC!4xc1}oKlKjF z*e*WCoN;@yt8KB>62J-ru?j!WD+_K(b^ z`iXjMv42ofLzHOtUUA=N4faOr`}{L{OK015b}4B1V@!&%o2_7R*gS3#4;=KB=UP=( z#)MQdzdavcA{doksp_hI(wf5~vY&d;`#(hvR%p6O+d+L!`p$bysV2Z}CU1c)rz_87 zD`}JN_DyP+dbLMJmsVSb$c4v5NN~io^HiU6uOElU5u}}h(`eb3fchf<4r-_p{bC4g zfU-$?R}IQ_eqS;xB%#JTC7_;**55diEd*u^%}z@3boT2Y5~e^W;Q! z29Tqe*(Yj*G}fLG=~>lZ%@;vOmdL?isO%gsoMl`}9*KLI>{&<8y15kA6=psBK=1o{Bg(swQOaAtMmP%GNVP7*bI`neyQ=bVXxt zy@#XO;OR+DZJ`o#;WydN!!W4O8Z{IqckJy{ictFJvLFgAsQnw9h|k|lyxE$ZZ(k}- z>4yOp@=Ly4tzj%(iht#cP4xo_I$inP?<9%2Okw`>Tn32u(k28)Q2LW%+Y8dp*w^*HBXf%%aWg@$vGW&`O-%bOf-_I0>(R%)^7oGW(Sr9amCI(n z1G`4Q0T4=1`8fO0GfDwPs@LQGjy{XZQiF1uxMy4Tvc}!UNxGf?@^Yz*IXP4rIitgq z_-8a`(63>o@|VL&+aVHxIIyts^(z{N-jkx5&A}dHFxT0dYpRvrX7BX^O6HDHxs6eX ziw`8()FlH{L4U+^jSa^tEXr`Mbf$iN$~J{f79g^aolh#4bo~nR zlN!R}Qj4EkCdUP)YsUBICYHIN@O#xON*TW%9pI3}r8!#f{4~XHk_j{rT^!RZ z_1PIv3oES37OjEdN=heAf}rl|vEXouWW5t7`8bE61vD(c3BT49Y7r~l*8)=pC%oPPr!TwORXGUa>B4I<60EkLXd{kvIcS=0%~dZFYHp;a8)4 z;s0n&M+~KrjJvUQ|6O2+WbYs?yY4brSHUhCv;J}-vpm%nqcOlT=*nAmclf~7Ml_hG z97uNi>JIW0e`a|_IS%*+@So6SEOrIka3QcCkVIfoLR1Zv6hOvD@S-(`-8{2L?pq>v z1w_OVwnz09Mq8!u}6W6Hz3C#!9GzBYFs1kaf5eB>4{y;VNX`v8RoE?V1j5 z&_Ut9S%r%x`n7hKWq}&9Wc+#s2BUn&NU7~ zD-y4c7;GzpPV5+(&)iG&-`=JMEQFjt!XIzqVkfKE4q3j+j-k@&O{`*RN-|42xn3)s zPsw_B&2j&AZlfG@4*2f7nt_SBXTGg6H61R8an+BV_JR2IN=8MGt`%Wwuc|)zLCR~L z(OOoqYu)Q}K52qYcX!`;mi^NDa+r=V^K`^&p^6JD_FlUKT>HRy8L*!C;Zi(4b_FUK zv%nr&Ke3&&Q1H@+$qN0en|PZfdfBDkphWJ7rR4)KH@Akx>4}XHw2)kxtyyDEZi@hP zkTG9y2$0}+)Lfa29$T-(cxVLy)@6_h%|qh6M?f%{m9$7i(CbDZA444G2i!dQ#`)*f z`6z7!kVuQK*Xi6?EdCt{qtBv(h3P$a{A;nJViKQhOV{)!xz%n=r=WqD$(B15-nlpV z9P6NTW6`Uoz7jOHyB{$a`{(nMaI9X3*6w#gxR@ETtf6)qYLoi5)hNrPCt;_t3P!^Q;iM6p5 z{wDj$13Ff_?d>?*nQM|{2`eMsL80ONKu?n29-vc`UK5EIapoF)1NuOij|ETtsWUIh zTw&Eah;cRzgh>>#^%cH&S6u!!r}OEufeopW`j=XwE!WdZcAKzc7fks8$gFQZwIV`>oeiUkwZ1>q%vH4=_v2C zXszuG-oBHSRd7Jpt71}X(faE$9TI~L5|16OxY*xu^=VR%ExxuLoN}auFdWVtXkv6O zcc|Mx)C%}!rCwg|eQ_m#|ChKO=4qW<0^a2<0iH$m3`wdfZxNvR1rcR;NapQ`^_VUay<`N=+$xyep~i%tp(*TzV9?3U$%Ttgx2xxJ5lNzcsMG-YKAbG@m>CvBZE+dw>gh?Y9#> zE~UL5-@!cHqq!UGt}1KVs^rjkSZk{GBJxU6)mPD!cgBV zjVLa4PN)<`NejWYO)Q2{m!cG5aiHd@3&sFK9PJp>!Lpws$q;m@Tz})&xK{QW9!2$a zK3lS`X5<+siu+%Ojh@zUTYwRKP+Bs19&r8&_w0>B@$>kI`wh4Kc4lhUoa9Gy6IR^o zVc*p0P?bZRUHiTEic03>O>BzT@r_dOncijwI#{fv5;d_Qp1w<^0aou5-!%;{2)h#X zuFG7v^jM6%tFTZ-KDN;!H_x_YdOIN&Sqhd3w2H7l9!qn@DfM?tN9)9n56nMdlU_fHq zC*hik2rnTmrN9eK8_U5+#Q067s0~eGhCyZ3l=>{B7^~3KBeUog}#g6`LH7Mff!Z{?mI`5JoLb zpR%Y!T`?v{Ov#=I-*S@TfQqSnh?mM$7!vcQ+?PKYY?~l5Wcp@&Y%=-G8(@9KsLp76 z0fxo3E9>ajb%yjxz$;H2@?OgS2t#!I5nE%@&~b9R7q&9r6ls1%?&O}GEwikt_jd;= znpPv-bcl8j)c`*OvEQ@Qi+}5bkP?wK9~DrNvf#GdVAW1?BjUpj4P1qqV0Jega69tv z4R4i1ydzT!CmAQP>4;YyBd;tF%uZzhpxC55pssZ&-`Y~UZWI7LK*GNhRs7?+M85Bwy2={tmoD-)dCIlF8g+=v z`bW}VHAu2_*~O!toS?Jp%O~pQ3nv~;fj`Cr#ZouskRLAZ%TX2nt~Z!`+kN+#fpyzT z^{P}OC?KsfPLX^4<#T_7ptSxT%Auwbk2v%+Nc1 z#3WHr@{}|N7A&M6_BTsVPGf#`!F8Cq$uF4zyy@`4K+O>~aRR*1;K`iAy6nFbk<<6m z+ZreFlf+28Q;^~gzyHGPH`Ri~cd_8v$$I~{9odCQTk$p<4G7P=NwWq^>2B$b`!5Jw zgo2JT@RB01Z;bM6vqZr^W{*ZDPav9>dLhOsFFL_vcgFa7RHe1jY=wn1!o&p=n`&(z zLp=Wd9o_Z{_G?y7)`8*ZtxY2i(}W5qSf64ObwdZ{UjXW-Q3iiGFIl)FPe7iu!-bu} zdGcK(a9lj6Rb&v@yB+(lDJ+q6QdmKeA3T;79@|Nw6e3B18mPZ$7ynC^SZip$$yQ~L zi)X^iF+qB=A6>R3&)u)yo9Oh}5YI5FxsoTAhC9_Y5h2ar=M_e4?g#iuoQ{7bl&r-| zoO6jnt%koNzUNt@e^H9VP<3inJ^0R7b9av#YaHU>?|VzVWAp-zNDICmoF>UiuD3PW zH2WzV0jmr@=Vu$zP1RUWZ?h`TaoHc5lt{t$w@=3t_0**6VXN(wy}U&qkv-BM&C+#M zA(L4t`4utzSWilw+WXU5hjr6?b@OF+-8!E~238MEEo5PLY`k=<_08Xj40vfaH@{?v zytfUV$s}pt`O`}|$F|6~d|pSKM=qtatdXR5cP`16gf*4XiC(Ki8ase3WY!gh_Pxg( zn$x^=y#edZ+tFZBEH&PwXtrflzql(|P)ux{h1sC@Q zi6;nx67z?cqTbU3aS?Q@B>>%o%pbZ0cZV*Z)NthFrQh5==_)gC(TcRLzl0xgfzRYB z8K!aKo$D*WtohlY>UFU&ya-rjl5ULJ$lm%k>`tMXw0kC_4DG2(Ow&ryDoY|%oCwB*ulGv1Ww)oI=JVdj_(9{3Q z^F7XOW`!RCB81%X78l)v&TnYDgfLQnujnWM&mCh{*B`;Nls_NZiIM6P2bU{lRk6Q6#_*^leAj&#xlKqNAfj2($TGO2sfMkWYyp%ojc|0zO zU($i>Z)$zK&zop|{%_K%75h^}m5JnUv=71Re=y;?&*qE;(BkUQNT)0}i)Ey`6 z7v1gKC78EG2O>2()mz9EU=Bu97FAnNyccHSsV1Mk8%a0F21H% zDnHYx8x*xwk=u+*))B~czqdWw8aoastJXCZ3Bc}jyDl7Li>io;TGu^tDu9BI`=G>t98C#rurpTnLO?FM?19O}zzVnvW)iX}WX(cF}h&D0dPjj0h zY?0zJ#B!NWHb0YA(y1~uW$o0L!Gv+y8MXD(Bs;v3ws--{OP;8&^qaoZGe#Tr2u7#L zTyj9B9k#KLM^Gr#x5U&hH^x_t_^oAdb0ZU)hxa(=i@twjhRD0X#P!N;xDH`5;e*V0)BL<>3S487rRxunB9(Y} z8_Q^7ladvcZcxTgu=5XCcxkO!6%dy>w}uWm2C4wtKwq}@t&mEX5-F|!~^%{rJi36gXdU zT7Vph%TDO4u+Yldb=GBnF_idU2@--JkdIxLdFJsf1Td(a)pc)LxqkUe5VMKAyVpFSwz!x)~8BB z&i3@Pr&xcfOqn0aMh-R($X`x**FJ}~H9?M7M0LFzJ##FSsX8(io%8cvl&XEtx4*Rz z%tfPwlo#ryPbDq&qTE3a9`)X%F|>R_K_6f`3X=dwm-04}EKcNG_)sv9)@L#( zrSwB5tW*Yi7y?t14p(f=6dp$!^4T#jP_&=nm0j~btgbV1u!2!W3zmG>V&5l`;hW(}l(XgeKjm$3G+6{}!l0<#64XJ<9la|i&ecA!4b3pEc;tG!g zwEx-m$WHMOv?7(kC-IP3$7yJ$-L{vy;3vC6lp-C9C8&$=ZR?*OzaY(2<)=AjMbs~f zp^INIsPe2JTg_1#n{=RxC0ls(!$5JytuIWdDad_oQ64dO2oO|0Gx%ar*dg*Pq}p#v zqycS%X!G7)C)cC0ZuUC}xGnD51;e2S6hDpDRhHlcJo7aXNcjgPT8%O&tnnqQvx?=x z(Oz}b<3J03h~qVnM5X{NUwaID<$=8uEG20^MU}B~?B{DWg)dB9_ny_Dl|8bt{ zuF8BK$+Tp|m&XdTC}&f@n&X?(!)v+QoTTfaTcA8ZL94zPzA0VVdqy z+osX&d+9%AGse*8iQ~DA!a?8@dAFy!zo2Ag@3b~S>sJ#zlzWJuV?PVddFB{8My(Ge zNG1MW{=iE@2bqQls}n>OR)W889%1V5g~F9~A1{C^bAHc(LbR=(9WT*TA6WwvgBG)H zzI}%+a~D-$oqsg1E4dbyI;EEr(O2yFQHe~QxIyv#G%~?Wk72rNpE*Bp8g9G?Se*H` z_4y5NVHPQdIEJuh{k+0!&{r<+Pxe0c+*`(l%`ro3KTlO90K=<6NB^rz1S4!IhD<6Lo2$G6W0yQ1+k&Mb;5EhKwpEBU;M?xSo z_OlGd8&^l?&+?OPN!4Gg*6I>?$fFXqPmT#*Eq-v&0V|cjz~`S&R3hL4TL*yVdg6}t zIi6+t%YI80WVs;5u$WPo#d(Y3sxJyuD4WvuD=NU#vcjV(Y86z{P>KPZvZbvg1|0cWusYyBKSE^SQ;pZphOBng{xZzr(-8;4xJzZV6dJJPYO!z=ZOOR6LBqA-g+0jfBLKa(~I{2zuD~( z$96_a37ShHyk^M7H9;RXzRL`hF3RCH~0_U#i+>bN<e@=Ls8=n#71c536Z zjZx`9!?Rr&rLz{zxVg|jX(2i>g2Caz(D8SO%Q;q1w^C8?C+2hoMk)&w+>xT+mpKKU}EcQOXYP_eGlI#Ewi?K#p! z%@VKOUhJf0worjdhOU~jR*;5IYubp0w_VD zdl<`iZ83aC1-A@XHQ|E2r=B=!cguWEDenz~Lx7E!G;L-7UGxPT-8P*n+W1+K{~^R0 zR@S{bWtGlX&ql*I&c8D1(S|>3g(o-xA8Amr-Ys?F1qM}a!yYDCE9O^yjq^l+W zwJS2qmBkVMn|-jR75TdIqJGePG@7)T8AFHa>@g(l0J!A3v)azVN?*M9@fwdfQY31qMYqi8@4hUm&}Q0(;zYlKiX!yD+cT<=c7E> zp7#vKrJ*dd=`?lBf-y1!MP{~wL=>z3V?|-uMQh+6WYwNUjn(zj)9q=pCnp6y?ZS z=WXmDeo4$c_gG)LbZ<&qJERK||6rgbxg?bCaNP>j0yQq6!AA(Su+jtS;{bdXubh0) z3Km-F+dAa&jfL21h*$w;A!TPOEGJmhA`i|P2VWz)eRa&yIwlz!u9jVOh&nj5V2#D% zne4*x7>*i$3M1kbQ2dG%6!!C4sw`4tI5p+i2;lAv>B`IqZY&EgLWe|+l1yOHB+XgX zvFmDp2<3ZlBDj$9s|Y(&rEVL%cQ3fgKhTUn^Ib)|{)%62Heyv3t!@&9_AEaw z0DL4AJ2(BwGq`|rtXnVwFn@56rANu5WJgOIxFzHAq`Pazusu5yrI1=?d(1OE$#Twd zep`-UXDDzkeG&}RQZ0WE^#Ta8c_A5o6Qv(GcJ;vu{UC{>R%;C*O5MX>gGKEGs~T}y zOs_wJwIAxNaq-F#iYj!k&lh+`0Bb@9D_FUYP;J4++k9|s)sCYs6`5T5UP~=O(7)47 zPXyLI3e4d|FiIc`29hQbp-X+XVmkP-=M5#!2XHnkd9u(i5R_Oy%v1%&vYfru$V~6f z!PlS1^@w?ac~5iB+$Lui`wpp9*^*Cp#F8qoChx36$MWqGo{(6>LZ{^D=)ukCqlsAz zXHm`IOA$T}(@jBD@T5#bB%dCtXI06--_<^f(9^K3HXy*{=^iX{5Gk!n-+|8rZD{pX z)f1uD<`AP`^Wt%YGfj`*6MifBKTqcU9{QT*7ik>;@q^~6$!4C4>JGcYc3IFE{cV@t za2Mm&PwWR*M#eRZ5)Dtg_vr`g{0=3_l?ZzA+#u71v;q8Hq#579mJ?;Io=x8dfLMBs zXO0R=ZJ~Hafjr4txM_=rtw`-~QbmW-PL2}^OXQpSal?#FZ!(4$2`f|L^JLtP8n?3R zP%Ag5%;dKIY#G;eEjKqoROk;Bm}e$c$Z1=7GKQ@#LdCc_!fKdc%QmiNj&U2{)*apv z;(Kr~Bb*4xARIY+en>J5q*xw7*0ZhW!2g~mHD3Mm00Sm7RmW9{AL?h2AV+gbW~SdPNA zj89Cis$p51px|!+hhFsM)$axAj#)4R2lVme!{H41Bi94-(htwz2xVt+pS&cVx88%s zm$PzALZ$+RVPelTBa)c3V3dGeZ7NN{Phi({XDDM~Y1-)CYV>6dkc{Rvp?R&*h^k~# zcTjezzpFSdt*~d&Fj@ru5eBv9bgZH*&zpkQ?b98WmFM)!Ib7c=ZKSw1qNO#%QuBN# zW1cFi^ca{v!s0rK(te1UfpSo69mVpsr$Fxqog9OSGS@9!m6La6*lL}iY^aM9NpB8| zOYLB4tM)eq!}orIvR1ZDAzBAy7kIy>xdC6OB}1 z?>`e(^>a3qtkN$_-}#5vpU^ZKLF3eVk%5TmGz3F79vYLA5a$V7-r=Ecd3@_tj}+@R z$Euo+MjJO-fjwY16FY=Amb$)c@nuY1r+GD{CX{;RPKN^sw9Ci3&V-6aCpUjmlJ{ZKvZI1H*^U1J6pI3p(hff1RLkRc&nd_QOZU5`*0`9|&MbSQElI#V!Vr_Kpfw{7hn(exEB*osh6B#CW^*$eIH$ ztZwA7`3No-XC9aXhX=+XOLWk0$s1HHXgPa^>zX>7SUqbk+DJf zQeB3%oTFcEt#^84mgRBR6AFxc-<4m>3l|e))grVh;X^y`d5#`VVE$!La^^#QW$v9e z&BpxW*xcK=a#jXvQt9PD(j)CQ?8EF&odmvsZKHl!S3iEbgF}!hP>;ui+a}MW%tBs8 z%VH=c-1O6*)=}gZhdH(* zl~CYH7a~3+wXouIB0{9=3JxcKyTP#$0?eO<$`W*do$^f!hn+*FAY+^s|8Jzip>jD} zVy%WhHPYGHd7@Q%ew0ayR$2L@k3~5XhnZImF~IAw*O`F=mWSIBeP1PS3i&7>a5k8^ z2H22pjVIM;WV(0I>vOxDF7gZyvbToEw=5MVzBW+Jg;u|~GmFmrzq~KOM64~J-Osq+ zx7&9}&%3=2s!|!2m_wcUt`h^>KihDedSIY5fo^;(NO~e*Ei@M9CB{b{A{(LXoXWqO zjTuhq2q*nX9<++@TDS(`8qh-74K4)4@@RaM(H6Xp&Dw_k3qTXfnRuEPV^Gr&Vg=HzQtxz zxz5h8-T#*kbwK|X0J#mpFJKHx>Eyv+{o=3YPw4Km1=Z-um=9XbwN$}b?Lm*XNkfO# zndG)w03s8%@NOPs(;pz>6+;XZxdu~ZPoD@tTwt28F0q_05O6InlJN5!FE|@gKx&ou zK#tmH9^{=&XwHViXq8u*GVq)rKMMqZT<<>7?k=4%XuTs=r|pzf<=#T<${&FB0N;de z66iRT3qJA;U6>LXzdMfvf~!$_GwD$VOE{`KS)FK3uacZMx0?+s$NC@2m%{N^A$o;I zqGTUhsE<^$2~p6y!q8^1Zh~4WDpr{?Z`=?twa)i@Ycb@vx4F5c(KSB8{A_=_HTH{{ zVh6vfJ5Vq!4in)wB^UY~1%gBk-H#dUXoa~xCK*FUTfB&Oo#>os^S_7i^aA_eBfYu} zl`!3QRO7X7*y91&a@m0BYyXfKROGavyI!CDo#p*hr0<#Or5SdScnufBsBNzQ;fPbP z6BW^aF~A+VmLXl$hB4~0R^vzTb^ttTk`?QtP}G32KatF3kR(%q0jDNAgs-e=NoOOU zN-o;)cV&Kk(-C=BFAnS)KG>s@petwpp$fGf0}@oW)~zUcT;X5##aYbMu2uS)&zp8o zXd+Pm(tESLWoN^cU(j+0DJkEOL_TssRaY01s3AkHu~G)rnSYs$^VX8Ft6KCJZ@{Gm zp2tvv1&oEh)3BJ2?fr}EsXr(>-k|@sxmVVoiepoyys$4gN?dT|-dsSQlnEc;4wIM= zesPMhl3yPKn2Z2#E}o?#_o;RAn#g=iH3w$Ma%BO0vLxp3KY()0V_lk+GKf^ciSH(K zOO;>80txY{&u>W1eURC)M9`u8c-FQP#T@ct+>9Gr6aAR-h;{->k}Bo~2)Y>xFYQ4g zzQkpid1I^ph%&*k((fPc%=5LKG%lFBpMqTYO{2*^XBUAZ6It_JgL)t9;u|RyouN`b zWBgmhecesRZYSs(tZxp*cb!5c#PJnCFC29U#P?D6<0;Q@(o8kJfOOPNY?n3rD4tL_ z`@@Aw`hY*S2BIE zDHTpdwmV>J)v&y2Jj!0s8$Ln&9Zg)H;v%^-$UNh}0Dfn)|Mh%BIN?BQaSOwc5;PvJYPOf5lakxRk zD`>MRcTY#TK0=NWb}om1QeJbeQ}th< z*$l3MaTm~M=IVz~4D$UiIO_LrtoP`A@CN_(_}GX@F!OFws%xL3rm9+Irz^=aIep1pmHT`?QwVdD4YHbu|mJmWx1hU_}5mRYJ1 z&^nABM3_t2KhpQC96rF*FGsTS>EZ`D*LxuXY=O)V;a7LBp7FAc`7@pk@jmp#gNbRU zMSKv_gw7+oDc80M5VA;NQ{D?AuqhAs{t*5Jb~98mNi#-ZwhHR9z@ZIgQW>O2n~`35 zc=%X`V4nIBh=S9Gl>N9D)+gX>DTmN;_d}1pcFVj>0Zv7PcjQ8@Ok1+Tl)XT|?p$|E zAV(l=>u?vi>FmXDNk5C0{g?EmCj3ayG#m%Q>#;sdaT*1gwK@CPIyo3XBz_Nm$JfZt zwhh5KVV&WicqC0+C(6Tu-4GG%^`m&79L*LQD`fDBR3xCSoL~b)?SjgVM<((BAsjzS z;lz>(Vq)BKcyNSg0;MGYIp;8n_A81A-F_X?I4rj{)Pzb%j!+@JJdEA4<8usOeg)f++Nsg z0;b)v*_&Xrdlp2CjDblIvAkPQQacs43y`G%o$8|pK8Zk-tVK$)asgMn*E!KO5o*S3 z(>HZh#1%@U6JpNF#Z|$on6&jca)a#ZZBdMkFpxaY`TC>a{mQ?Q9;=}#@K|3w^D#5| zgQpG%KS@>fn++viN|^;-f7>p!*hN%KBe~G~0m|Gll!5fX$@llPPH&?@&<;J{zbS1e z0jKZRK%ol|aQ!1#I3QZ|GAHg9-*zGYgkW9HMg7fHP7$iPLxNi`-JMO>0#UIcZ1F6I zAp`JFe*-y=>H8K#E}Vn`b3@@{_x&%HSr6(g$2Gf~y*7qD6tZ~U$WG@f(_k|2*P>5v zr^xKVF4RYLCLvMFipA_Gsw|DvJuo=^B3rk(zJ(;dpEQ9LcA=I;rIg1FC#0JDCq#lX za0O?Q$*P${Gwd$N?$Apq-(bhVA)xRfnGb=Uch(XyPEyuY?1*-Z7$XTuu~b^@1rh?H zMX9@o^fj?Ed$YWmPz{pPVp7blm{598_Uc7* zWT@oK*4JKv(DS_KrF(emS5D0YHs_a=A1lj3@7i9BE2lrY%BJ5leiGTKF$yz$1m9q> zg=h`OqQg4}gtg&eiOyV|!=;s9cTF&!t&ZE-;L)QkcGKKIz=V(VPMZ_KU<<#xZSCHO zA~dcu)B|8v01tD6iiStZ#So;c>6y+;~Zb8|M?RRRGbb-pk`upIjGF1$hxGK0Kx?7AT_b+MDd`9BGE56IUWR+ zt*10pzM9xkuVqx3MOoGZoZso`XfBltgYxQ3Xo}d>)_9lcqg62zSfU%sS4ouxnEpXk zsQB(FGsgBg-6TS4j|hak`X>Lrg=l5PxN^`KlmTu!4!T<{8_N5`k%&AIv7uu#Op@3L zKrpUep3}D0m~qtmKO}X>pbWR`n0q(ls!wbBdsPgu^2%BmnRmGtd^JG%OTTRvNsMxi z4tF0DMLA@wQn4txjl4C(?Irq*W{3EmX|n+3pjH7Ij%$E}?oA2# zhjpvsBVdZUH~RByq#Ga9>9tGX$>AXI4t}m7SZ`2cl*f4>H?AA5?KIC2ojHncU&~>9M8m6ATz-Dp`YW{Vq9L`!8!+Yf}s*A?rYIG^^KL zL)$Bqt%#OwX?+shMzqJqY^Zzb%bqMp41izp3S<8^V)Dxwm6jCxV95(SQ*U zRV2=NiHVN?+@6Khf#uzTaRjyjfEm$Kz( z0I4cNTzRN*3w9h#xm%;UiT0YmRzVmDQJU4zb5z2i?b0F>n>uIaZ;@wyHdWiQ0X*}W znTNNmlBYv!?+haHj-K?3rE#gc@w{w-VXjO6VFdV)jezC;ks}l4$4V`KI!kqVko$4B zC-QmaIUK`72nR>vMm*NJ*Us@>)>Gfq?~2Czl>djx&bub$*16W{jEi0hVa>V2&q*+C zXAXYpeFmKG<WpN>Y(NZk2F>_AXzL9BVO+BeoQlT;IhRGGS`7k0}bjL2PoJz8q? z1%9>-y(v3u>P5>t9V2&FEt&JkP^{1m{|hs&O|#Ff4CCtGvq6wiyh>4`qke+l1M@L-4_G;`JR_f^?!>{4ExG~`ps@SU2CCXB2Sm5p zTz9F8|oS~?WH{oCNy<#)FBE6x28I;@n4fGrVf5{m8K;gKL zc;W0yh#z}^IOG0k;Y%Bl1WcvU1=T39(3XXHn$pF6%vmCF zKcBAJT48YlflTYMOs6>omrpnj+GYrCejN(Ow1l63H+Gn>iHUk3aLkQxxu4-QD~j#1ITa8DN;mt6cz z>*9V$96lN5new%~AxMpJf?1&fj^dH>Sj|N+bm;P>8nNAP`xMo-^w06O0cvhtV?vDL{wH-PkU?`A_O z*~lf;dT_xl;LuE(J`rIJnshYGzsw%H%nSAGWS&T z~XF%8^{vQ+Ze19MZuRlGZ{Xt=-wOXml(1BN;6(w;sp+EZ<&j zh0g^kYYz&AEZ}|@AjI-CGYrF;ow-&r3R^7apF`pWrsdrn`b=fC7q>Xp##Cg!im+Y* zC)@!VeT8A%f?03f#TK05L?z!=cV>4&Fmj#2cs)w%dM@otDu|5%@Eyzpf^&pfc|z=j zt+lE25Hfpp5b2Rwt^Jb;k2vP%n~{iVY>SgH)bVf%c85~YUKFZhj{qZ(-o_xTTR{ZK zT_%;Nu*luXrEfw0U+4K}EX9k-We=d!iKDWwX5)9tj$M$fm79DmD|j2k_ZPaAJxUVoS>#vXD=6#4_GpsrTNesOOqty<<3t&|5xEbD`WMIe z4w6^Ekx2~sl-p}q!_@A-21VOUTVskgq>Wx>4PKv&Wh?Tf z^*b<}h*`lY)9x_Xvu&M@JgSA0*h^`&O*N|A26N@1(VzX3+0LLM=Q^18AR}POJ;Idu zYXT4Xrk{ zhGZWPdz(TM&+v{<1ME56;E#w9Dq1M|yzQP*4X-$Z!9(HEk%W){7rjeERa~OPA8rQ| zDVk1nn%_0nQTPCy9j(aFnWJdyibPRHbTmpHiwf#XYjl{Le3GO9!ZnS+gahHIAt^!ZZHxGqAK@)yV3n|BnKpu!>S z8qyN*K#r$9sQ40T_CY!bYX%S*yEvL)0+K;rOV6*eA}hrqKMldJDdM6N&aS7NYc&7e z!6WXW(jaMMkVQIrr8(&HW%W9@$jv1RUE}vV$%J|FPu#Z^%w~CLg$C-lR*-#s7n?-@ zsh*Wp7tHZ)Ch2pDl3z+*HBaWqPM#$i#qxKo{c0^mJ9KqcB#0$PEg1Q-lP79-GR3E) z=5D}W(^d? ztwDbL$qd3(i_j{r!i}h~w3Vy;e&o`0x{2QG#arz$W(TnGo_1VlE{wnT#i|4Jg1VDk z`3=v@dl=Q{*Mj7Bv7G5J8k{#j@!B}0p4`}d+?qKqJtpk9q`?}=Cw9sQu;075dS$qz z?gMdc^gVL0OzwBog>(OS7WQG%2-BpU2;ooiU>?xH9H$SoN`+$1rDR*bCX(%|-YgDt z7lK&nxBv1t-7dTUm`bd+qR?fd5nGQISL_d#W2y_>pOl&nfczx^AJ7f$3FU5U?vaZ{ zb~owtL&);4u*p5|gW4!D!7ds4LN>FgUN2lqZycegpy}RgtE?@?ivG8u1*|ndhOhk* z=gxde3P7+pMtste$nkRNOdxHLpp1Cc8-z<+Bg$1VC4+Y3>H3JSYbS1H`e>VckBU$$ z3^@kUPxRf9a%0}d1oCcJzh6Kyqk=}uPGbg*iRrbrifL!IH`%&3t~VDg6VXB1J+v`i zjaFr0)1de11AX}}xN7;ZZmpW8<^*i~D<9a&1jBjpiTi`b5CPeKZ;OR_VdowE6sH<` z?%x=}Hmlo~^3NPsEewn&d(ei-T9UV;MBbel43gOu(>QDCJ-uz=|31;RoJiOzymMTX zP*KSb-;5QT9Y>mPWEo+KI0_BIPFM)SDkn27K2c~sP9PLIaJT9^Kvg)xELXRKOL`P6 zt^P>RZLe@m53G&p5!9hrBii-_sA{=}qqj@>#6J|r6u$k>SSPphTfS7faYHwy3FOzYpq_%x3nZ#|KhHE`Sz48o{u8@&9A(uk^{Dp%mw05rd=(&^OMJ&t*8v?GQC?WE&^)k=Jk=@n3v?b$Q#uW$`sLAZU62~q zI+TCNu$C>-VtNRh*%2@GK6zqXlYNx%OtMe=iD9rI?#OQ;jEha!t^xb2jZ6Mpm}_KT|NB;I>P>c4vvV4w~XD?1I&Z;Y{2Cp_mYBy;|HX z#$x!?F)^%zm)g@XU|WC?=do7FL;~4YG>F#h-Cx1yeW9FeH#7OR+^|y0MU!r{pw&|= zt&7mb94#!|p^7UEM1rIV6$Fk)GvZDvp8(P=?hqIQ+5AEpi#xU_(PZ@B3;ppoAF83l4cw{#3qS5Egu6h37O57Xi0Y*fsBy~C{yu;7fL zp;2_O(_iU>8e1H6jhkVT7n^u_-81Y~iFZSOXzO!9B_gXUhi6`S$OMf6aitd1#5YXR z{Ry?tWCx^YQba05$W{jWH*xmzsUCsd$l253k`EL8#oA7V7hTeB+_bb~#5{kerx>z& z85ayDJ;7(rusHz7P6&bi^`~+q&aLV$=h=AhY^77=UK;$3?(?)*6)ujTziNXq1gRD= zE)*KUJA}7Ai!O7Eke$L7Noz&1y`QfuE&y3YtZMw6-9A+7|9kImOPu*nES#F8ta2~V zwt&Nlxb}^|sxb29pgM!rAe|djVXFoOT9lU&OkVAg(m7@v4l-AegJ$^8%<;?dn;9c% z8zu{-hG{hAB|$#ev2dh1olBwqx~gSrqd!nzm`+TRYEdM-Tx>&RTQe)~D!Hvm;S8_E zZDXo^`PwhISC|mXa@ty6z)}Y*ava1eS)m-Zj66A4mwjXoyr4Q4RmET|zE8!YRqRvW zxq;(=kt>anTQ0lA9-MwQlvH|%q`cPw2K#1g6}Hs^pX#d{*v68UI;adMskX{x?zM@OEW)g@2NrNV#5Lc;7Q>s$xfeoRdOLB3^V{X107~b9l9@33;2F zq52KVZR;A@2S5$jaacFJVlhFnTM_rRU7mnAS2b_X7eo`efHShNJ8~SDXUk>2YGE86 z4m*w}7n(vZ;9j0G#H)eR7@2rF$?_h8COm%uBcsN`^ESDS$4W{o)Vf)+SK-uLf1o*aEj9n{@+g@bS#~z%wMqC%k-)p~IK-&YKP^V|HB!n&^98oH$V7A@jQQQkU_MN9CtD36oA$Fd7P+0#*l+Rn7Y2*rxQISBCjs4d zqWymRpq`IpqU+)-{{1pp1QR^)nvsbR0|F{XwI_;4d0s~VS?MT_V2ubIW86QWK^crh zQWgNYISI^7fe#@EhDIKv;-S#Xb}bEn5-CJnTvz8nvr6~zd-?O!m0qH(T%2dxKVasK z@n3IsYDzmFtYH8Yte%8_Wq_3DivzQ=j*R2QBzx0pm9IGz?EjfHa3JeQ!dn+x@j417 zQD2KJzN!w($zIvayp^=1w3D)-1NLS9ne0o8JlwPA%xO`N@{xW-5ZCTaTD1GJv%luw zjx?6Ne9I)-v<^^D5E$rQz=urNhAJjE(4z1`vHz7yXotdMeg(wxG-{ziOqU*$3oY_E z^tE2j@a-H{QyBU6OlS+V<;{(==xVFJ2Al1mQA_e4RaoOuiwLq7w}`0jGC;ITi80#- zgNal@o!nYni}S_;c0E9GIN|<90ra)S%zB=^hsc{TnU2|H)#?E3LT-+fxnWmaEm;tB-62RuRi6E&Z6%( zi%!vyz7OYU0|Blp9ce2+S0@jvR;-EvHk1=B+l~v*;5oHhPfLklx*2nje;ksQBXo^x z=E3yhG0sQVbfCBwOI7pQmtu^d6dg!ODqRbJ6;nE%`bad#5ixM2y_Qd`@M{4@VtAM# z%cWqXF4Q?otc<57DD?F2GI^dX+7g5v|2|iwtho3gDc@&Ph#6`$s79TR|JSvYQ?_-3 za^@324`d6xyOVjt2bg*(V;l5jH?m%{C(f+^! zWi8lm>0}!OT$FC;<2e&E?GbB$o$5hOoXs{U-wHsYcf;MQ`NO8rs1p{^&Q&ut50$RC z=xhEV8IqLQKBYN-oZ{i~IpszfgiIl7B-6mx5s->!j7m4Z^^^M%%L9(x+txas2b}o1 zmx|eSq;{oDTMihy44NI=`9mB5j4BJJ;H&}Xc!V!vw_5T{!3U=y4QDtryt_2h&*h(gAX%J`)gv9a2jS4Uyt2u-s zRK+2J{+GL$1It1YT@#5!+P6L93J<+`kKeIagkqoFHMGkG-iXgcofUK-_k=_K5VT9S z^QQ|Nm#`iV%w$|Fv)2ZtCO7nreauKljiV&?7j5I$+JCDkoO9@c52K{<6Kn38th{f+ zN|9IBQzaOMvII@LRM5`;SMlU~SQp@oi(wQoDe4{`Qd9ksH|-#^G;P$xo*J#Ro_u`H z#C1TM+^~qohSqBC>vrP$@d6O12@txlhZTUON`8NCGy&ZAuO;`z)^z9oF)P%^>5+&J zLw6b^oMJ7xsjO_`Eq%QsjbrhT)P_VabKnjvAS0$h@;p7+CVs@%V2H)Z)KWwXxyhIm z4a=Uyp32+34jI48K!U_JO*X|2I>bTcO(WZW0{X1m3CQL=w_uc#)?linUVP6PJ$1z1 z2Y6RA@f#VXGQk+I5l&SG#{?l=0lxelJ+xDwAa z?}wV~l=c2ficelxCFNoWI3YfQg%jES>!YAsA4+)SVd-5w2$xV;Yg@wiKx;*%PW>x% zmmHVO9;@hI!5y4(y~pae+TvP;G7YGcNq=vJ;R&!0C6G$71U#YG=7X~kFdeiw3zSI_ zzzs8(T-9~oUT!y4AtS6Cb>B&yeDk4OIIYZhQR@sB2dcyPy}=LTEq&w%jhD17se%G} zlK~L108Qg;*r{omDI+chYXgyW1hVfGO(v$j!lEJ${3q~w$T~hY=0qF@IuLla7dwpK z1se|QnEX8&sLn;Alu`Lez&7WxxN6Gl2^i>UkJ}QE0dlSGr}1|8$=WM*S)SpP79lCH#OTn7?1UH<%38QK`e}PT~fGXrVfFpD=9*h<_KOVpG(HGk98gS4%K+s)Am1bqXqY?Mtq1HZT;E89DWVoTy4{$DvZjW4 zXhWYWa^=D8*}o`-BIAT_MqfQ3%BG3!FF68uCoe5pBF$yb(w?#LhqrL53zU9Cbm+AV_(mo5z zE{|VQvWQG6;2xyXl2{n-e{K_?(#X5=al-5?2&s4uggB8kWjPx09Qb$znZaAyMBkB8Kp}nnF**lP#Swlbl@~bmiT^k zAFQwJiOkKxJ1O$gn2-n&+;egqiPR%;MXB3ThFuVd^eG?z`JXC3Di31iL zoJ9%9qZgKJV?Q_jLU9Sqbs58h;~+F9l_(X~FDtW6(Bk>1>Pe2YY2OWJnSe+fI(QG`UBiGQ|!vDC-5KH3LHx<3H*XSdBo3!bQdcS8PxuCuq_OR^;W_NWQekfE2rf-ZhL8zoPdWfg}jsPg0pYrX=;PyZ?c9 z23K~PNG6FW%VB$>Y4yr{ye;~AJ*`CiYT?Cf2Yp!6e=J~Mc}%Fi9k$_P8$MYF4j_$S zM%nu?>1>YM0Wk)*`dP}X~H{>;BUle3e2zyXKiu{0`V! zd8Xy_@nI5ZjjB9AXy*~L@-ANVTB$vRaIVJ39~qSnQ!HSCip!{$B>v6atH0q?(1hdI zVx7f1NMgU&`eAKdRP1tt85ap-cItoh8mQdFc_PQ-SdgExQ=Vj+tnR<3W`i34d*D#W z1dSUonP-u-g&SHtWdVI4m!uiqhcr+1p9du9($FsNk2_t*nFH!ra;FOY(|ThDTQRKL z(ev3_=`R(}f%2V?moIdM$n*&G|etRysV!%|{1Pw2wi4>9IcCw4~NkF=5?mFc0G<>2i z#V{J>7>Jy8FGWtqJWC*XEu{$Y)jgIvwDleX@16`b$`B|&XGk`n04pzkBMIUsm)Gb# z;fz`?vYfnK8$AWua&-UkV4iCQ9G|Z#t2Xq*vH^x8y{b5&>yT3hQSo4*z~h&vtmH!t zhLcDh1lnUfd3NSGC7QaYA-c=0uWC6xc;IWumT{=)+sZp>}Aot)Sn|2Q$+8xh1$}8trO&yO>M@{eeD7+Mej|S&iHw! zH~}4u>bQ;?V_?IGVVVgjQ8}IxpDKOmd?ILt_L1Ysmm2mtzHSv6w*%-YpK#gk0vAWQ8`B>5T&w@Nop@o=za@`6Ms?92 zjkL)$K(M>a=?cE9O5rsDa`qqIVX+B!GtamjmK7Q|4evYaiiy_m1U=^Zt!1=oB_Z-u zz*fC`BOEN(`-Bmr)B5qFi#fA z-@NXUs5Iu*2s8fQIf1lK&tkI*4RsD5sq~MHs1AlHFBS#MU3uU$WdFqkceV3F`unbM z9wE<99=Teq+cx`yqD}>UFK^M<{|VWeVz}hpRXLZxA!~(!q9UbJ&uqceNZuBY1n!VW z00g97yN`vr7?Lkdv=G_!{z$r?@Xw#J%ZTgb4ceFe{;+88LDCsZIjhAZ1a?>SQ{-%b zOCq=t9OAY-ZX(XHLD!r&ss)=TYd0;UVCY#11SAtYRM;r8FHu2eqNXyKRQ)K+ss&O0 z%>kgGH$R<|AQy`c^{FuK6uHGh=nv_S*DE6deCk+D9{l*DB!Ox4&{Y0&6Yg#sjcd_V z8!<$0^5Fz9oF686HaU*FU3p0RC)6kZ>Zckh25y|J1A;ClW&r<#IQs5hi+3RD=I-YxL4^K2O~W~hc0=naoduM>hDvJUmJCr)Q550kCQ z?X~AhF_Ftbj=Y^oQBWU7+?|4I+-j0GE^TGpsDOQQIoROBUyk%Ac&DhkIe3s60I;;a z{aNc|rWh&Bm4_)K3jUGctpZ|C;w`HoQ_13`y;qQHRr4X-g>KAQ_gE9`vIiAQha`Pd z`J#FHZYaqpGwM$9uN+~D`j|dfhDS=1F#|R{$ZlmMx_h=`ZNk{?P$!l%jU@^ z6&?(^{GpmbeQc4H6HUpjb4RFEKOwFXsW?rxHKQcPZtf7S=e1kF3e$w>I#w@R2XZca z;Cp?aFp>FKUl?_P3Y)+)Dca((^X|niw>Bn}BR;aF2(z99;oqI$%Q>ricmn~99`Wpd z^I)ougqOp*-OxJ;P+{4UqK7dpL>vE)j6Mnup37nlh{|3`!5a%Wq%iZH00mtYG^OUS z@((YFv1ulck=-M}WCX(3FnVaE`eQ9Ge~uQLJa5=n4i{%fFg<^dx(DUE(ezMyMc2ZT zE7arS9r4=aL&f=3Y^m;YN{*!FD%c33bwZ{y7oojD|A<-RVPpy&72+~{&S zSq^(SM=)*LRzgWbCTy{aM%C6^!J;3c6h}^TgeVEBQ+J#zE>t8akOz1Z+XR!saf|l* z`!~P`rGd!9%t0nyv?`+VU{8H(~1^YF7+f zh)ZJ)!g&g1WC%Ok^a>mlz;UP8U9xI<168pGQc4!OY1DQiD2(_|ng4@Tskl=rTDacR zy?LzuINbT=$&D0{m{LJO0Ue1qUrGk^1eL$RhMsW{H167B=jB6|S-5m~Z%Ld&wv3y> zH%|kqAJNtjEM$bijbQILoT2E*FSA`3nf*EyEi9hqDno0AQAFUGObQ-?8*dX>x(-{- zUnBh6{rGcN0>YZ&Cj20eDlow=m_IUydw%Oc@KRkStXzyO%G$9ou!+B&DP!??V%h}>2}!CHmrBts zCs0-p3I6C2@rI3YLY;%Xea91H9uv~ELL_hI-Z{{)L`hmeDr(w}Q(gc}P$L;?AYvY^ zaPx$d>@`#$6HwbnkN%g_1Q3AC9?kCM{!JK4Ga!A8bhy@4eVW{}CWM{+FPW7d7!KAO zeu`Qf|HpW(0KT@fSU~0ACJ~a!mZQs8XP|)MHDDy2wo+2-!Lr377i^bKP^vsitH{Cp zVkEyTnUX=u;|e)ct*cWd0dK&55^c~qL+5x_sHfON=fFe&y*Tf{)lqD3eCMr-juThw zu+ZLEzixF&yjZrGUosN5?f(7Yp&t-f(<1XE6}E@wtxjB&@aDRrksFoSWuO|V2z8!v zx@20*<3o_N%%eM0|4OSHyFJx2`nHxDF!Z>v;E`BRl9={N3ywt=G{*!A$hJ4iZA$_F zxt1|H@eA8)5g*ipzqyChxx-bHSE-M>bgY&qX7urt1V@k=WydW>i|xS_^xp81c+bmu z=b((LOu@trjx6l1^Awu6{Z?@Na}INf)mFnv0oy+$aumrIRJj^51i0qWkWpurKM{08#EsLm8{l+SaCbw(1OyjBQ|BP!=qbhR}=@$0u`mT zvsR$i=npW`nz$JA9tS#y)oUPKC?uTmWwaNqIA51(AlE9>D}Y9Gos3w>`|&lL*d|L{ zs6#20G}c@i$@sJsJ#p^c*wl7AqW5LMhKS-vlt=h18gGmqiC(8s#@*?e%k`rbUhHk! zb*^9w9(_7_zBTIZ=T30__x_mp^P1tG%?M`siTB9iPL|C{^z~;)tUoz+W0pqctF1;| zK<0AL#?eWd0_ggTB(not7ME2Z`L9!XQdp$mWUBBAcxDs&wB#K`r}zA2N$`s#twhaj zJz;Lbuw%m+eR~I5oLODN6?Id_xiZBZ3q4=3K^+kr9!9I3ye4c}!gwz;6R2bBIH|0UO^o>cr)0j{wUs|^Em+a-Hp z1!MVOImm)l_EV;=J7)D$75eHp}74^I~{4GxzKX$D#b1;JuhOhVqK zv}3uK*}}f#1kfbjhj!_&^yLERR7!N3Czkz<(blFc6)Pl10S2x0vXk;@ju@a#AFac? zPE41s(|zCp$|EWYf>H(@S-RX#fJ}&Al>Q}A#ukclFDy!&>pQ(1!oC`vSa2tC$%K^A69V$S#y)_upRyrVATT=$V5u} zWP3TQap)zi&5&(45Yq{{5Sw#&pLg7#vkg9d)BOT5Zdn3*1oAgBlY_JXhA|ZRVJrJW zd9M2CugDAPywwsT^m!0Ot&vE>dgqIb&uWPZy1?>n^{dBj7HUXA8~{#?8B=O3B%y|e zufL!gFABzBu7rD=JArP@%82$?(*1CsI+vo!Kaw{59)+zLq`6(#-wdvc)O2Fp>{JsN z2eIuB$ZP!=M@rNzoX`@z7EK=t*rRAjHAtlgx4S}oNUurn85)4ZNMIqpe6Yo*i)dd) ze8|GP1HdXt@v3HN>9R1^)^05E$7GVy0u2@&R9o!E8Kb9t@mCOkx$^Bxl zSIS)DGd?b+on?>lRoEd5<~H_h#-cSmlMHUHr4_Zs+=2weQ%^XcO^9M-I6!-*Ol?8&@@)20q{ZrtJ%{>P-=^$E6^60emx@&d+65I@n5wkkT_;zdh;gmh6jv zzhyYrcR#KvUtwy(!2G3E{FJY=OZJE)K;%{OC5W3&pT5p0F~_0CWzW8B#k7HQL0}w( zgPBkMl5h*Hk$!{~NM?2v9?~>SMLUd6%`M{CVLm}S^1c~X|_cOuP&N+GR&CcnGmt`n}vkB$j zYGJXM%0{E@^V58dj!_7R8?ue8dDdg_(C9sAywH{2+-Et;EYcC*zyR@>#^Zf}AA!sZ zmX^fOF%tQNTx^|jWrY7U7uW3^V9HXuURpxqV}p&TNM3I2Lm6NM8#y5==P>rql%LSB zw}I)MT@-EQ{uIPY^X}Ta91aR3j0cFyrI%4i!}D~qz|)8%f5ry9gRY+99_9|8mQZYg zRz$g8JI3^TXs?l|8I}`+Rp@^hvyR0`ai^uVj>F`s0hB4!BDOXNX3($_*;OS~5iK#h z`q)Q>2C@Gfi}>3f)T=?=v2~k#`sDiTjL;k{XEQ7*?({Jyu9JbI@xERe9n9v~6s0B5 z8T$4J-z1Xat&u#{)Y^V<4l<_)LF)}AKvBqrRx&{*5J&Ula~hH=v`w%z2Qz>M(kqa? z=l+p_;lSHut!iylouqYq+M?Nm5&msU4ovUgb@_DgR<=DdqC%$;J|~mVtoU2}J(*o| zTxDohnf2P2jtDwT{hWp!-+^VWZwAM@vd3On|{W8!yX zYmhH$rS`N!>+%4+PX?B|hA8dok(xt-hh<~G&3zeF zKO3idO`XulGgi4(TSfKpr;Te>Nq|Z6I8s~6sqQGPh)yh{R2q-G-f-rtd>c2F-kZTf z#USL{`vb9sn^|3!e}2CRMD_|NMKo7EYNo539nfwf=RAHzySf@zx<9C>g^2Ahm}9B) zA*`*5G2@bq{je#l5#qwaS}hGP`WhDMG1t^tJ{PzG^UEn+S(PE`Na>TnfUuQu^Vz=^ zxlCnDlQxV6ht{w`y7f5@FaiX>praSQ5La~vE1~uGyax}J! zW^J20!Uue{%&X>eLkd8tnNUm$8mX*A6SNNJErLQP649{D<@L0o_@NzU?5iQ3>_+yq zLtX5HPhXe+C3(TBe0EVhkEJ`xiH6iPoLwZFyWF6tZS*wC1EB8nC7Z~gU4l|}MzR(W zIwy%8?6GzUmLPwpVqdGKQXW=|LFe@Y4JZ8PHhX_@ELty1<@2?Zdios3lzFJwA~0TG zFR*Yiflw8YSVE(W2J?RF0sbLIw}E-<#8U$h0EZx#?-nX*YTOt=h;Da2M1|$We5X{cQ~!?s(mKw8RCB6{nmq#@*vn=$~`L*yczkf#TXw) zVMwGdhQP!>*SA*5X|AziLO5Msx-6!di%iaF9B~Lji20Qfv>_pl~aE0iP8Mm1$x z@gNepa`9PgsiWPq+pf1R;eve-O94V;I4H16kA~gARQKq(nVYHDi#+h;)6VJ?yk^R? z?n)uN;meH&!grm$l;Vd;4p4OCdy!zTIg2xjKqHhFrHO6`fS4lZP`X!V+2GQg3bJdJ z2Q_@yE!;^92vJ}g{W%D;IGyG+9aKBFbH;LEeiyiA-Xw-Plt9wPR zV8}9rjJvrX5jA5WdwE4TdsidJnD4Em?KhT)Hba@?xLZcM1KON$ShuDSr zuk$&|f^VgZ1PuOhe?qi;-C)~CZ~KrFoD`1Q%mxK(Ef+6@HmliwMXr0_nQtlF7$5nU z)zT;A6PM?Md?@CJ(S%PH&~~|M^x5u-j4Xo4y8EL3)4W72MunO(@In+wrwowj3+V`5 z4=~iUK6MR(8f{V>)hCLU5tfwk;j$y1LPUNI-xo;01lSA?c+g)5Vm9_3rdhWX;qZ{+ ze_9=M`4JFtsij6fG^EB*@nD;4X@lknpZ{ouJ6;B~o#xDCrEi}u`{1j_7A0;+G@^DF z&5nNz^r6omaQZLnR!!1R#FJ19Iz`+PnMK6N{Fe<2>A&L|vAestg$dPNUq?+_@V@}9 z6+ylBS6t{En&Bq{AAUamuV(nI=4DFK4tD3f$$icMRVcLL-4M~7koRPSGCA|`bRj;u z(j3UsC(kC^>Oe4ZR}M?KS<@c0kCEhYAu7w_0XCfE`Lj^^y-b!ZHtlnn|=&T$>R~G&G}!w;4Iv3%jp| zH$Iz1o`xcWFqaW;-Q`DLOtrNITe(W!}%yzzfG86#>1C_ zJO4@#B4HSyThNpD*C@<6xB!LtX}~5ljWe_dG*}rsTiym6=>u_-=G%?=O^DC}G%;fA zIV_5fG++XfOrMCw83P!Np=04sIJucX!xLTr-o0OT%W!j`kLn^QQSam75B zJ*;ZQ65R0C{#B3VJqk9eki>{n;T~Ji=Gyr1>n_c5WH~$O8!?pDm;4)W@On_K#oe(_ zT@P4ymKByJh?QZ=G~kDdflQ)-h%p9jDv5`MS!!r9rI$xru zqXlCS{7_tWH8qNVbHm<0T`I+FUVXjxB83Dax4qNHhdKeah@A0g*;wU(o}* z{&aBLJHtTafw2X2^aKOAav13Xh4&PkqZ0YZnom}4Ic4?xhnrmGQh7>60Cm2S^7Jx; zhjFsAMyrrWdR@P=-@Q1V%__~eC`0mxAN2dY&sBxCG$f4Cj?slhBy#2nM{XT?dQa?N z;XB*ElbNf$>3YjH6RD;7Alb`51SM@%TdXY}awZMJ^9$1c;CDqeE9YN#y8mZ3A$%ls z!m%f>mJDwmK(!YC=#BKX7}12ac#*P^k)u2wGu{dt{01 z8UEI6{)@E!`H+=y5Mw6mY@&p|bt}L8ImV-@hp-SMA*&M+pmlP`dmxB-|Q#Uh#-m?Fg+TCcBNnP5e{d|DVJ5Gtk#UHgi zn47rh9XmYVfcS8{6a$!!p@NndUF=jVfR3D;b9qI`;#H6la&s))#>Z!aIHV}m8+=o@ zxsth`a`xBt?y$1+P+nme5A^!$A)vYw~OXD@0DGM zemPXq+&kiL{lId!*3>f()prfg!SB!-T*EE(S+vRW^NX~Gc~SKx1d04>1xP8}7SweB zA?aa6KbJkx$B3|Hz9)dNlf@(NSVkqK7#F z67+~2uyqUMTgqolyBP2Qzld=(vGc!2c0yIM&KtiWAXn>%l|tY!-xVGJbw>xf1B$$p zB>5Av661f2xmzd#ZhGkq@2rHNXhGdvPbvo6{H&lJSn$D3(17CKI4?jJ{o*YqOZ=$} z)P7j$g8^)9sIVLdTXGAJ8M#ed!c}YWR-o>sUDb(d<{pOMCDB)QQ$?*`ch9WVM{&@E zU%8U@9Hhm7sCIbsL}N@fv;g8dXLZNLOQ*ANS=}Cb>))iOR^6=C&~qKtV)#j$QkGC5 z+1^*&D)F9)k&4M=@m?Wt0|_V@z!`EG;SKAEbsL?A-}_Qifv=6t*>Y%LKvwM#x32E7 zn3LC30OU_`1PhC1O|dbbUqh`h9*QI%%_p3DmD6guEN9$P+J2_f=%if=trtv~;$<3X zaW*1;7vJm~r$f%fW%Mqih^wXY&jaL=TsHc37D|pJdDY1_aWcbD*7MGYTE^Ds__!LJ zsy0KUf;RP8<`8LdJ{_g64>_^Y4UimbLA302AV|43uY^U5o;O{phKuJt8*rcfTkW}; z%h9B;PYSK>o>)C4(g~>*!m44%H_fVx18_l0l6}u(;r=#nk}c94xXYR0+4Nm&I1=~c z7T2sw4zX8?;A+TdieW?fw{6z;!h1DG?6TlBs=3w`9srfFw0BPM8Q##9M}FN(zEb+N zi9XP+nwiu%TYJy@{SK|{lHFJh7fGFlm+_9G>zlzia_lYLfX?wHicHAX|xPUuWLVK!at#H#mkjI zB@kNCXi;k2STq;u&cFOwBWu+?yd2~fA#*T8*OgbJk{9qeRQGeN|1vM$}p4D@&H2ahj>R+ufe z{g(qCO7I_>eRmc}8Hmkpb!>BN@6J$_vj`gqbw{z_9X(qNR>Lb(8_PP8cHSnbTsp2{ zCx6=CR`Tm@!Q$9*!Mu4hu>PbwcD`Y>5{L+Xyc`}m0p;P32lB3PYjzg-TX0Vl*eya{`UlA%$i>{6|@HBh6JvTD;5^) zSPeU|nBF?LAaWH=KJX{N8?A}hU9`QS*A%~9M9v>JIy|17c}7kn8Mf%FDq_z_`(N#i zMB>K%07mGm#WQP(rM{niQ#@C3Zpry_9QxbC>+;!;S03f5J91)!H|c*qbVRb)r8>2N3Q$T!eyQI>Z=m<1*HoKx+?CF2pk zx{OH%L=zj;7G(cK$cKF$lP)8#yl$&D_C?tC{%AwoKHqB~%_`O*zbrh(HV`4rw zYBX-@q_areK0KhaHJ~@(9|vShT8JK4dhr!i$6eU)KwN)=Je>J?HMlFhY)K1yk1wa& z@JHE`phPeGfAkeKs!M}RA*MN_ne}19xnAH3Xkw3g2kScoBVNO2bz})cWaQu2jwIZf#|GH{KnMWV6J<(_e`YoubLvFpg>PUHt*`$YUut0)eaR- zBmM#awF3!|WW$VCE8bH^Vkz*L80tKBGpu6xi8IxKGDtXM6*QC^?zDms5HnQ zXml!`G0>7-9d7W_oeGZ|Fg^S>?6b4uw@YiJ`qNKMH z!};JDb{@PDwD$=&ZL+P3fi3fSn+q(;Z-88MRcD|YmeANPl|FS_S{M9B%hHVA`q@pF zH<=r=mMWgLJEB(wXX5|-Rh>5!cTGZ=sPba3?w$g7Q+k@7^x>) zF|MWKR8t>ujzQaIHQ!|v_S70IiD+Z^BJi~3+NQXmBu5ld$*oqYyEIPEi*s%~=88f= zdRlIL-DS|tpu<>Qx9Jj9I=6)G4U2hvZ1aqxZ)$-06cAE?K#s^kPIfhNHs7Ad^?SvHZ1j9TLMNB9Ehw(MFK^4hypN*_3 z0<4A-h%rI1C1#_3A@QfS#<{Z^ntn&YX|TKpah)1QcVO=$a4nAjPgoQ_=9{(tgehF#nye zjNG4t!|j#?+3X2M8Kxey(fJGSZSe@spI=0=Oe7ETD>#%Wz1PjOk0Ha=4QmpiR>)j= zl#oiqMZozaYQYl{pwxF1kJFdIO6PO`$=C%|UNG@l99;9e4_{Rdi>lJIh#8NC>S~X| zr3%+$NK4IcTDE{+&WJA0Kkw4p1(gyevA5VP|E&4I=PFt(#R_eOEo!bw(pA5Vdn`--Ruj`F8#?pUTPth^L}8p_w}eHNQ46BJ z+}EV9sz);9VcGh{gcl(Bu6_nXPBg>?m_HE zl>_Da74-~Y>1xF}R^^Vnd?%vQT7u46) zAg0voL8u}RBa#WHC`_FgS#4~Ga?xX_5=7$_^6=nIhy^z=!;?)a`I(TZyHg>c^`fRsKez2jL}E76V;;|iolT2AYz zHiL^(2M3p7r9JU+F1u&}ax4||iIrARaqFnDQml_>9$%2{sG_BV>`w1W%UJE_7ip8H z34D9-HkCRFH$d;i>PFeTY8Xa!rKJGbd|w1Ski_Hg9KgB%QsXvg#MHWD$eHb!_aiJ~ zdUZCMd1GP_?%h^Q#OP3Ak^{Ljq=wB^z^icnpKme)KVLAQVj;gX5;4wu?aKll zUxPcTfo>z7%*3JZ{q+4J)m?vj2A9v6N?D``&p~-*YZPVvoFp7Q?TPGPd7-?7F5wgZ zlF|EtXzGD$VQ+BOUzQJuzmQ*MNe9!ZtV* z?_rT?I^_FQzy0!=#xSOfq6M(b$akfkDknaC61HZs1v8o`{ zl-r6)Ew5`WsfibPKNcq|f9{%-!a8zKh;fdGd36>rp+M`??W&@zhn`;5DPACKY#%i$ zYiDP@M^n^)da7uz84Dv76#L|XmNAPvDH*h)E6B-GSlj7FgQ@=Cu6 z{(6xFlI@Z0_N%S%lsZ;@9cU=(2}kKNRLl?%rX}^QaNSRmvjK-=I~gyYLSG9-FNBB) ztr|8kXSG)igS??goi$Cs4OXb{h|i!t?Lp;hiT243G8VvgVUOS8zHyo=oC1$tON9pKuRltIxpIqT&2)QvC_8M? z8&V!tCyxVW)Jdx#YyJZ@>WN5l)T%$YDusQU@E$I)TS3)B5)8aRNgv#FzF5^lSW5ur zurL;pR3_AH&AT@UCp$(}ktC50%$fAwz&42-k8-6w%Qs#6=J|6a zaMN}s#N(+GXFPO_?*hI&g$ez5lPg13R?mGmswAg&<1da9zo*B?4)z>c5#^h_r?|Eo z(UnsT)@72}D(yYcPeUtb8Z0LnhzVsyjP#CXhVG2wR-Dpn97JYBSkEHz0JD5!s34T-1F#`{iV;ToK3(~Eh>rKAJ;Iu9 zq1FzvC}(1fL>8ULP~A6#KnK6o?Gu!8;dEH~-F#kEU0*m1=#78#-==|BX94u$%;AE{ z*=8`mmQS4&C1P^+b1F)AqWHaYa-gySbDtFbLKmP(6qaRuwPiX1?=SkcMIupJNpcO| zTkPc$Q7VU5|9!7|Nk=6D&bKT_?dSF+Jm)-=z_RjA)_lNHNlWbSYh61#@GRcu-|5#EKK zb;J4M{+^eEWL58vuS!5bXuhonw#0c`LZ3euF^Fko5k>#zdB_hv!|qh0ixLRBtrf4R z(RX62o2eA@WyPd85>DNZ6nF$rGm!0uInqd%l|_tkH2A2MtdX^tzLT47p|9Zp{}Og* zaj)O?kwAMl-)UK^1udIt8o`p#%WAM^H$le?muv>1MVro&zFGg$F^(Y?!+8@1N*`PN zkAQX}90=LD5rF``q2m;gk{Y~5VrG=HyyKL6(r*AZNFZBl?92ZbQnVD94l7=$Pc`G0 zDCY0JsxbP(?yz^ACMjqFL(V#x$dG%>QC94w&rWrQqh{0X zJ0nLr^tEK7bzdML5jnNR8a|3$EwJ9Ix@%z@TFM`*m>uDR0KLa{t|d06GR32hg}dRo zliGaY8x&n_;IeV{UQLc-78(ud9`oUj1JT9uiq@>82;u>)ZZh603#^x2XfhKI@}`Sl zJI_G=6mbd|$kH;(ce=k%-=|*y(#}g5ah*{m7_ABC{@MW5mW;YAA}X?&Gd|Lp4JI_IoNF;@gs!)Nwzm7-`!`_v=$7L5Zv*65`uQgD^#l!vGEE?#c`?88Ef z(V+-azP^>&Mj=r&K;c)>6B+Zt-ZL|1`BD$le}jaKT5l)aI6Poar>gk2))#L#UyNtO z=iD_`Qb8?Fc^EnQzFcjD!0xQJb>VVgk|tUpw-}#i!_G2?nHP-uqIF~w=U&dyR_gRw z<0X1~C2~Oaztc}t4sAySW)RlZQRU0*A@_fg30*;cOr7Bs%LO0+uWSpD9hk&L68~`I zUvJwDW!e`K%WcERdbB%JPUh@-?mGIvpIs21kNIXAn?T-4$J@n3^#G(&K1FURg-jJ` zn?|h?Zr1d=*8ec2PpO71YQ( zX}19-Jk`#pkW=f_kcxf$k4r*t!#}erJ=rjcyP%Nfu6}C6Z}@S4uej^3?NHaZs>Fz; zPPM;w{`6mohNGQI>FLY58AL3;W%G1t26vUQ%rapTV_+9lXcZ;WR(=IKIp$lRA zKFV`sL6>}WzFGWFjW{Bk>SS^-4XOrElE3bG-iOEV#a+-Di`e>>hWGs=tX{tI_45mi zIVPYpy#dwsK6UKWm3hGdoWs3vxHTDb+ikU2)wIZ&LWbDVwbSSSPtojkTt7dyvaWMU z&{Jy15(;@+9h(t`^?64ag6vf|UU+*JN;qEL+~C3#a|XeQomq$p-I%0Rq_~ncP)cXW zG$BvGlnT4_5pYspiO|0xLD7C_XH~8*1 z$x`uuAKl-(LS?cXCDNje-Ar`aw;6L#L8>}+`_db(aUw`Xm0Ha!wi6(FC6X%%-*mF# z_rtlbd3buit82h{?-T-p54khp48(P@@0qiDFag+D*#+*|lH#z20=IAsB?wHm^)0&) zDg6hS03pJo+r0Bc7!k!p2I}9PXmSCR`4#N7^>-)K(eE3kul#S>M^fVNLk5)n16_0M zAJZi-W?UuqfhRu$foVknQSLf1tOp@RBd#+b%E@h)wTxpU7yWrevsX7a60({i5&Myp zuVs?qEsS#rW+7&JwOVlr@Bi*7z~%CJiNIV=)dGrPzSa0-eW==mI^X%>K49xE8Y>qq z-tu25yojWElgq9po;Zk#!;oGnYebh)?^p5MXBNb|{B3U4a0W|$ku^$1fV+|9Gd@Va z++jXKYbYfpT0ens2~@9nydVRP-B;2l1M;CNnNKE-5bemdy>~2!>sOcP-?^%4X(+Tc z6XVb2%r%9TrrRD-CNDo<5xTQX7ILTvn^&%=dO!5~?}0_)#XDD(XO4W2*glP`Mvw+a^0oE zDk44whK_Be?hM;gnAPwxQJMZ~st;Ep_(vG`l81)~*XD^=8#8hx-=iAw@IU@}K)=nW zTq6fPIEyuqT!NA_&1*<3J&AnN9Q0e@+7h2WU`LcVs1 z?!6F~*i$c-qor zu$Fu-9l@^5lMBa(a(sP+adxI2t8HdM-KuNublk_3vVqnVX|)QX)KyxBLWppow}V)h z7fq@OEHzrjPku%?D)|~f4>5a$=^0G4c4`-C#`VhvrBbt5aeQIy%92pZLb5ran}Rew z`yJAgqs5t!EQaq5s>`mk8YL)AhOc!R)QjIzl8`~lXg{_d+q_aMPJKPW8nV2V?i%dD z7#r55D`p^Dw>hk_$5t8a5bG67U)Wgfn$xXJTaRL(Q z6DJk&tkBFykl*c9$6RF<7iFOm22LSscn0`fNo76`anb{B67B;`NzrKRBVj+-ilB-M zLh&FsKmGh4nlE8%rJ4t2l_V5c)-{t7(zxAn>We0q5nn8`QcS=@`$)vih6y@Opzv&0 zGp_Q5?*#o34Rzy5IlN%GNBbSm#sfe_c7{+^oR2O$Fz5q(f0J6e-)I{hHQZsHs7aF{ z$>d=#T@s+3)P+3sbM&(4H1pY^ObiF+Y%X<`A11xcVLTLH0ISOBnuA zr$Q3V|MhonszLF~xaS7>b1)}R1b%vI+z3Q2Qd8F(*`wUg+fI)LxoN|~T|Zc$QNnBCIDFO;fAJ#4c-WkIW&ySF8yMq_2usTM zS)_xe_{Ap`N->G_{Q&K{S6Tc;zy!;TC8__^l!dT3$i~zX7BUD!ln{<1tk9aZZ;0^%10Fd)+BUScvv(0#DbTh2lb>4CS8ioA^>( z)pt;mabVbTd#Qk{*52KBp2-g6o>?$39{0z_Fk2l3AZG+ZYz^6l4jDn|#<9V5oW6?j zHuJm%@LBTJqK}Di>XhL6r3-ZIzEx2WgTKXgK$(ibLeyyY%KyAm&aOiEkIjCeJ7h`G zldR3*C=y{_^fVU}scSYHZ;fWVqOw?Tq3=$n6NNthFE>MIP9ZJmO0l~u_5lThd<1(g zGZXGy6|HA7E`Ue*^qb~ghwE^LT}<;1&txj1$vZ-f-;`!(M@KOM?S1S1c`MwN1(F9!ks-Ef%&qgt=K)2#*R;BVSCEN^g&b@&A&JP zDtBsPG(UX7FYU;jc|_pG5I5> zI+Y-^F!OiALmNXk!x6HCnCkyQ-s~UGS9Xq13wFO}2!@bvW0lVqBV_+n72qg6o$Sk` zQ~MYL#(r<O9*tD6RG?e&FD-`6m!n17m*LR%mRE@}gurBE>pZ(du zVEWekm;K5Hu848f2-+q4Xpr;ppsvl5huQ2Lml^?Mt~|ihJ+L`S`O|+GeQ|0bkC_T*4AlRRG3NxltrnEqcDXda-AW=&$TP zyp}BCN4zM74mq>{9j$0lZD2`U24556lB*N}nT(JrsdgMglSBGhLaUJ90SAjG=osz= zP=F`|>a$S!);|29#7f&z9B(+7ldfBK?mph(tl`SO0*~ZqIOGZC_(cgfD~iA+0huz! z4pbv{oh#F~8q3C%DQJn({4uU41e=~FKxM1)dG}$#GKsas4t&9n$IAE?BH+5oi$DC;?s`+ywK z9`#${SGcPTzv?sV4fNJuLv)hC)IfZu?8s_Z~_%VGa)3igCWAMrIBCRu;5gvM|P1CoEaWT(7d(_^TnR2L>G>s7QnDa{j8clMY82AnPYdZu#=(Vl#SBm`)`Kk zP31i)f{95Kvr;{;_!y_wdS0L7RkKX5r@dE6vj2mvT8QRxf`GNWN>)YiF+rvhm8JpU zOC=P(XpDY;ZQ^1f=W8LduooUWH8#?Uhva!nh1XgNI)>0|P8zmfuAZ$#e6N`-E7;h< z)y?rK^u6U@YXd|9j(sFKnhYi&TsQbe zvaMKn%M<&O2UPLV_Ct0%Hbo7{|7q*XdPnv{`otIQEy>61ZfJdR4k25@eZEho9K7pg zO!j@cf%eVy?@O?eRs;>U)+{C+;ixU1Xs#!Ix+=Y6avAB*zjGt;>dI=)ePvZ`-zPCL zh?1zI?!Ok2f?yv)roCZg`LiCGC#naGcsc8|EdpJ1l4ua8G?RZ7V-T8HVicfN(y15+ zg2v%uDY8nFpeLG+6RTZnZR&Cioh=-p`)ap?EaAjK{j{ptNw#ALa$#{eK z1DkD;IKau_<&Dl&v#6}D&|rek@*ol2v3NkjyiX@QLupa;Rm(%oy$;`c&IgIQV>f4s z_7OkqW>Ne6fpsGeX+t^ThS<}%qAwa01)t^Z>9z(Kw>c>u7q>sf`T1Yg*#BS_*9rCM z7XW%v*t@Y3?E`@*xDx!TH?*Q?UB2_SWy0LT;)2_ z8o+!$?iNvEJy420st+nq*~vzOY7$f5ef0x;l1GEw`>`ZdQNA;e_5-brZz?Ef$cxb~ zui0($x;)$1vq%LgW(w1ftFBW2MrhPw7dM~Dt zvlg}h+^!|3(A!-ATug)TfDqZP*_ccZfjek$u=os?I{X7JfIykE@JuWE$1I&E_`nwX z4$ou#FDW4p=wO!80z|<*IcE_KWeU^Jmo3pgsJC;|Bk5y@NmNtbUvdyYJwF$;j!N7q zmN0meOX`!rR)8t`7+}l%qU;~3oQJx&;$djE&9Cs`A5nk7D&Ow{wK&Ro3+8tPL7&JQ z5tm`@EFsRNtA|3iOdtG{!AhO_Jaswx8yYFeYo{J;SQ4BVtX$fYdBkVfQ_B~HqpsB_ z)-qR9tsaK}QXomyi2LQ3%n;BAmY&52^yK@r6dl5=IJlzH_$>t)s^Qfp%COvmEMoCX z4jfQyNvzA-nC(Zz`_fjLhqG04HmPk%^ML6*E>R)oJC}bJ5QUszrC%5m?Tc2R>)IZ5 zPXPUhYV|v0dXrWDA2pkt1FVqPgQ7-nj$awPbh~U6{$w|CT1)ZWtN0{f-f@x^d{v0!h(dAYF^;(f#JVi?ELq1tp9{%54u+n8wj}=( zT;~YwY%`^3bp8BnTm1xdgbpp zy+I0&!jkLVI)%sKb66D)toZC++z4%RNrSbxNhJ%VnD~gkf{AqK31IhgK9_|C2_7$} zyxNHIISx;U4!QF+g&I)8FAdL~bAk;b80K$q+HlxqP@wCGaW&@ZQT6t?`%P<{u=omKgP1sGL{~& z^fUwRIcqqilY6vV!gNXZ!RyZ4weiMi&|3V#`0lLCn#dR;dDnmWPb6DstdsU z&@&tFu7A+Jz8150CAH8SqN?u8zErtbqS@+FDeS%?5Dy>K(sXxQ0R=Z&lB@j}%PxPI zwgW?%D6>fCsEIGEa27#3>@0w4BXT?4fn5dfm|$wU3)%_wi+sq2OhlXMuP!yStUWU> z)O5;o#y9DgW{c3R=pidpYvTBqP7jRA>S+n2H9z=9wn!EP+k!XF#np+*kl+kVLjNhz zRl<|~*^wH>l0`(tD(Dx-2LZ1Ol2OcaDt>>FsqkR4In-2o6uy7cna`Af`NWxnBAy7O zzjXp6G9K={Ju#68t3eW^Dv$=2tu|mm+Y@P$Ur15azC?i7$t1PBsvPj}gkJyB_$#jY z(XCXV^MdJYy)m!29gHg}CsMij3n&tWXl~&D>81y2mj#30 z@`*p5o*`jQ;n`BIqatgaOpPqCt2!KTfAN(qR(pC`Qd7UgDwI+lx3bNJP8fMHLc_!$#m>d#9 zic?T`In#$n)NEhvilC#~{zjk8l_MJbLxmXJ68O8Xz*f45HqvKQzKGSvBJw`;ri2y2 z{pXKs7jYBqt(>$_Ge%J6bRZ;2=ueqFa5z}5&Avtbzq4Ea+5*!okrb6C*a;lqp@3gs z#hVkS{$oKmD~uIgjAh?>mpYg@&K`>^y!$s~Ub8+!6!ek?{87J(JD@adY#!yo`Ve}tWm0)nXANvvSX>ehjDA6 z5oDgHIz4ZRdsb1HHGt4tjaRKC_0SfO{%1Z*F$iox^NI1kTS}aF-V}La!Jy($E>UNX zz=igEDeJGAYd~I??hoATIpzX%>Q!TBEVl_2$)rg1zuoQoI5q9(-%R)s$w_Aa9cgo& zq(74hLQK8e*Gvf~Y^D(|mJgo|6AyFX#3R4j0A%z)B90Vttb;}d`XOHME^}K#{_^`>r_axx6;{@vEN(7TatEi9w_(00dmk0VZGMNjE?muv z+M=(KGB3`Qto4PM!4SGTS-D$(IpDg`rW@mk2;t6Ja&J8}nRR3YeX>gR0!JWKm(s`T z?uzesrYy0EN1UuW%JlDYDQfO6j(u|F#(5v!eD55%uHKu9wIkz^W=Gu1m&58u!b%-@ zW>CS)m9x4>d2u2S$*`_>8GUL$kTX0Zg5uWHxYLMu&K6~uJ)c)hl-QZg(pO%n{McR2 zKY~KP2Y@jvJZVyGDEYhh2)U3A9jqJe(cYI3o*OZ?9dmlY=AA?Z+A&jkR2o1?ppUad zuyrf6AcoU^f;=ce636>j+kJWo_C{4+uq*DW`WN9>^|MRcxr!_>H3=*;rh|+2`%B!B zb~jE-rBM_g_*pZQ$k6WX*5LQJ8#Ls^rHigQxIK(@Yie-7JMX$lD!YY|3(qXGXeLYb zL2~$Fho?eR4dyH-LpiUeuB3lO(qyoVvld+S`D6U-PB>lmu3KNQWx}zc-mVb;!xGaD zAo7RVsKphrNj$6V#5W!aZA$?D@uA1Q9RivFaE}Ya<`Ky=5xKcd$D8?nVH7xZPCAV? z#T#gzm;;Ele;GkbCZ$NG7CcwRioE41Ktj;W*Us(XPAyF(4!hHipp%C-eMq z-DS&BKtIM|267oDoeafCtW%CH`fk#?LbM|v#q3*N9af$^=cV(cCX{j+u!9PqLqCdU!xD20h*;vX!m zOT@R>yapRnp#3(Ph=*nv>f03SXmAX-iA%0$^raa1Ml%CJJId)D$jzR8n?VGLy>4W@ zxL(vUZi@r{`w(^X+^`j{GYJCi_hCs?a4t-q-kwqU{^I}I7t*=!Fh5vLgu9sM!}C0^ z8X*m&pA$^&?Q>H*93skUcv!>uL#V%0$&_ptuY zAE-8Z8H&cC)VbHK*J}S=KR$y$qv9;EhIt9ue2=i~=hcRkI6~}fK%UOq<|IKost~G2%|7v-Cs}oM8+XgG^T|ZIBixVRD zb%@HkCe&aVoM<*Zfz$j|s54eEuBA?~U?zI+B(SJrKCh6*#iED8M{;cO7kfhVfQ06m zdjuH~n`G&J7Ql?#(6q#ANXTeEo7_FaDGG}3c?X=ffqtTGC~=M;bKGvSP0|X~5BEeT z+eL|g)V__d4*^vKiFk5w@)a!~YgrWChKKL*Uw(}BM1U;P`|W2-u}=9p-o2o${q}&!%}H9SYn{8qnlt+Z*|k+s>TYL65?BE_MJ^-Ff@P;?! z;L41snh7Q||JqTN+2#3Yq>Ci}hs~gf$&}O-U+yhHokkJeAr? zsbog{W(cnl#nP#P3bq)rTkZy&(SRD6sX>*LC*~MAE@5;kG-RKLyJY`<6#k}FR8u2! z+q*QXO<|z6W>}scCFq#k4h&2CuPQmqw24&E*GTxe*Tg?NOCRN|;%s;n6tKF#ADM> z>N|e!V%L@$-X`DP)Y)QCvWpwc+hM(LcNWKwaD|0Yh&Vv-%PUYtnGhT?K<;qFzP)RAC;{g&6v zZm8s8F_mp0?YM#AGQTydK}DCV>lWIQbKzU0(uI?t{|&%YNQk}r-pHzL2e$P}u4ex7 zm7yRTVsmLV&%C8jxWASWMJ}qE)@B zBI)ftQ?}T=L@T~6b-D?HL^DCf!m^(3O8?1^7ORzFdct9~5EE@LGVL6zv<`uf1tZ_* zZsJy5F}1=J=cd|#!Fqc(WL9V9C+fLi-zD0HN(-Ur0|CAvvY$zLPadz|egfBA+1yuY ze*NdPiEg7NO-YA$zPc|_iK(K965tF=dBWsH25%%~z0P=C%k~!1Kpr>g^)gQI|7n~t z$PziGelRC)D2LV3T|cCD0&Fb$aN4ZfuY6Dmd@Q(;$* zlAl3L^n+3u9p2#(d?-~5dL=8#JMQQhx%9TPP^b!|#JM@3Fxgx);!899q~auMJ*eZe z1cc*K#bJlI9r{Phgqi^9{Twt%#?x#4HL7%-GuyyRWhTJ*ajzyfEDG;}0kO8Gzp3!> zAUn7&xgyM&9_n~REHsfPU-BM|CbJVSB5M&KZw|2>5ur39X+VLQD{8N`bfOq6KxQQP zkc|E6mABEcFf40RH*cQJjO3=If0noGg(YRt8z)^^5T!gY5!J;GX@be0de-92xfXsb zpoSyP`!K`l8-aR`hB0-1o?E|>14#iL?TWQ^85nV(7?8a}wvNu<-fAMjBb0!(FeTG1a8gA79^1D?$tI)aUFQ_j1dJ4IJvMRFq}2QbPWh{qS$x8)f4r6~ zhJ>Nmw+LlY8~I0e_9bdkRyvlth@%Y}YaZrV+wIv*IPVbevM$ic8jGkjmts2N8Gcup zWvK_nvUt9xlRX$DWG>AfKJ#Jc(y5-%(! z|8Js?#_tf+?3ZO)aJ(8lTy0d3U@-A#R5@;BnnY@jr`88;)^Zn(tmEsd*IHvKf@uu# zJGDBq_DG4x4u3Qk*E!VodPd>__M!qtp!KVll0h>W?V6#fIFe1*huFkZ7gC)%-R-Im z!MpbWxk}E*r4KT!5Mu(=%p2pu6e>d#J?}NAL-2G5Kh*Mng^Q$hW$t|N11UqNE!&cGPN3r7Ceah^KBj_8Q}QVr<2wWg(E2fLV2OLN4t14X z58(C59gc(S-nub@_~VPu_Sg&1=ij6ay>Ehvxo87!C(JM?`CUO7BL?rY+PUo_@( zzHL+Ep>zsADvdq?_5F{S6-Y@z5CII2_p65=-48}<_$-mZWj`w= z^-pGpg7Zx4l*8_+If^?}Qhg*9?+d7Ue-eZrh47By2mw=;lif+WVz9x~oL~bhF&B)S z(c~FnXdsf~(;0Fq3EYDwBcAaP{)WPn?-m!4aG+x$l_y*RzsWGp6-J;0iE*=rP{pPJ zyrGDzQVLi)C|pXMwC8PUFt6H#pJwCekw+J!ZsT+QHiYfb@>CI80`755-Ru9AXAF5t z_XtSfPw%cbg#Z%A(3toIZ+}#}s;s<_&{N zX2X~!fp_(rt9P*Zk?n|1*@&`J#@P*}3*s!*irU{Y(&-oQWOf2E%r@f6+=pa=@0~_< ziGjouoWTL7?iawkW=>mFf!C#xFgS^Nxj0TBm1-XqBDiv3_oSrQKhCF&MNHO~75yEc zZz`r45scL{0qs-lEu*r=8nS0c?WR!XJyXdWW`raYbaCtC&ka0nG3()_dH0u-DvRl` z*lQjfAs>FX`oLs=%rPlr*~KPpxo$ukM4RBr91ib}RHa+8ECWg6Lem3>L)c*}vE?Ry30gN|$jps1?RktX(nue$X0STC!4ASYNj2U{8V(cS7@O-cDa{_$@kQ@U(6I z1#_NGvR4mi$Pp0cbJ$cN`&0?hT}UjCunl;br{FkMFje3x^{{UQ6)E}dTgSD*{m@|s z8Y3gsd!VznwhMF!K!7ZiUzP!YpumU*0%0-rc~8*TU^ZXu;1dIKyQ|oROL7W)y)(Jy zmvjXK*K%R6*Wu4;8$HAN^^%5nf+j(v;8sJ#Qh#lwbbXDankh=2hYS#0@9Th1 zfSeZ6*;L41CKITB*UkqZaChqZp}Y?Gc^@znr!Adu>|FK^0+k;%2F%CMi86HsG)Cr& zr+YBPPxfo5iT#k^v_>T$W@5lp&gd{-Z=_V1=5gh6rc=`Qn*At)3mVmp0>Tv)X?42# zxOQdhF3D3%{YOJez~=XDxyfPx7JwoQjU< zjo>0N%>h>_rb3xaqr%3r-6dOe0)Fga8~@z)oj=VfA<%h$`TWzq^Z_s!d=Op`gyWtsZ=r+yV6DNyTaw6%RfQDdMH;%M`lOCgp zOUJR2X^CQ7&M@sEWIBUnI?>*);H-U<_)?x74=d))Q8gqVV_qE*%0*RC1`mqpD)e+w z=xP@R6aFJXCpicKU8RIG!KIfp0!43NxLyyasJ7S0yNtQg#yD!09NN-WgDtB z7R;`K3&fEG@IdL9Y8KIUG*EVKKZi=yhhv6VJ{k$a^IaVY4v%LK`Snl)m&q+*inu=B9#YBhC5x3JbwC-XdtF_M?D%(M4)T< zMj_eaEGSVWamOGk0P}x@O>1GgTOeA>K1)}1*aU|34HcRx_>BnMNhBh$urw~^VG62Y zHNOS>&{A#XXQ?!G?eP*49ne@iE4%* zu-TGfq&Q>}Mu&VZTLhU2b$96B3g+vcP-o>#(*F`)oig=g6^z;RCy(C*1@lNdJGIh4 zMmeShDtSUK?=A@@La>UgN`Q}&m;hd(ZEgmy zn5YT4bV8zWI9dz@qF~|3FP{x_SYM$anZ&CE3*cS>Fg>t-#)#Jk$r-Q7gDo*YMx?-i zlvtjj)RjwXc=aCL-rlPJ{O4iD&Jb1Sx9~A2w7_kQGt2oW@F-o^LLqsClZhxRED(s4 znYV!n(+Sg}is@~j9S_!0d$i?pgiSPJq9{ZKT4xUP4s5i~k+Go_<6s`Arq(k7ECNl` z8D>>*RN=L8uo@gks?PmwRP9km!?>So>%Umvuoyvw4Mnm+k*#l zX}{ad3#YiVeQO;8R@k47cC38B;*qcE?$g+~UAB}XM z-|u)W1c`Zbi}!&hBsR>t!?^NS=h@p2YJgcrq#YRZH0dQl$EaT{2X+2%8?2{6fSGa~ zl_jz#10$(Z38LE$1ak@$fu47c1a`p;9y$gLuHItsR!A2up;QrGFfV=po$>=~Ptx_4 z=?|j`!&4f-qaU+8*%z=y$bt7t|7>L09-fHg0t-5txDZ@r54eV%SfbPPgMh2>ZKw-g zg7KnH!tiIX(xAO|Xug{XXnCHS{c-7{F!6YrANS;&(uBGzZ7njEi3eD-CHzbEj(-RG zliPLrrMi1#;c=z+(0cuVJRF*Uj4iirODXYw?zT8qr%mW-;@z0TbeF?LBF=e(O|nyf zdQ9=9D)YyX=%*9)WW?GYQxelL<)Br+_^lsaG5m(bWFRtqKo3uf7o|4 zhjP(+{3d8`>-DOEsw^BUk=og|J zuejDlAPlHFp07B3r~>|zC8BhqH0_0nMVjJ2UP~Jam#1X`WxYMzlxjDY z@{u&{Q(fQ_c^2y!3Pao&xGKmibl?!-2lc7I;PTq5RQ?fkh`3jZm!b%o8MN1ZJx^^$ z3{rl?CD`q!)dcq28#Hg>mlEoI{NMJJpqhQe;uk!o01w`T`obc!HLG`CmuQuZQIL4l zU#rwtmfZGVTc-9_##;r4a=E2;I19Fby}?=ItW_SQGBNubTc%T>@=RpY69=+9N{}vr zd@@xd7YOc13VA*FIGB*DG-Ul^73?TzNcRvgpM52Y2Dn_T0D0-sM-IPD|4sc+>ZoSu zr9*F!qzYxPp{z}cboCQrlMP#PMyMXuObE(CNy~*Qgv66 zFs`p7T!F$h=2H4jE*~UGh6n{?BHA)%QF(!W;b&>u)DxY56=xD9Mc?q>;C3HSua`tj zfu&(Pls`lVYDj&>wvMB~P<+JUEvULuwIe&p`*v~=T_q%`qzVc{IG>5?Dg_A@bNLW| z2UscgW9=yL|K|VwFulkQ_;|CTR(3ALt1nnbV0#!<0I1ZB99I1)aGb)j{=&l`_)!3! zVDanMm|Lxsbbuq{Q4v0aiM(NTq5&huDjMynVd2yp(VqY_gW|F`O;SdFF9Jc@n+eCjJa86A*$ zR3tobSOkQKusYwKnoS8Oqt{E5LkT^0dPEGW2I(Qb>%34$!HWi4)JAh)FQgJ6TZHqZ zEeaM4ThWRe=I9z6n6U!On{)DA@2#25%BIkrM1LykQOe34HAkmGAkqQJDHjB*F-i5& zg#Avlh12SW*;O4136UlLbyDO&M+xz?)F;~kN|;S;BIfX;5T{C#bprsN`$ykyI5G<6 zOm^~`cq-jscRk`U*pgXkeoSQ#$9N*DUH~Kz^d@s-KCcf%GLO$Ec*h7gz6W~pS{@nwc8cu3DCk% z{QfTeX?;h>oLqe67_%=*BS~*fxrTxt7u_n5@}3H3ioJ% z5XWS6Wo1HK+D6yr`(}=j_C6OX z>o>1a@ZJE-6dc8{YvU6O*{tn0wN91~-p65Zjbr3r7=pCL)yZ2kmKhJ;siN4Sb&#o? zg+`jjYm|8a{V+zly$7(JxIUj~&?EOIQ&`7@_)1*5FgS||vg%&$$m*9K{@iJuFk!v* zQ|^IHsR&^$E~d`roS{%;=jwW0k?O$FpMH8Jk@RfWyi*=DxdljIY41ko)aL!Gv{f zr*f|TV$hH2Bq!fuSAvd}uga~-Doi53=wy-$3&CBV%Q@13 z7uTUdn0KXMqM!P)o9LaWh3*m8LdrDgfTB)dbA2|a7X@rK$~LR1#AEE0pMMU~>u~-R zHCCkQr(`!c?BX3qQ_%@`k=DqIAy(^&ft`W5@q@?Rv!axvEB7?y&s5kQnp)*MubnsX z-Gg+Wq(!y4lW&I!kO+0_#4hO0xKVqqXr+3iE^?yx)w8f4xOJfSH`Z-92F08u@5HMr z=uoG^fNM|O&s)lH|J_sTSCyLja6WWKV&+)84mPPMLxzptbqdTkr(!c@#0?ynqY70M zJPoy^jlhz~sVArj<)TYTIu?I&uZP!SUS!8hgvRMrEgXPuej<~tag}liC%r!;&?QjG zL8;Eu=hS8(z{N$%+un z9$aT9=TM{kP3NO80gNB; zLA~T|>JAESlM#nI7{Y#N_gEIT94=qEgm+`#KhqbP{QJ7JzCDh%Qku>FK_(X$vvEcpZ4bIzg3$1$XYvRt~U1lpoq-2*?9Z`wA_Q^y@L=51kH-}PExk}@CdY80=d!~YZJKy{Gm~v_PGwX zu^zqzt^$gumfgn;F1NeH#>$d=-*-}?l55W=;v6)%AFd|i%yl$n%@|_pNNHU3qeS_@ zdR#T_?bG8UU3~P2$6{mbx#z7oeK>J`m3-OUeQz{`_w8%evO!Flgrcy9q4~ZIx>4A| z?G7gzxnU0Uj5AMqm>kNLDHmdUn37sy`p-TY{R)+qA2+0`(#aEzJ`f6X`%?ib&M2P634ph)*2O{OE2_=)<(B_8i zRP*TKPVP2TSQa1OniEN@>_PFMJAg3d#o8zmQU4Z;xE}+SlD;BD<@^gQl5|1IB$iDa zb~i^9JvGn4U?jqELw}rp_R?VEQ%Sf*)Th^+G|qydeoW;9%wJ&Kc55P0(crmV>4+gB z(ZnCRuz4{<~=)5chF5@{}{8v+qPUPE9{l~6|o*sNVx zdN-#vCqf;#qdg*s$-DxXBN0T7>rl41-f)Cml;;~eCS0q-?y?0_peW&A0Ng@ees72x z@IQKS!HA)6N{o7gBqU@;7}5(2Scmd}S@+?Cl;Sx)vWBg*%t44eK|2IdS?fuFT<3|MfguCg5-jO7&}qa(SgS1T7iNp26qn>*om zu#`c@E0K1U9>p!60ucid zgV4HO5>7|Wa`jGaefY8KY20$#^ix&(Wk1>e+t>RL%W|_iLq80V(}FMl6xBzHgP`e-8W9jj|C- zD$ck-`Gfk-16>nML@gi3D$}r?aGk7pAczY3hinF=eQ5g|yYlUU)IK1*ecV*mW!I(n zi2^>NO4kR5p~^y%p2aZlP}FuNBLLC%o4$4U74!D?z$&4bJFR)&OmWnT7g1iY?2xc| zh7QRK)XecoeguIE@L9_x+^Ju%`_ZFj zYH7dRK{Mqb>WcF^I^BB9h>|w3`u*!?dE^tf{w(wIo}ZddmE+E&JA5U=-5`YM6@g3Hzn!oI5PInc!B)no?WeP1sfSSV? zQ#~vahA2Rog@QU_(@R>wv(K6I3|DrGs0fdvztdLW{@G^bMURb1HGDJKXi|j0Xt=?m6;)0=YmxrGQGs&jD{8AE??Cyim zdbDTt*M$A+1yVqv6$=|V9R-{au|nBMh2R{6iDkIIa&fK>$2hGoT#6I>*M2$2b5M@n zH#TX}1;DPDVN;L5AB8OZQ}k+1D2tP{iu7o>xZx|DTmF>|T33!4+B&S~oovK2g}4hj z%0DkZ??9E_e&nTs(m29RBG5fPQkxE7g`YcC{V>B=TK+|&s}f{3haGc@H$d;WJ@@_C zcN_klF{V-S2Nwi(Lw+26|FC*Ol-4?#^nD87Ak_e)hHR`aVfwVO4rw;-x@0_yGTj>k z`(9T%H%X^P!CGd0jBE%Y-)YjB7FVaj;1F{paxBCo_;kCEK8x?hBm>Xt=xAIb!0+9n zsb(%Ev7j;4EU^j&v>z~b1pj?EEu1b#U1GH6r=drY>=1~k4ri@RVe8w;mvN+$rq0B% z19akUH1q1WFk&%M?$*-VL!oXbeR3X`vu3{to#1K)fjC-7?l_Em!^Zw8 zkk3F~JEF3!v<>{tPvkCIlpMXW+l0&4fNK0 z&660!LM~g|z@$YHiO%n*c47_YR*X-$iQLpb-!2v%X?YFlb!R(VMFhJAl_!baOCdWL zUu2~2<&ieUVSf-8>!VBSBDEs0Ocu~_SZ5Upq>FwL9aFlnk zthBMPK-NU}x^GVB;(q2UXM1uGQlmrH;HN}<#-_|rCiIK<`a$g??QTl*Z{IB}`l`s@w98E4%W#V?=pj{l3)(?9_`{xHwuG5L05P{QUdn`t zr3bJxxcT#3xU=A41@VkfQc_C%gOYK(h z30d8t*$h1)xLK+va}Pvw2U>qKNF?GK2DS9fCV#)VOmJ+$foolcIQ!9knJ)cDpOs?m zDi@G>#DWP&ih`!eOq_&WAq<#K3Qsng#sEJQdlJy#r1h$*zIa}&e=$a&?2g?OleEuc zXeH_KC)vh=)qKX5y`Ax2P%K2erZM$u+A3Yv(H+eoXjRV?PBdnemSb3GDZ}cJU?S=*p8e@}OKV70L?*}2YN!pYch;LqQ9KnHvrdpy zX&pmGwHSpm+}*U5|{eE}Md3U5Z!2AY7| zMH*fTi=n{Xhc5dsvx|L0ApAPH>dtM=%vA)$DHg@qgcY-;>$k`|)~K_Ta*-uF?rfU3 zNcgFYTi@h)9^zkkC!*&6K?ZqJ(tFR>xz9WJvs2Nt7I)VRt{|uK^^@+fnF(JvK3_zx z0`N=^6JM*fNj?7kgf4sddIKYV`Ir_Ja^hutV+ z%eHvEwjHJx_k~TKOBlLiJ`_yqju%;1X*^Cu{a(Ax#;`eAD^Mm~rtGqx?=J>GE5 zZqKtc3~MCP22>WI#sC^1NrzD}lgP=hY^5l;B_)y>aKs<+RzL=`Pge$9XlsLtNW{D1 zt_mvT)cUm*_iqvzka@IlHajd$E024(NAR|G-pZ$mQ^_Jnfeh^KG${1E7vsr``Zc%6 zws;l{_Qb~0lcbt5Ks<=mG{-9p7$I~Yn})1Mg`5QU`X)3=4gjDFgYi+pET@IaCaqYq zHNdF4Z9qp(@`DJ2>Hw(!Aq116V)prs?)|40?=rsli*w6dh_q70<8O8q9S^XkYcN=* zt<*z#{Ke zjyH~SkHx0elQXQT^`#tV!Z+i7jIs-qPN&ca+bPj(vg(g`=xuQDXIp&bsxAV;gVYT1 z&OCxbPx#fMn)YB}VlVqqkollz(-)4#RJdwdS`vsOSJCz|jBJ2F0L_J55IS>v3~aV1 zOUMB<49)O*f0D2I{}w8c2cW}5FhkU(TMBrTT9O8( zTE)=tKJY2Z1|fifrxjFlmO0ICi?_j54&9T+OdGr6S+qj_);&pDNHFR2yb*3IB1GxO zjUiAnLn*hk|3`+@=bRyDNJvr}b;sSQebTUFWQ3~)IV$qAjV3t)B)KJxu|F$$zyN#n zwOOd|^?4=1A(e$*BLoCUM~?PTauD_XB2n(dg>tz)Q5AbmbnDM1*;(++KpM{?f)CEd zm~5JYsu#pHW3bf5AtM@iT_Vi2*F8X-x+|yr z%hgWY&aE_Pz&yFik1!PugMRP5;8P_BwEsd9`QC|oVa1|ZB)_M0i&tte2V?WCAhSXTy4wtCS(=47b zu!t88hs+K!=Hw+OmyAk4M%vS>-ot>g9}IYcoDSwCP88mSZl|a8aXQSyog~Xz`kwIIxE$L7!s>Kj4-Yv=W8+`wb7A*DE z+#fUdN%fO6WQk1`q{=T=yM#MW0g)RK)H?$8pA^)8iV17a6ZkBHm$Fqtn=Xl%Lf6Rl z&^l!d*S`%oC*^$|#20$q7GRfg$hzVhM9Q6r|EaFdOO9XG+jB!8n0gklJ zh(?j3bsMN1`zY{ePVlJ|3HY!@hP!!P+2Pe#4QwH85~d_=WA0stwVxwti7a5&PWvRI{{m^ZOB1 zztqHgMqi^6Q@ulK!0yN7>>3JOk;p;AxA&PReE<6r!Y~kpMTiWJY0U=WUSlEwHN~HP zmMg7U4hZcUV-vm1AOYOR`+qCdb^)$Y51%$J)W4S#Z?aj$-gnSCrhBziLP~vXBr)Wf z$^TFZo}l=Jl&bw1-!IPd=)#s3IWk}i^n^jeTC-8^)U{cc>+)6}ezLU6EYYo~xV9ox z07gK$zkVmy+9VQR*@-c4@jHIHWU6FG+Qa#qvd>eJ|RZThAGSZ7? z?L2%(iKyWMvdy_GjMGD>Tqx7+Zb~L0zZHb!b4Z#I!iwnd;TRe%oo%G}YD~Bb15RVh zU{zJZJ3*>wpLwDKG__#0MNCsaNwSzb)p>?BE6e`5$0E~NRzT~e=|e`o5*VRo&Bmzd zy5v}75-AV6TY5$(toB0Bw69MTvomROGPPpT^I&8P*SWx{QK0?M&?tjy;wGphX)m*g zo(e`d()7eo1nb{_BAY8{qv-+0M+j0ZV-JuT?YBtF;$E6txV)t?h5l4>ZeLKm>9A25 z3lneC%AMlY^l=B!S>K@V%*H>>lNExQH?#=m=}l~x`hDtn?AtXBHUUzmSbZUB59ik3 zYIaII9oiEi>?wsD@F`K&`HB$!4k0nDv?oL(eW}Y6ePDcMr=fest-3%Rp6?Mpnd-w8 zX>0Lu=uP-7!SevlIwV=cvxBV7RPL?0-LrUj0SMEM3$?12cJktZ(TF2ZNcCN{ceMy%}&g-}HI8p!6tdOnO&SiC)&f|s1c z!l65CQ<_PsC)Kc12NrS>?GUMNelRasH`xWd3QFY&!<^s&M+xYAYy*RqYg zEw2JZKq>wu9Dk+%+#*E`^#5AMTz zpn2iJWwFkoHtra#HG}#AC3=POLR9G-Di1X`EGOYwW?e~VT&;#xy)e2a%4SFOVf6{^ zuYFZ?2}dMbLpQ^*CpaQE$?cMJNVH;D_8jicj+|w?RhHj3Cd3m+38!6d(CYxY9PMPG ztV9)|VS86VaraIAXz2LB2vip?%t@e@S$u(rK__D?3IYF z8H0*|d7Ml+RPS`7Rb3$f!Z*3mx(^b{zv?OTD}xXTXOpbCYBUhM177+F`Pc@=Y;$wz`Tlw3cN?mc&^K+186Z3vyXNDV`E^em9i2;vmR1Uq_p~{xT>O>AD9b#ANYNKqcbp-K z<;lIBEYpRjVV^u!aBw77*p;Thy`3pY}FR3^^W4)V3cu|6PF zm<`a%zH#vl%p4pYAkt6^^FwIgcY=!wS#}^>p1L7%B~{vN`E8Oy*4gfCm~bRYoK&p> z4CI*$Jv&olspFkb(l%;bR0Q2Y1b=B}iapsbBB*3hXc18hQ8I71p&2G&fx?1{`{+Q56|ZUts{G` zSH9$1J-2s2XDii$y(Pi!q}RJ;{>i%=r^%FdjF$+UsOA-SBRYy+r$X=+jdV}*v~Aue zK}u*&A##xEJ7OF>vW@7aP7JyKA>Rrc{dzYW3IITf%#=_LX*y_az>9XisBI7?yXPl~sH$1X)yD3L&kkx=Z@hJh`*T342uK6M zzmgowV6ga#YMWyPJT)(=t&-Knh3A_muWUy<`iS=Cc4Wer9-KVbiDzPuBP!Xu*5V;M zhTeAT47hfph2%u!m@RQm-FU%@pOLuc@L@AmHZ_`f5C|Tu+5-pr-%lc))tE{agptTxT}{pbdDuibEY|P|<}%J!~t* zh|=Bf1pB?2S*?=z_rl0KZ_|Ay?M;_Mz>P_1oO>oy6%bMCUs@p@7}jl0QZb8ZVx^~; z_d}fDL*Rv%uh0}<$X7PRSNO)`>^A?#Wy9Y?9b`}`DQtA4%#Xws*JzmFylr6#CqO{7 z>asYrW(Snnvl0HF^WS=wR2#Fwj8lBF`@-LO5Fmv#WFEgWi#EbBbRx$(HQR`&bWOgx zJ?HXae|Xf<0_J$JfSzucRNpfjD0*TBH>?J`l4ihafjD#%UnruKHuL#w7Y@$@;Qe&^ zu+tMC2kFo_?-G-;8_EQH5OI1l8oKUV*mQ(-E-1a@=Y4&;?wn zu1C1WUTjc=nj4LB21kt~qQv7}0RA>t%96*X#%Ul%T~JasK}R1t6MBo&eKrYPXn!6u zGN^gGFauv*nsT($qR=;g7eLJ(uZ~8!GSymD)0R2rxkeJa{)kJIjlF~6dA6R{j3@?M ztg`A)BYBrR+;$xJEOeEO>^yU5Pn$pYpdp6du`wbA0e|I zSEiefxJ3uDA5`c37WsUNm*(G3#lYriwMndjSmiD>ZXf9&j$P@Mwo0JVvC(smP~f7s z6t?kRIz>&jKnyXV;_bQ2S@sgg>fK(%q;9&GudYU?}R{v&F`>gtNsK5N)KHP6K z6X+X;G|8w_sZcb*PZJT*#1)C=|4=NVPg!JaO8kd`fh-bXJ&U)&4SHwMOr7E0mc-wX z0p-mS5>Q%_MhqYW2QWG!5@#WTs31NY#Cv*EW&?BnXHS-6!nY57v1W}VG+X{mJqtm{ zva~$&d(JmLZPiFZCy$x~>lxZYP~%ogG=%?x{h?!LSCeT@IJMICA7`W+HnZZk9&g(h z4F-sR&FP^cMCI!{u?PV;+k}o2j(8Ax<_r5rz5pAL8nfugWy;O}BI#+lleZdyr$!iO zdt+d0DV6JnCGEo4nFgoZAW1lJ7d2~GwpY6I4+r{RZ}lskhU`zZ!$w9bN)ejP!*2g? zcx#`9V%2pY5ChJkB~UkLA4eWVhgsLXcjv}L(-+k)ZjS6H>uMQYD~s?y^B*ROvNPpf zT>}0G+|$sft-2V*sRRNPZ1=-%*w=C>^erU6nt*DT;j*bP^+iDj;&Cu>2IXPz&6IS6 z6X#n@(=M;!L9@;h^0^FxhcI~OVDTA(vsg)%4U=%YC0w~=4X&IYOVWCO{Gll5?x*)P z<{{W$=R0@NMz4(1gh>(4htU6O&pTmtw^;1OxIO0x@L*nJcg$!*plD~s;%jTtb3X@J z9u`~BN2t%a`lpv%rO-S`EfU&-=6Y<&-SKT1^?iNjrJc_o!@IZ34|gngE&Bv-tXPOe zE2Gx<0eJ@r#fQ~YRptV+xBv+o*KZt|$rD*sqbxK#UIZ2#f|a>DR!*$@UY#q&;7w zLZY^IaKGrFwT`DhG_9$>(dJ2Qqvs~#SgiAFrBm3T)>*7kII!>XcJBLTmbo0_X{H|)vC_mVTL1abtPEpVwI8=NmMcze6ac3e{7=yCTl(P|n6 z+fFQQHu~~k5?*X1NQKdBdtb|5&v#7$mJ#kqv?|r)JouD1H}|zo=`DTI@(yQpm~W`t zyj0{O$yRZq|G;4Sp8JRMM}99uI|^ zUUlixnt+(+dc(FX2ZqUcRC30hD{-O!V^O4(CQ}NSCs^ogtR^$F?@VK5kqmZ9flwPY z?(b+U;Lu_R`q3DfSRu@4v z@qDu5Ka*(COvEokZemn~-a7{x6f4=iS|gRp#mo~hFAM)4S`$i}_AKZnm2T{ets`tL4|V4NoQ<(5Z(UC7 zFhjm1CG5m-&!M83?Iq)FF|zKAipZUrh-KpSXZs!U4i|N-%o=u(9XH(j+*lRK>P){4 z`gWbTq~*)0>3?u`7FH(vI7+lb_R31*l}I_17JkG89(O|TYtTcP9{xJy(%F)!m1xwH z{Qx;C=v|ENaKI+4tK}P?wb4lTB2+BJc_!XCuPo1`zVL`R;ol&>Ku)=&{{(8LD1$kq z6jLxknPi(tqThP#n2gxr4KYd3$>@I*Z}uMCm@gJBk9}G9=h?&DV)9vy{TE#7!-U)ZEXzWTtPH0Nf|M5j^95K(mvYZVVg^ z1DK)@cqO|?MmKFr*yn1!>Hq5K^HAGTf)bLEup=t4S0RXDZfqqmaJPZNcyZ}yM z>=Y~N(i?}9m+*HlrBgg!0KoR78_j}nSMjwK!|w8Ri29Jq7`F&5+vQ3 z#%wVs{H;H#_sB`%Z9J1nH9U7|!B$UGF|v zY+BW<^}9n~kB8DkR~QaSEP$iPc6~RJlY!czMi+cv9Jinuaoww=efVpeK>S-;Ah3#q zoE{KILMw@dZ& zg^N?VLL(R&Dit5YJWCqt9Rdz0p>w=q3poea-@&473mJ@5H8JdrID)ofXgd z%^7S|G;9C9VX8iEh(IFqnH7ZR=v1*G<8_?Ze2Lnx`r03A&$T>(YCzc^lfx6Ku}&Bu z$!r}sXh#*Ji08l}sgufHb4@TrErW@zZ%2T{s?X1t`k#>601Zz~L(Qe66g48*lQ{GK z=AXacv?S({EkgwXx`a!3D^FjMRf6S@Gf7bJR*I1$l$sCxwPH^!)}pzUxv;qZ;JsEz zIiBT(D+T6!4IFayno{Hp;iYfg;B7H)hTD%>9}?A*k;T*Qt3{Q?PkRak5!kUb!6VzZ z(9=>^#`WrE&QbimacIMrJNjI7X*B&yWAJpQ1UQ4{_Iqk%#VMquzQ7NSi3DfZsqfi7 zbiVEb)jbP(Th-70WfMVhjV&ZZYGta!mpSn#DRXKKrO2BZOvT=ZcglG)bHKF6r&_fk zMWg$u%|Zl(9F+9$(~kX6zAjAfg9O*w8c0eeXnrnmwCZ^qaB!cQs#5N1cv6OjT$n7Iy>uYWd3)YY*n3DRrdY5tPNx*dH#6D?8B32wHY|XPP8yODkR|Z zsd~6rZYyKGud{mHjyaqE0?d;MYUAJ9n!lZQe`+W$i*xD5n<`z%H^o?RnlE;~Ioe4< zh|5mZb0CD9*BJx<7NZ`pi`;!M^~ej7STbm0?O28__|zT;r_G74L$qTJWwyw8-z{cs z^H8sjYc=H3i{fX$gAMyIfjZR{$o4wIg3__!y(gfZo-$L#qBOKb$^T)N43ZRH%{c)}M0oLq*R6UiFQyX3z3Q;?tg4>h8%D4Xl)pTVZG%1&L70RMnOW(s zc>4iawD-skSH34FoPJ&7Ijo5v`{mKekvc zS$U!0f2k;w8M$^wlem*y-;)zCG4+13MjHY;_})L5x%?>jb`rXasde3nk)5agU*svt!@%9WAm-smKcg^pA zrQmZ35|_OQqFD5SKMuD{c+z;!^x(*uH>$Yb9$mXJ6jg0G^-BRe+qaK=KDA5RA}jyN zsQ_qKROt+*ZS`?^-RL%YllwXS)t=YEUvc>LkPJh@8kIj#F>jFdv9bKLlW^*{>~f$y zgeLDfiov9I&|mKU=A#|`*QQ%lAZ?^wFxO}~ z&o`fh0T`eS*J3!})kUPV-uvcH-q(B4$hQ-CDmeRz?z>8|!C?3g`mYe#od2X^nULrP zY&r?E4gupzU$-bk!QBCq$N&mM@1^bg|3KTUc-}y8HiB`P90T=g#m=?1oPUnDceFr# za}(h)QH_8~@X|4Q*i2)p9v7|N>w&YUD zI6xG(Z%cx5_r)NCutU7qilQL1LQ1a*j(3te+O^%>B;Rzfib;HEb6k((_OahXT)5wa zW($qS%KS$JSh8bgDdz-T1{+$~+B20hrm8j9z!ZdW{5tO(s0O5jpBRCVHJ?y6O>eq^~Q_1W$)v{7MsM&L+<4Sn5+weW!z8Mr=G1c$ZsJ&H>+{hESZr0Wkm84t5mK0S+>c? zm8UFTSt(q|+Jedh?LsxV`nNvF)U>blp`==4=YE{z8G3FFzlpzpB0A`7R(u}P8$$%g zZM!4MyUA9;Q%^>E2&~ZAa+XsbZPLsY27;Xv7_u1EA*^F;>=WKY9_xo`)I)OL)MrL$ zZCmu$T0JyN&D5J%Y2P)?imOYC_Ju^@S8tUHXlt z%>+A^rp#`l+1!?R*%!)hHjQ#l*1qJFPbN~%G$ak&MvQ;Nw1uTjwIn01rBb2Ni@cnU z=T^vY>O06$%E!JL4HG-T+tL^@^XMm1Ci-w^~@_dUk`4Plv%n`hukqqMru;#NVEq|Xawv4>J*{9rS*lqinaloYnbw9H3Z+V-m+xihI) z{HClBd@(>MnGc>2$Gel2g6pZo*b?ADhF_{#u^pb?)pJ21o{YBEp#i|FhD1Ory!B(F zmM-Mr?ITG*bsnL6trHeZQ9L+pIY(LzEXYk()J3qQ;Db|z`;Y6DKBRt%!3>-c6$uX&- z7;wfR{l0Pxsj2)Er@4At7YUnbXlH%l8p&Mk&v!oI6FZnH@i}Rey0v`(W4mbrVY{08 z4!@hNxJmjDXh>r@w+H#^fl_ol);Bfx@d`OC2-q27u;2`C+R zax%V5dR9benIOfj$o>U}=rtY}V%=(3HRPp>WsA={fo2+~?qm9EADk~w1SVnz_)J7b zZxc;(a~o;(pWE&trhNk+x94e3XFat5i|6oj_H!3Yx~d*WUf|z;%9veZetL&^Vx5Zb z19Un!Q+sfK!QFyM|1y2=oyt})+w2u4eh5@?UJ_v`g+hCakJGdfvAs2}%Vw4GbBIx9 zIR8F?A@ZZ;tat?up!l^C^_wcI4uC5zuxQSF?jBscwTgK0tO3mH`iCnc-w}LZdE}kI z0)NE9iAY;}prN9SSpS2D6NfPY&5`L7y`8h|M#QgbMxkJ;nl@)N2iaOX=aTDbeovyU zyPvF030&KgZ*-a*RnR4MO{oII15Dl`kZk2J(Y53CyVg@^N=Zt{oD-pn)YBT`hD(z{ z{H|~Huhvdb43$41pAKy``PwMYK;!Mx3$D4sg?LmhSR*Q$&(Xy@gW!I?wJ|GI3hX3S zuVL1`h$d2nei- zBg(?+P+Vb>Ex$m?9&F#YeX)~?2+9~|?=GDyvWDqxrD$HK{y59bc#MA*;xdK&Vj;bR zvY-q-Tj)45MObAnF}$&BW_^8!;iTfRo!$_aoHHl*O znD+yCaL%ez4-n0705SZBZSbW~(f1b0KaGbp+v4qD<#p@2Ku`5coqbrO6%hVj`XIdJ zT}s&y`ABKNsWX1aX=|mJigj|ApOzYUhnOg!2%i6glv87kBCtG8b4l!T2sfq|vIQ5R z?uqOH9jFmbyWmq(cVtG0kTJV1+`^@lZZNx&AYBKEuU8}M)tZC*f%{Kf_|U-b%Y#tq z$j^#IxCR#|ohP56{o#gfjv(e3*+cq`XK|rMEjFBD&6EMtp&8bS8;}R;rl#$XabhCc za7qNv@m*vw>rPaz$M~}#x(5^kzRuV|h%Hf7aVHv>U)$2fCY$Wz zK&aze%?jpMw3X15Tqf1sIF+L(c~bD*upm!qY2+9f zHs3GSzc@Y!a{N^WVpG-Ry^*W-FKLNWX@qr*Mn@M~)?RthUURP&S@$}!HkHGF4s;g^)oy%zD+9!ouPVP2T4~8oDCa;YqMRJ1HBy7)1Lg}bbjtNQqYPyCV zWHViB=qP=!{pHhL$WTBlrub56nFT*QT?*@!fLe^4IH5V9`LNq(-xJ=lv9T~QNP$&s zFEFjl?)D~O!+#bTWcE(g5yQhOi8CESnHr~x`}asoP7pCsv>6fiO4#Jdk2*4e1cZ@=GVDTq{!lSgOnOjj=V3B&6RPU5Ox zl00=UdJJE~8r-YfpYpDt1xV#&ugSzH|E!v1(f}vnjON<}!+tlYZy_unD)9Q{NIf?N zXdAM=?!-Y`SAPJJe`IoHp7e8{jN$zi_&s$KZ6~Ia&%TCRG^Z0CNpZ))GrC1>$UA>H z6u|{RX3^A-PJ&g`j7qs5HcZ^Lvc$OM3lwrtQN9CSr#|_z#fAF9c zP96PM)!BQQH~sy`p%>M=Jr!dVn53r6Ws?@e2wiflOy}9>{AH9V^Hn}KPj*Pf;#^cu ze`OfG>{KS_83<+9|dqsP}7wD4=pNOV=6 zq2Xs(1UbW;HrMpg#;5Feo=_x8b&L=k*}9nV(XZmkcz2`4=x?kG##XE_n#lech+msJ z=+^=JX%n;_EEf&fuc+vQ+IU575TiUH{7hNHf{;*7F3pdT{#cz!DR}E7VbA7It zB3jif1tygjZfrUlxrotuJwBt`mq=N@#s^hM`shC1LPqOvc9aM=k0o1Jjc;G;_aXF+KDZU4BxW{2J<)RDn--YH0eg>y zS$_I@?)|l*>b@U!N2Si7q&M2D4PSR+|D zWrj0^Q9VSE+dh`hDplKt6>aiu`o;b&Lq$G8KB0GhMC3zwDfY|08cp7LuBzGqG4}=O z^q7XM{=bvU>L!$^3C@UmQNa2`!BYRl#^XWBygs>qsJs{q;jw01G!op4Y2of;n_sm< zP2Xso${#*?joyh6#VlQ2F^*zw;nU0sFI(*OwtW(p4y+YB#4A45T}3K~9boZ^AC@~8 zylYX&5ik&dJHnirl4PeGJbLt>XTDq-Y0%=q|QVU&-PnUVwx9>sO(E! zrWYm-O(zv!0rNAWfsXy?33pVPQB+5lv|(Ycd)h&Js8r%S_qIuS-N;FCPXL_;GIvZ_ zEz89XF%GVZ$jZB#IjdP$c=QPo{EXzdPjsVo3g|2E?Rs5A{#G4Q0%CLClh#(-~;#$qB_4H1gA9bC*Hpst$ zRrnOyqDUXobE+proJameI=6^k+7+#J6~4T_;$_9xZs9krD?$%mZq4~JVZJ5)kR!pp zEfgZad0-p^&jnAJM7wZ!WF3MBHkRJ@WU7@ODzUk<*1bHTRGZe07;%;b^bpVT$3WmU z4eL;Q@~)W3-D>hh+4$zu@l~n_mYzoDE;hG%8=|(KWJgyYmtyM^XYd_AeUDyAVodK^ zg(yv4_%`dw{in&{?7 zd#8?}Y_{_7x27{`w-OUc(&XVL2Ek7&eWRY|s~Y>p^p;*HNNvOF?eHHlfWftRM&6P! z0OpA;j_hD0&(;oWdmzK2v1c4eA8J*b`3uwP&WOt{(HjFnc%KgBf9F=ZWu_~E?app(;wAZ?dJW{Y) z9c-8E?e^V8*e~PZ8x_io+LUl@_s3=Q3hQb|a|D)HuC}b)YX?IK=5n_h9hhS@0|QT+ z=cDSdDVwYHI|QVa0Fr-=0(l;c-7dh?Xe8~-7S22T4Pkq0Bt8ebSu?9a4zAd@rE$l> z664LUq=w86&?xdTt^>AMK*^|+xFO@G%Z$zHc93$XnNDXBIv2! z+5`R&jHbgy-wR_Q908tY1JX`lbnUfUH~wq&>qbZXS}6=TO`OcifH9fHPgfQJWxLpa zUa^D10y%v_2p2&%Tko&@totEY4p7mcg7f4*bRNNvn^~#2Dau66_a7@rO1b2TwVzOw ziS*>@Ol{gjZ7gVp+m6g|c6@@U!>aq=1zZvFOr};2Pg#80Nc>)EMyqT~^ZHt0be=-O z4Itgfu~gusqyLB9xF!;e?e+IK)5#IIx1VB2F_EtS8Zl8mogugj>5=C2?!9$P$?3N0*E#W^SlW`<&SlZNz2l7AgA6{d z%gf_V#eqUXs4rT2<+KO!IvYfWYHsC-M0j%yAIGEXy?;gp2AY5IhaN5d+TN@sOP8M~ z(%X*MMg%eH;C`V1+`lYnU|35g5sFBzwpvUHwV1M)cm)T*M_kz&EtX&f1;ib${5Gm$ zb06@sssN~?QvyH_eD9)dus&dg*|dw}9X@2`Q>qk^w=t>U(7!_9Syc^t+$S_)T2oU_ zGa7jB>{ma7sRGY}3QxJ*vx4yOI#hFt%h@y3xwxBCL{t!pMZh?6fLpSJyBadm4N1GR zdbSU$lv1ceHt^3I8!w}37ada%?sd6a$hq>DN-qWkX>B{!S8ri$-+nwM?qs^8FVTWHU3lK3}U)U;fRJ;pm5nmG@yugsI7FtaXK zV>5bVi6bq(ykm-f!~KUf%JvGWofKPTmpS7HU`DLLr^eg*N|`S?^W_Zx1unW%Yt?MO zNzW2%C&!H@J+I2L#2|Bq?*K}uar*_wcVGl{KHJu<^5WAo!y$}IbevIqpLxwqpCx_) zIfI!%`oj~{hEA2&whYfEfPK+rZCF38l1nXKtj?Mr{MSD{EFk9B!J&DsDu1YzBxFiG z3Fvuzr?shmEWlSNxdGgD1-rN87(*T=B9}rrPgS?Mx8=?=2wK1dab7HWht0jLobD?G zSgK-o!Np!`f&*3R#q?LUUMrVqE}THH#JQl8>kbD&en#4|hVUSft1$D^tc!^Sd^UfoO3A{uAOhY5enYo8e*W!j z78V20*NeTQj>74`-7g_eTG~V|?pL^Q14Kk|K9{q>1*L%9 zyBy4%00{lN2IGOpIvC%^xq7KZD|t5s45yQll2ZuLNwTx>DmSk-bngu?vaf{w14%Oc zb{OUd4;J(h!Oca=YN}PTs8X%4Exm`Y8Pnk!31^eUDFNhL{xURO%ZnG_AKi7~au@?&db#m|6yZ_oFH9d1lBtP zGBi}Q?xcPCJd*8Iv9@XEI?l}Kjp}8La;dAfB^YM1tUwK_R5C*sqJ&;gQDu;J+fG|4 z1U9jy+BYA$~c7;Ff?mRGRUG@0J&|(1Q9Xbx6({4o;I!pKU-p7K!dprZ%cVKgiSQa zf@F`-83lSsue`^5P^qN0Q5+G!fUVy{hBWKiO?P^R|G3H|(#`d%C!RusKTk$xqy z?^usx7(1%(rQJ|z3#{kJNv-gt+&bV;9b1*}D~r^*M#AR@HVhI}G1qkfX*Uoqu{zplb{pqrd#* z0t*iKP*|0inXnz^l@C@&2;6<_N>xt|hiJyM7bZt7a3ShPG0agW)Rl)nHQrr#5vQBO zX)P3>xSgxmZ9SIt1*(fQh)s7PyTu>75N)+Z5rxuSI5h~k);D0W5SAhVyOS&mh|af) zQum_Eks}+-ioWfs0I4U}s_{=ABHA@`fqPJPM3+EA=qp(o%2rGJNVqQG=9*6a);<{r znRdWok6hHtX~+cq6*~0ghTZ^#F}sfDR=zoYIIMJwYUcrYqM*`#-MR^nlY^Z$kd2S< zM%o=sT_AyJ^%8N9OOo#Eao99|_OvYRkHpd_w+xF7T%on;h2Exe-sNpaQH%!Q1I2>~ zxoJ~tTLBOgvq2kQRjs2N&&N|(%I1sKx74P!YjdjCm%+Gq^@QM$wd-bu65CDGNON~R z@D&wXOX{V$1vU+!(p_b#d`J{?ETeMr2V)RzMDwa}-N(=tUAs2*rEvFH#QpD|?#Zc_ zzC)Rv1Y7-&ZmM+5UCCwr;Fxu<65R8Wxf_dj(GG<`1%F%4h#58pwdYq3Tzdy%YK7zM z*LT}~PB7Y*Q_iKbJgMt3L||H`%xdFfvLk}?YA+BPx(ci~`8 z7oM&{R9njMc7S!7A*ZeoJ74cF^gTxWvhpJsTd;xN1o3D;awsFo!5DMe^v09R2htAr z6mnx1W+Mj*B1l{8=Rij#lIqKObO<#btmV(Z#Av>c_k(A9Y~Q!djnKGkwpXj(Yanm! z3eoIx4C~w1`j2d%2K`}S^6C8;xb*38JK}Gk0W+pvhZuS1tb;j@cDg+6I+kXm8LM=SA#tJeY+zmevGB89@;hAn&k&>+g<65VYa!l^%{7qP zTPgeW2{SLH*Fb@wjicnCC-xS01iB98R{<5^6|%PN7QOW_fiKmb^N^1})dWuun_BnJ zi{EBO*nEoe?Z6md|FNN}{d4>nuI7yn`?KKJV9}QrA@%xDBqOG&^E=*Y3L2G=ldbpuV2`{x&8pAX z>A=hsSXBT1@a^*wYm=pmw+LFe|360`DFX^Ryx^~}N9RB}#?2K)=HyuD67=<|7E^Ku zHE<^)7I7D6v8k+Q0Zx~HnSdY2aHh98dQwy~343}Hz@Id9kA4M3@syN!$K~e+4FxLh z1;2EHA_zK5pQmm=aHkaHa(6~6d+)T;qSYy955Z-o!Jxu@csiOdz6c1A5sHPiQ40`_ z2pLfz_39t^=**b&2NgY|=qSR#q4Ux!90km6nE98VF$N~!hg>aoDIuTG;BRE>veC#g z#SYsN-Km*SJJOVVcwk^W-gw!BuI_#5jl+AMvDnI17|J$kp!qbzOw50D#=m4RQQQIVGR-C(h(NUutb0>H@H{M zh2`=I-1X~iz-|`u`@XauHLn^soXmn~4YbVH$}#n`i~(h$7kD1N*vsRnu={ZcFo3)V zV9A`LV?#Ele>CSl!b>zkHO*_48Y!1+X?k|08Z3b(rA%k8qoSQfNAeDnlRUrrCrJOe zqmCkiFtUc5e^!v9=a*Bv-nw2eaq8_$SFN+6pblac@mu&hYtfgesH`!~NYUg1WZS5sS1ic@e-$LU6rj1hyVi< znye6RM-#B+Mi+X>+(cEgUPC^>4_x@qrJBb|!i%&`lj7IQKxVIj8^H@>)o^ykC89Ck z&9%=W8(kXgn%k0|CQ53dKczfeLo)(#k6N6_EnH(c)lHZsyxs$l2Ph_LFK`(NXXnkW z7U`*{r8XeHX0Wr}*hBqT)h7qp-iv-AqtPY|g(Ga?LhK%eowGjyPJ%DHo56GhPglee zD`UahzhE!jyMq@Syn2wm0xCUoVl>Lw)CJ!k{-A^(NU^=#1IBRB;EKvBj>oce6r=#O z#|nPXbuJNN*0*3Tk&_LC8^$xLIk%&nFpVu&njbDrG5&9`_I7?+u>{@e!{5 z2<9^%GTXvGlFW0cZ7}3R&|rhJ!wl$(D11{>_{%?2>!kmaS((n^b|B zZ8096xu)_*Ee&yyXcSVL0lB5G>Dwd-LZR11>v3T88754h|Mr>UD8zRIWtdrmlD zFZNjHQn#;m-+%tVEQ86Q*E4%|W`!*t6jmO^Q$vL31-TB{3akzw=m2a1o@!}k5YV(k zy7sB}Gu>NV4)J%tH&|NIcy5sf-Se4M5k~VjBT+G{90`{kLVp$6=kpz@b18QowD`@v z=6OM2h=H*c|D#q`WPypb`^2D99p&vP|3Rcq(=!jL^y`FuH!{0}j~dpZCC4@l!eKa! zf~e$C4~Jrxz91KC4j6Sg7nQP3b0R*POl4vI$Ud)sg%0F){DP4-n5uc zy?mp*t+H=iFcq_8jW<+&eJ(E7&9f9owD$k%gw`V7qPHRH{KDA+Hc5SbO z<}H08mRLGVu*tSk=9|=k8DssPy!jf*!1q{Z^RuO6PcuI};vOBjBJvr)2h3NkcNzk~ zDXyJKVr$}r93_BC_f5{lnb!Vb&T=PBJd=k+`e?v82FdS5I4c@>ubpVw=mvfQM-&sY z0BC2%f`X30j(KpA$~K0FQgP#Zn4g_sdA-^Dr>MdqK;edkdvw47*A;XDEbnesBnmGu z)q@lN4XLWw+_4@LJih8dlTOEf3J7y5IPL7enM~xr4ox=csXo9$=b#(T+tzc556lI2 z0t%UrT`X$i!SC2DL?x}F3-zdRP!m84cJs$V*zmbwb zaU`t9_##o12?yc`l;l^Q;tMV2n|Db&f4i-892i4aRyN;igMPhj{%w*2O^XbytjS|)|XC>u|i)yb`h{|cvKqFMW;$ceN$@$jqNy(+vIA?p^%ouZQJW2?(!dM03*H{4eobuPBc{ura zVA1I!Y7Wo1==Mgavsr{)tG{uldED=3UBMUa0;pfHmXwY;NqnteEY^KoPgoINC*Mc61SgKIK_%wm00-Oa$A^gkaO=|SJ3SW63|X|+1g=2V_|wIYk*qia_8uiw3Kk%X5bz?iK!(ue zSGSny=#CJ)2XiTQ_qLvvFX=t>*x^g=3P0Wm3;2Br5#u6zMu27jvGx=H(F}ut8VW0k z(0G^iMaG5%(+d1Z?a^C_aPtb%p6ni?G$b_MuJh*qHpVN@kv@h~w}+xv?d1XDQu5cC z|5NBB4}dGI2@#MFCQ@LoXeD(A+=2gkTd~!lAr%=G>om1L`Q79&vtFmWvI<^nJ&;rc zI9Adw#&2qx&yYVsT7_P`qh*Fdv4zMzo#MPNdv;}cRm_E&^%I^ZzhfdAH$gs`PXjuJ z_%2y@JDZ6PV8C0n;~fRW4OQ3zWElCsIf@aMdwl5+G)7sO26IcFxs`YbPU$KY*Y#*y zcTm`@QLM;NfnRc4HK&K1@{lbqr&ZtX=ZovSwVOh-H$Op%Hbz@QU&>?#%KPQ{Ttrz; z?J+a6Ff)4awb&309MdtdWRug_K=JT(An<|YgM)2lWvk7P zl+IH-N_t~eB=Py!X$Wuj#XtghL~D$QsmztHx1i1Qmt#^)K1A-d2qSn~#OD-MH!6E<{jVT&)rjRg1$2@ z@{n7e`lXnmleXxz8bsd#o-0ppp8|9NhO&9}z(qTzg36h9x9o)W0PM?kp0xDiDk5`h6y2r8vZvu>E!-;d(Q zR$b#!yjKq1e#kQE)>jAXJ5Y`Ygn`to*teWm9|@EOnkhbrJGhbRK3<%Ljh9W?P%b;1 zj^PcgnMr4{7)^=W`by^xd4i>>`)J(mg$m#ZfU-mz=~8go+K96QQCysVz3f_izXFIS zE|>5o=4X#`mBvex%nB;^%~ausJ&d#N7n7-LISG_B7#aIGo>emjX5hu(Dd<(o#p3JmMV-~~exXS^v&7TN|COIqw@mWZsXW@v0 ztE%z=duDxdTX8;cCt4(NFwyBAEk!79`epyTT~fVmY3w8`>-OOfmMLp6fTC^_RB=rR zLgj%}&ZG24YO-|hly;}RZ9mxAqM%w^OL}^$fXJywMK*44l+dwX=r4P&-|{c$tJoV}UPjc}0)u3Ih%eCpBMb><=FCg9!KP#J z(=C?Jq{dtht++3{N+myu?HI7Gcvy$sYk|tmdf9HO&>P^@V+l-rkp(&f z$~rJ`+)$RM5zLwL%Yt%EZ~Dht7n!&m^VL!+Ya2G`va;o`IzPmD$FX?e$#+%*E*i1B z7cEUghP=$b^N^G3%pR$p3;2qs#!Mg+DG;i;;)>0xs6~A2;LCK((09-yc)8EzRwF5N zgPpw}&ujKAcdV)0G%T;FeZ)_V5yD#EUf8}GG}Bm>|G)p7F*1X0Fua#PaKS}!YTtk) z*^o+eR_HDQPf_Fke9GaId!x^gXfQeIu)ab-h;y|MW&(?~pleM!S%TnH{DK_z5kJOO zQuJTTn$OQ5S!)29576K8ApHoA)!%(Cc&HtZEwj?O@mY;;NtqROuG7uSLx*#b?kO)2?M**E%_~DE~G^A>H>cDwc5v+{B z1e{L1vf^3x5C-gG=Us{e(n~PUl3}I#q0w9U56yH_kkKu=B_;>k(mZRMt_Ne&KcJ3Q z#koD@NaOn5$*n{ai(FKHkmVH-=Ryx;EB_<(^u_DNohxebEwtJyV=>`SC(asvRpiw( zoHu97QMMF7k`p8nI%Qg0Au)Ldjcn zqhb5YW^sc6HQX&G3S^}O9-Qon~Zi!u$t8xB(gB`BYE^GUcz@L5PLxgVGd%c>P zyI~|WEUZmc*dxF}+k!o5yzRpD^8^&Tovo-u4lTwHir#D~#OFYfqa){UGWfz|%c$Uv zue!vSuHlbFklPO-!A%Y_~kO1>d~m`!~3QBWXOx4rFuOYnACm+(HiYPzGH>bMMsaC z78RSmM5QW%n_rUcS27g4$@7B#eyE_PPB!|R1)9w1kL?kdWzBVSspiMuAMXe~cA9Q% z1)9#rJbsQ>eu;D)b#}6$^*#Ef z^<#IH&&)~N@cw^P(5Y;U$7t&$ScC^90)?h}S$e_210*9^(xme-F%TGV)!LyN*(wuE z3B>28PM$|Z&OUTR?9uE#I1pXj@(2u} zJj6$AS%?S_y_j3u|dAqOtI=EeSfG**tb<@cbWgHB6+D^|9QLPM8vEa*u0B%nQ!w zF7Gd6UwaGG%jvUa2feaq%3+Q(g3FXgre?x`1l2c4{`kt<_-?Q;DvfI`)5MZ`>}`(}`cnU!XipILjY8ujes%Y)Kb4i`6Jl8%Vy)0r^6#+5 zICs*IsAF82QpX(TnZGt+cCS^+6-O-gD33sc$q)4z+PkXh!s>NP^p+j}oi1?mh#u;a zW2%q%c_xyZCPz@#c=INXi-sClkL?2!c9acDgcZNT_3_fdtbZAPa@R*}Ihl_Jsrps^ zT0aVSv4i#Arq`kK!wx*lK!V*vQ1$;5MMt`*D1S1;s5^!2FQwc4stuN$cE9fE-96rM z4iohe+U6hF4DNsaRr=XO2DqRcNXIHM4sey-){rp&*u~-NG^hFxM4m5U2#^s#7*R#8<4@j})Sxo5pWY|Ffa z-2QAYS+#RTM*X;@jxG$!#fhQMLX~Z2ifnB=dUlwCvL>G1JF$%q9wB8MK5Dfd8==iR zNmyX%XrRL+mrUpCxhXvuDr+QeSjXjRtBB8gG>_BET_L?wp={e2%3fk=0~Jn`=Q)fysr; zJ`m@lfeE;DX}i#2TrDQa4mv{}bz`QB0`Q-25i+}`*E}IiL;EsY%Pbs!WV^(ox$rx= zwiZL^H1yQyz_f7$o7mEzYz*`d(I`oc6Kk=-VM_&p$f%A@byNP%6~HtsuWIE@gUAmo zRQApW>OLF1d+E!5FYS=-NbvtLH1l6#dNfyHO)C+O?H1|(>}vPcKXt7tu8!LC5^m9r z@1`MYBKvL~I;we(z=#Y3q`PqvrrVU)zgXhIug<8VPygSF1IjD3#w+obDgzqZO=1Na4YES4N}}YYvJJN~>kFH>*As7@A0}>LadYhG;2VLv;|-&gbu4sjpP!$%_dh=pLvblmb6;i~wsh2B@SH-;e8p zWvPHEs}R0D5inqdQ--+1WN=rXtA36p46FL8kU3Z|egwcpL=(+7_bTc=6$;5zIP-~W zq~Pydd$GoyDV{X>BMB?YCMYLQ%6c;z+;D4eLpe1t#22qi9cK*_*^8~F88CuTlBILa zU8C#D7(Iv{Kd=sz=dz=`^2tgq!CZJhRZTtkoT-)XW88Wbdt#LF5doCoBOL}4zyJdk z#|b5tH01MMc-OTh`NW#I4t>%R$APW#(Qn70a>;e7$?*`&J~gD?6z>0FZnAoPS5Lqf z_cmm=#X?(=Jec2ft*r{_C4QQgsMAn=#k~pm+r{anc~zu@^!KdtLIx?R2D{#R_k+=C zP5^1Cbs~(09mc0LzXs4JsTO z=5pl8+~ivFG7VRF;wlD9r3vs2BjTcMd@!fG4E|9^Fs!>{^Q>n7Vpj%NP8j7Pd_2SG zTE{k{i1i`|#o@*GWAq_W86DoZodEK%+l(fG9Kzj%A+gH$+~tqpMzr^K7pSRf0ADIC zjvRnTzN4E*#LJOaPrpSW*50zv+#nblalwA0CND7YGnC|>rZl{(ia?;bG6^K{35x%f zq%p5O_nfqG7wK2OsK_kt*5IY+x(d5t574^@?zihO+fH=9CL-w))XAusyEj~W{{mfD zmCEF?ReI@NGnSu0l}1NHxH$NoO>@kpQwqHAlNLyIzVbSL{%;1sH)t>8B#ife|GRhc zxIhCyLEzj;_}{~Yv0O*kIrpyfNFY;i$TSZC&W5tc~w#4CeDisY&<{rpB z-(!{pohAjR>P?4a)}Kc17^dcI?GE`R;y%lT)q#MK4)8yZT1{0ik#=X8hEBKh_fV8Y z-fKuA@^-*2XzRA+!e_ujjqk-(cTmTLqAKP|+>p{EnEJ4Td0wve3|j53flT{qNk1^F zvzw%c8$=(F{CMD2HkT7YeKFP)8xM?a>yMh{-O2}L2i_Rjd2-M_EMvJ=8Wg%6^eS)2 z;3^uMm^r~{3My%*^~hE9dcC|8OfCpLge{TGH8K_x^B)p_*bV_Mqj(p7>P+WlEFkmR z9fugSRjM`av5i(secD1jGpS!8pZf7$HzF8wk5HbQ*q|7k%Ll!S zFH8M-51l#TB`QgOhXdHpnaqJYL9Zk7^RXd}9-u4s5%&VyzEaS=YL6QrvrzaIosM(X_&5qJaa5a)av(q8%d&iH5b8JQx>!^h#e6@l|6f>_J3$lIzU3f)|FsoU`6?Qr67|stGsC&`~_!zBG zr!OFGxiuHjU6gPLOV{u4qJzMsbXvg78xOqx58I;mf9O^}cu4oV@9YT9-Ph#n#s_=j zezjQ*|7&PFaYUHMy2RpwC(V$ygUZq^IIxsyJ#lqVYs)6}6WHb0m?#*`R0kRvi^zSF z%0izrbR3j~!H}Cv*W@|#?!V{7$zf29c%8hc{23Nz3<0D3Q$-RbXWyyh+v2WozlxZ6 z(9XBj$3$48i->Y;26YGJ>`9692{{a=){p}srSiqjN+IXrERans=^;5bf!6P@m}7?m ziD$g*h8isc#sgm@m+d|774hI`lv)9^56Uo>^y+|b6j3^HKuT_%mnq7#x6cxN>5X&t z3TKUyQ}CL8um8DTq{S_rN<}nff%}@!r)4}pH;T+4q-N7T>H~+AJ)8NUO2dK`I)Mem zG>;n}+tMV@Q3Jq17#dL{$xQd{0Wh+(PgY-6wI78UoPMZ;bgs3q>Z9L50MZf>oG(Mz zzTXT6@MQx*BPj=R)_^}K@`IJ7y<9`#6h4n2V8H>YPJ&sTt}?-$un$cDAWnSPC}?R1 zR}p9UT)N)7!ht?Kdol%N5;;(k5G-Z*A@_c3^-N$UqoMd5TYcu8TB&ik2!2?|%(~~P zw*XEj(1AO@0XOs3R533!HYwD^hF;>_M3ol>4E=$?rG61|Ok#)TN6}HL7eC{*P{d+j zLV8adltMbg*anrXq+eZ#Y6vp3Z!M-=ze;CmZ=JBbwJL;ofMzM1*nJQ?@o8O2aUmyS z*t(3l_#B9tgX!M?*ERu|g8j2L>d9B2R}Zhp*FBYJ*cjp{RoZ1Z&rCnNA}!1VTX(P@ zWE05<6U{gj2SFXB!AI6nf@y6gw?V;ce0k3kaA@kmeF;yA97ozho;cpjxK_H8D;ZNt zkchT9RI2uGZC4dtG4|a-Ky7wZXlh$+?z5Xg18vq&0GDM>NBKzfBvpi2A{nJEWX;O= z_kFmMJ)LW7?;w~2WIY`U#C!JLi&+<*00SBnh**a4&)c;y)1_;xjv)^5_V zt24wFAxH?Iu_)1Zk|f2eqCocvb%Gf9Pw`dWBsuZbXjyXfxc zEA%`IiTd(aHWhoA<@VIEvk*TK?1nZ_uoMcw3Jx8wWA-7=@v4{z?awnRg627^XhbNl zE)yb>-H5kHdUq8h#=!&$^#dv_bM^N|9X&y!`Xi_DtV8$LIK;4vqw8ceEH$=>^Md`G z9NY1C=xOR-QNSGiI#C`TKh4BGBS0hf2RJt80)W!t#o(o~KlCKs!g@4#@I#6BMeuC% ztm|N@b11Mp+@2bPPGCwE=Cqh!KBz8AR!CVbms1BSAfG(1m;(H&4zZ1(zSTAj%grg# zx9uUz%wHLV`_IVU9y+MfMeVp$LGPtNj{+Q_2m@OISrFKQaPM~WzV!j+PeA8l)&8P_ z)HaKH3mOa1MJpZKZ478Y<~lQ(!a6NQNs~Vwc?Q6vVOdQZtKjlsyC%qGS;K1Y{Zo?v+qs zS!-EEPNgRDxUXR%Vpf=NMq!k~!y&hq)Kn3A>96-h2VPhZdXmO=KbFJ;lb`hw#%ucJ z(xlUC7S0u;uIDAGD_Icj5k%5xRNN2Iu8klicO)#k=Hu6%cqfPMaOQ!Ud?-n0VS=;>1+84x>OV% z%V?1)@EoE7G!nrUp*Eg;m*DN~`&LAvL98xWfLAdT4w;hX6h7CHaez5SGnYnFl^MeH&A^z zYc7ClEf{|~@vt@bHOlAy9H&b~ag*0nI23E$b|{F@QoXw2>Q7;1JK$fxfIHz1Ix)SW zHM0VNcY^x@ZmhMN)r|&f2hP9&0}x;v8FNgL!|Th`z_HdpM5LOI;>-$so_5F8g^vfw z1$Hl4AHij!(iW5<9kr$X^{w3+OmRe2k*@`Xbfk}B{N!=JXb5y(F=%k@c9W-%2`T}` z<7=dL?tVe6V5`%q)U&yr3t?-yqeV!t#J>rbBA^fNwG%2D+gMH>;|=DT{8&=i-X9Pk zJWYv9GNIvhxXpql=fDhJ=$hjtJdB`Hg!2Npl$S#_V*LXbDS8T*0iOnt*yeb{3lFu~ zC)GKmr_)Gct;hVjV(%SFfIIhb7}TAv1iPPsR+#K>CTwIV=%Zpgr#azSL=kpNn+YE6 zU`Wl`1^+EAVHM0Hk)L-=W^OgdC2TyOi)Pb*{uI4lp4pd91ROhRl;Q5 z$|4)szox!!o76^0h7Y*>Z}g$v>6dwTX_^3NiVbdMD>U1v1je4jized2F4k^5fDgHi zGY8w7_tbUj7jI-&EJMIjaXK2p8ERB&(jBJVWR7b4TGkUol7E-9GCcwYA$^?QwzDu zG8QLUHQ&JFbsC}-quehRKbI~L;TpLMAi+MRk(BwiX823WkjEUL9 z41LArI>>lOD+M5lf(~DV+7;=%+IRu?XPdkVpf*)p|djO5JyJc`#~*R&sk z1O_1e)0Wb)mBc2tBaSgVUd~yM2>FH*5DLpGMI!-ZC}|GA^2#^zfh~wV%X<%78vfp} z0T3lT9R?f$Y=PtK?ohdrPr(I?-ee$y6<>w8gu|tyyh&gAFI=$OJBHfCrKRT|N1 zk#*>OxAeT)l!|dNF)uM$um#v?ogVw;^szzjnK9y1o<8@e;GV~7*=I6UrOrC zp6R6+vgs(!@{&x!bX1X_`gv4@&j+wqv?R7q-3ZUBAoEoDHUdHmWa;vUU=&u+`{ZC&EbQNMQY_TYB z*dEHw4lUYH${=;z+F%pgEnE0-g7fq)dsY%t$BR{*U$EDg?lOYKL%y0|@=w6Jc0ILT zzWyT7OYb}www~Wl(fEx}Z>)nWBM)R?!TzR(aPj$&7q4T!vsM-)A*%xLGKd=mXSsYO z<>0SzSk0L-AT7x3Xb1BLR2(H0*)AiWx45-~C<8LAP@f0{ zWr_P`{Qg0-ceO{xlK> zOcWX9yLg|u_pu{ON+UvRd|pv3W&)ic>*}_4S_EISm_Cyn(>WJLqn9|i6ObV=YQLw` zK5K&bk(PQUV?Q$-wR&SR|8nOX03TmG#+0S>iAP@hcOjJb;djBY$>Ddc^tJ1sz72xNIZhRlDMki#kzN?2WHxKQmAh9Ly*8DCCHEc^96=U2j^6ai^-J#C0D>3o5BI*vYv}eh z1iz^R(V9!<+Ym9iSEm3VmUxLk}i`T)!t1zI+7qnkUBIG`<}NECF1-)U*5%^WmmB+B%Nf;8)0;#2nhnbA@n`S<~h z<74Tj4o@#@u($-flHQ`_miebB?{sBiX;)$na!8`hM%x4+4DqQ*tcGCQ9f=Kaw(X7X zhQN(=FhDUYDmR)@pPC#rRHH8^F5!wj6IO4!yOqkRBC3D{L{#AdKdyT^I}``vgVold z9MILUMGWF_xjp<^n9$8FRjpll>OhPfI>Lu$2Qm2#cNud8F?erm@&x!kmX@?xTR}-pSTILpo-oj7g`(HIGowXYL|w#n$hiM3G?`qLzim=E zV4$`o!kr+@0tNm5DOxU^M)P7;1GrrCa}u!Oq-5OQ%_Rq6Y+rl)S+hkokuEnqZjJ8F z^+dAlPlwyGtZ*$ZwCa#Orh8$7SRX(K83MAhqFGxx)00C0&e+nbqZE)8{_q208^Bv5 zGgxViTTcTSstJHSa{;Ewn@VJ4aZT(IWw_oRbJ0B~<^`cR<7&airJNcjQck*y`E}cD zW(S|SAU-&GpwG?)5U?5h-^RJTaxIbxmqNs-?%2_~nl)`?PBR*JDZZsP1Giw1?$Fx( z_QGq(d(1?P#myjZfzKUQ?}wpj^xJye5}EqG$q7ASA|H!b$GP0HN8W9eYsj}c{t}ti zy4O%NY8iHS2;=|2e#ow%8&bIFu%85K=-+pPGaX{Ut4W$XlW9;7oyaP$GFmxe$b}y8 zhO`F2qK8@n3b@>hj6?T8LIj*|XCTaPNZ|jAed|+dZxu{Su64&2v^#kd9 zLf>;}#6}+yJ_bL4|G4n_JTQZjF%wAv7^UNp_}l#v;yQG)9Ej;61WaK>OC3t5glwM0 z`FMp?qd8c%7#!rgJc5s8(np5@hjx;?SNFM2W|RFs>*hv9Hm5gw99cn;P1V!yV-ZD( zW!A>1-#eOBe{dvut+rR-74^OPwsS_&91W7wj*p<7_AL`jaWYPdXU6Bb|LNKa))Ydu z0-hmjM)`*`Is{bTR!q#nd#52~w($ud>+->xo1Duyj;S+9p>mW~{c41(7l zN*B!Nq^vBF%@25nffr@)U+pgj^stX{_aNFzESC38XR{=W9Jp!+ehq&3PKu3C0oROg z(f75(BmHLPV{~di7~j!(kt{d)b7Ba16Mp!}huHKi`f~+spJOF5&?RixzfR_yiOJ!Y z|NgBr-{)z7CjDow2mIq$4OLdi7UR>E|}&Mvza{+Rmxt&@Q%EBZZK+4z8q>|Jr*`p{UOy1!4~@*yp95vc9lhGsvMYjPDA!aHqoFvuv*~H-KDU1fa9*?0g16 zQb{9fD`yujsn+{Nafv(Dw#xJ<7wh2o0e>Db;J7ig*5gI@QPP)jtD1J>wm}bK$Ltvb zd7yRHs~Nbi5ceUbWje-ey0L|vWv!E;o8A|uAi&dee%q#7DfbsZUvl{S?|u|=so_3< z9=mLH`|&%{AP`vR^}CR!yK_)FGud}Ruz26o{vD0>A6yF7x{ce_Wz`WD*eY{OxRnuUZx)~jKQ>}dk5bNVJ-{;*lY0#UZhX~eaqG8ErG7Yh{MTLUUwIVBmG{Wxv=x7vo)1hI*4!;-7l zp~a#5LMtSa@;ZVr15ndK4Uzt6#Q-G$p)Mzw zp&T!&MMdGuLRyR$`3Ek^T{~urnrb}SqyQW!nj^P|EnS0FAyuUpSuA> zM4?2PJcAOW#ab(m(3J&ztMUN9?--+ti;YTh;*;b&x-VNytrD%(Z8iVW2IL^^a(_p2 z@0e1?shQ73cx;nJ$Bvv*!GHhd!m0)jDj~I@6IP=h4c96Q3eiF6;UZ!k|FZobf(z3p^L%&2WAsgZiE}U}#@J99$v@;E0MF9c!AldUdaJXYF#wG zjWHB|rT!K(JQ)YD{caniNtlCwb`a}qHxv}4`U0?W)(|(m5v(cL?!~T$PWWzsC_aLo zn_cEP7kb|2ark^*VF;Ds41;YGQExiBoUs_}|NeHB??F)fcuQGO7<@HoOJ@8Hk#9;E zbh_)bx1Pi^szknk*9rSNp^@lT+)WvquzE@++}H+zn5mif5ensdjdX%NXJgw6TcTvv zw{hWs5J?g&a`%v>iX!(VfsUfU(~WTzCKEE%o=MYN;7krOfX>&N!z})2p`wKI?VsfHP(uk)!&ON|)}=;WNcQE#4`-NW_lQdcIcL7(4H?uLB0|5kivA4ap2-$97&& zr5xXK$h9#1q8(Igwn8>INO(6?sEcKXG{n2IE9cU+P0&4UY7feHssVy5MR^Qz#Q)IM zX5K$iV+Lk+gCXGP# zsW))EQC21446a`ItbtimDUvWD@*3txl?1&JkkiGUQBUDpi9U{pHR>0q+KA z6uB9e`iY)N;;)AG2!@N3^oyk3@%o=C{Y2z?Hwqh0t!!M{vaLDkP)8~E^I=9`#q|!^ za4H=mFMC5gIzaw|Xn^sdc7$|>;XUE6NnKosB5`5|y~T-sQduZJHq$r<2l!@!SN_y5$~hxK?d5KX%D@5@Z6 z`Y2?;RrA`T%B9>Ni5B4)db#@8yNXI)4EdaJFkGtl^iO_i%I1KhDer=$tUt8KoE~pn zD%u>{`~Eqkk1T86+N)y~yj_W!bH3DNX0mwS2A`kK71)jaMLXg8A|)AJ{Yx*M%)<8B znPae6n2vrNR^ci}%>NAPFwGaJE>F(6$qUUdc#Hwn>H)QFG;L<5;G0`L7>+z3>Gj~v z-WL8gt>W2y(>w`@Vfa)H{hMdyjFea#lTmNJCPd>GaU~+FW+MYr%}HSA-${33fY)}} zKgWVfLgms+nH+cX)W`0huXKuO0?`t7a?C!gJSh6&6jWpl6;{)*o>0t=vye1!zC7u4 z3i2!GN1MpE(D;1!pw8Zg&oZ}`XydQbJvVirgDk~w7Ed6oF+9)~vZMy0&b%2?@q3T- zO&O%6(sD?#eFdNVSi2YtH2|!4?t;z4RqlQolOFw7mZSjW@HX9>w|k(xI6wt80B^DB z2RnLXaVsOvD3M)S>`EOa2785_>3Xy-PWj-vDDj%wzowQW5R=~cNE2Bu$giS16w zFV7sfS09%lu!8t39AolDZhdet;|P!dAbDG;LK6Mrw~t&?cr?U)U7*_AL0$&+O_^^1 zG0+p>#n_S$*Q8kfRP7*ObFeQ{>t(KXGYL;5mOy0%*Ol2ZutjR}eEU}IL4oR1-(<|^ zgLSf)FQ%YT2i$1&vYKSYy#pjgCu1r~36sc!K3VB;B9tlN$D68w$gyEhC!_`yW_E+7 zbA+k1tQgKF5m3I48$vrT8Y;)lieQkjZ3LGtVsy27%stdM)s!@)bI(n0il<^2cgPNI zvbmp1i^b-uYIL4D(CZ3Ep5v<5Wxj-kIbPG3lB)sQ70Oo!wGmC`9b-)*Se~Hxwt5py zjrZ=7zYffKAfgUj0bk$}t5LUw+urRZx!Incn>dCM4>E1LfKj|9H2r0$X(WLAbcqUG@46`{8VG=hjH11d)a%Srftw zRGZhX1Q8v@V^8yWE;@6!*Fb`eKH5b8X~sqPpc&g!jm>Xz_e#{5Gm+EDT*M&mbZFpx zUUC?dSJ@dzTW$QxrVs7GKyo2sjIFL8)Dh3fOw0;D+Q^ku_)!M>An4=X>kInl7j|Tq zr}M->H2UibsLWU;YvTa39Ba!RB>N6DJ*RWAzyJa?lbI9d00`W;+ZqoCQ_=~|KjDEo)8udg7)w+-uUM6PMToO{k{AxP zReHX4>eY`eODwtR#o5T8l z+ie8gFK*GsW<9!1-^uCc)GTeWA!`Mz9!E#@<8cBY#BsadUan18cb!72EUH(5ufXmF zYq@?CCimF`E$bztLQto>K933uZ@*Ap zz@q-^IhbGG227(wC|58E!&n>=YT!*1D&zK#6nKV$z|LW{_b5=$3r(DTFZ(<*a{7duO*Cb|LE zO~2DWSEQc5%jp`o8~Lkii6eEKa~YC0qL5fX%X6=0A-+Syi#LgXT?MHUfN{YuzuHeFBEvNeYKfD*Sh)3simb?^pn6MzQ%hFC6BRn_Ng7b26GYLnf zlD}`5fql>a$X>1|4b}wpz(@aREpf0gLBF5_2R6Zg5h|^SBqg3R12h1WejYqy#8TWt ztUnu?4ZZ?yxB`1drO&^Yz}OKC*M?2H)0IK4wkY(WAPT|CerEXnXolpJ1IW*qhHSf&%;$9@Z zSmd;U(fhSg%XnS)Lvto}=+2^sgDP2=q(DIed3QHol6=Dr1}u_7JHgA%0Pe0$YY@M3sJ02bSF~pi?{va1WzD^zG1+qo%Zabx$71 z-*QN32_#^;iO}2)@RN#(MRlO<8OjIhmq;=XPdC|g= zCHx83k*zOs2cJ*e9?^6)>;;*C2Re{kE4?1eh*-t z5@|~?rsiaHZ%7holQmufLvE;5T{#o~e1t0vbKR=SqRW*u(G>A?lPcgQvsEVBXV;U3 zlCRx;E}01;4jW9f9)~pBv~OA1PIgr2J4%(x7Kwz=PG$s}O`!_M0=Jq9hK52DKfUVE zi{O?bLfy!lYaJGFVY>@3#mV0bIsP+E`Q5qnWTi{$eFPnE zRK)O1zDRDt!{W9Nu-!}C0}0g&x&1gfOaZ=jF{;}F-q`PjKTvh$fx^v8vwB%L*y1(a zt4D3aL)hbHj8K+ovQ_wR5>2&QVecYyqpqb5SmXiK0_b^)RK#vt1kMgu7wK#1aM$?2rpv*MuU4(ttN*J-f#OB8wmP>;kWb&I)8ro&!euBGm#a*U75l@ z9ku0Q^?c@D$fkZTfWE?6$UOQV+j3_sw6ZlK&q40Edv5Y_u&6y;pT8Ty7;+T}C9ko= z-h#J%x<{z$w|}#2CWYYr*7H=Q*edI~GXGqeqTTj%+pcZ>{54I<(^0rgyr`knPzhgF z!&t2Ek?XUc?U3m0IILgH6S!uJ{H{zy3hsq!0fgOy*Yg9LCvW_ceBb!(GxWDR&NMnw zLvoX}yf*h{T-m8U|EF5IS{#=JIiUWR2_j1uW~E#AQb!9)lqT7^DIA(gXq0-&U!)Mf z%_o`yd3(IfH50vXq7cVY3CG&ez`q?KJK0KVf(FUOK$#P?OJkgK??2o0rGZwz6qbIq zfP1i_&T%3S3&+Bgl8i>mxRo1s)+rY;hm};i7WX=vmS~cT8&q=>?=HyzGzlq62=^q( znDi3Z_+}00_Zr0^&P^%~Qp+7f^D0@t2(rErj=MbSdGD_Pg>wceHyr%mNO1e~x~+-| zA3+N2e;^sQZQ14{eh`_UIMzi=JezR(8Fy~Ou;CqTF_zX*0*bQ>T>)3)Bz7cwW?$W? zuf7O^I=TX!Xjs{^bR{CIrK%}^{H{PI6l{c+Fq0mL*)>9>X$YAEW7G(pj&~v`InynCPP|Jn0rx}~xIcdt-S_ zk_(geE(C%|`WDh1K{s$v3{!-eVG*KsUYP6&hr9KqIhoRBE zCU<`P#pyO8a47A*x|eA)fzD~=H&uTc-@BLyR)+!0Y{)Z8FSES!($}=~vn%3KMnzv= zBBPEI!(EMXI`cZ}Q0tJ7fN!mRmBGrC(VtFt#>}64Uq!)I!fsV{t?-^A0(9xLI32PG z+yL%O_6fklvvlOwj_kM&A%dX_DS}Xkn0`LGYHvUH!!8qK&DQ=8&fWF`RQRlGM?jPc#zmd8)-B zFY=3?enxPMaq8OC@=^8N#lEF*kh$3;;I4^h&Lfiiq>xHg+cS%C&^u4pk+-U9GTV7N zl~PkzyMD`;R~{AgZpd3_}vA^VQeXDFK6@M~#Ip zHv-Qg#jEvtD8zV)tVx$vi}6Ev4tT}my-`fp_(+8IvGUpT4-S>wrad9dX=e*Emx4=q zif#1ctv*B7)~#->cQ2F3s|X`*-y~D5Mt<%E5daJwLg+&Z2@S)t%@gK^8cwBj8T?+} zwTL{+&c5^yJwnr$gQEzT;Liu6jpgA@a$P=4e@+@rrMD#LlSC=%q0{K48*`lS1lqWT z_Av_$R#hUoK;qzMCjZ*ga@uT~PEFE$Y5n5-H?Q|eY#8Kj&CcQxyQ!>;Zl6V=7R}aUCRN+=wS{?ONoA`;~ zjIm#KP!6d<4i6fDv9_OLJa|&qv77 zXmkI=Grp9B{^L{3tSnM2*+d+N?>q`)aI5G}a?MaMu4hwoNBHC}eg}tzJ^GpgjJ~iI z1UpX`&=&+l@v_Ce3C0|TC({(8;jn`=D=hFQ5=Yn%-2f5u8!#!~Y7k!gX+5i$@tR{P z1z|6m&Bs$K!BvBMj&)C<$9Ln`SW&jqZ)eb~Qrd>u#`A!(vx7dsYYzuzar4)K$RTtu zaWwb!^l&LWI;PqvGBzhoeD28*kszn}}= zo1z~H-FdeYTeoitMz3ci_1k+4c(owfn~{vKzBlu^STfsY&Gxra@2LAsIj)?z^&VO6 z7P8@@J4D!x5~^mW-!O~qg=FrM2Y!ArOw7?(*Fdhi`4>ZHbhHKe^YZl!#$3;|J#mwI z*DH*VYYgTY9xSjDjHkRd6|C@^V6^wyugb_i9%*4jP!S03b!^WrLFpy|Y^IM=WIsY7GKyfu7Peu}KD=G`Q<#~-YIRtK;I?L9b$nmG`tG;AWNEoq@{qkU;y16J%P#0_0%Va|KNz&2p7I9S z`}ln_z>(kemu|zr%q(wU(Ni%dikSsASkLW#;ROW$Ep_4%6<9t_pErfNM%d})B7Ma^ z>z~6St~Mx^DH=Uld&*V=8?h8~?3_7HSctZX*J#AN`W(rPOz0g(48Vr3xF}S{erbT@ z_S!pt4ik^p5h6_ze`@^x8eD<>vWh!i=tt;v`~m}a>q>e=@Tv*wa}QcQ$uhNIo7#HD zJzl-Rvv578m?)OA%B;`WY6C*E!#}|@H^t#fnTuWu2Y`PS5dnR9mF2ypDjrtnVLJdj z_o!1l(^h0fT}r`&^PB?NCSSli@39istq~&y+_b-5Zt;adFV-1|c&xdHFO!aTHCd`h z8A;vbXG4~|5QZW z@5Xzh>I-QJy+dvbD?IKY+uykM_~EjeE1aH>uZs4g-fjK@Eh3GcWX0DV511tmG+=ia zKgGOM2WWp^TRo{**EoO3GlxakuS@w`cSZ~{@GNI`=)WVTmk%J9fQ(Qu^{gd{HMP%L z{+9>knivT&=MHO(9eH>4@k$OGl-dYH!jSmm{rW4gL$@|SN^uySPT|~dh0ut<8Q4m| zbI4@1|IIy6UNq@$XJw;FJ8f#OMBtt5`6oB&-Rq!W5LxD!x$Y&r|bE~@O>wwe&wD%gkm1{W$Xsg z7nh#?`e^izy?%+zVvS0B3K)o#_%$x35+!9}JTC%O@y0R&f510L7bFR|Pp)19>738Y zjDtb*Ih&2E)>nXy)(BPuRuLKtOq|sUf1blnAnJU$GX}|Ww2+3a2~k2WO4`^@v7NR$ zUHlK(kMk70Tjt3IHKE(e0R!72H|>enoFG8>LD#e_SzU=i4;1FYLje3Qo<0E!t!oX%3`n6X=?tiNJ@^<@WcA(n5lLq%2uRk_P+}O63IovcOzVI%}>y z$m%}JU7;n_FZ8iWV(@BKDaL(w5mQ!tIdh~8-RC(xpYTVT=U?!2^Ni+xi8m+Li08p-%yqT4jvGG zXyd>QnQ@^6Zx!7L`I*a3NJFgVWxVrK6vlYdFhC4wlf)}||NgrrrS)>>7CEpt;*G5V z`x*mk;zEW~h1P%aywDET@s=!ea^bsvP{L_Knpt1uGHLOst#gtX_9jalOgy9np(su< zUQ2tL57`qIu}?(eElx83dFp&8eIk>@l?)!oDe>|0GNW4~-r(Kl-)w;a`ViVrO)#Y| zkEWm}XxKu~QpgKFz^m4I)^Lu;#dm{E+y;j3dZ+wWpT!oPA3yUs(JY1wP`Wuvuj}e> zPw?CJJ0_c*(a_l7(8ccn@at>2Z@9+&D)MQ3@picGDwW#F0nUL-FE@C`pTcSLJ`?14yoJPc`#@MI@MF` z(Fr+&QV^9sdO1B&9N}$kjv0hKU_fD==TcBm(He7#dH!i+QpGW1Qisp@g5imcv}RFVh{$KYyY3sKM{j!qcc2 zfz%$Lunetj={a}$zd;<0bfY`n9q>X7GG50S@^Pu{H1n5$nP^Ez_a~raWVxwSs=M1iQGtkja|Eu^7VRdr#Y=x7v_nE43-B zUpUVq6RW6~kaYud>}j4Jpha*O(JY6!pbW!UB){VgK5DWjp`%QP#P}gsm*wK_1#@Uz zl1vfy8O&8PX;o^t_lYc{9JHPpeCs1;5Q1nhoqWSWI5Rv3b+}ElWi>1^j|KY@8wl>-^SzEwF)pvy;|73Tfd?BB_@i1n8pn%bbZO`{Y z=wJ=}k^bkAA6JY1bdd9o5yiDLOo9?Ii*Pfcoh$jSGRS)GpS2zj+-am-hSG3A?I?0U zfBAz6Fw}m@kzNG8zfgdlXqAT~9PUYu`+*XCW0mt_eJm>Md!)KsKD=|jte+6cjEj^5 zs0_tG3jK)E3(UYRqR$neKTui=rW$Gbfc>ES{7~@`f`Sp(Iq@qgB8Ee48ip0xR~Zxa zRW*z9s4~;3(Bb~Mw&vPC-+^4~5m$RlfU;E>wP*o#@SHgf;GFTCYj1ZhJ<_kMzNPCk zJZ$xfpGO~9857BHj#nlk_32E?4GK67zz{*{MhoZHWrEs7ez^>a9iM=TUD(mRFIF8@XxYmZ zJNIb50r|P?W9Ub_d#t3erC&r}EATt*=LQP{%6c*H?ufRH?9k6m(3fdL)1vURiUxvI zAFdO^2ouP%oR~WtAN|TG%s+#n13xpkG={yBZD-*Jv&M6Oprmlu6^IBmeQ^r9_q$Q1 z7-^*LDMP5+mBiGuU^b$I^}p)K->O*r3caXp$st1>V2SbL8^FPA$uE{;Rg@##YUaN) z1_AIK_A(WrhIB<3YLjB}5Je{xal>~(ZwIUF&Z$~+Xbz{=zcpeJg0$Vm5$NjDvS%&xY!x~7wXnAm-qdx#SN!#s1n`~4RmyM&y-L9y>7c3QFU77b0- zmztEB>RPe>N(0*}Dtvw#FesrHsPiLfC*pJ=xO01-ih13QH~x9>z`ccl!HQN3hWcVDwek zm|a^Q$k0vrhFep4oHe||v-ze{O(T8T^ko?&;DHyU!v3dAlaIsZH$nB(0UAT#XICr* zrf;=e+Nt* zMWq8~TwL#mwlfE}`x;Ngy2QoaXY1HcoYWqJ%u6|NEYR+{#%M?FdjOJUJk8$KvCNZs zgPX{=ThMqj+Er5TBBIXr;h7A~HhSWko*y``;-d{J5=uUt{`1QjN*y+F=~=m5J-{DX zO;#tQSP-JG3k@$F(3&Ee=|UoDiouFF>e7wAmqcN=Xs%SZ|FG}Je2o!F?&(*N0Y?ik z>*S+SXz67#OhU9@Md`z4r?iPKnp{S$rRSAAEnUJMVoDHXTB)=;MePFsDw!#izS!pa zq}J55J2PXIY<6lIG)WzDTfNUAaR?~c4CD&a!_$6gS#Hsz?2X<}ssD6T$^(E~^Ys^X z!sXde=SN}{%Ohe-KabiYeur1Ef`5!$n1eQm*BG`1BLRY!bm8#1psyG_HWuwusXr=~ zP{?ESUYA8yoGZwiJMH!<`enSYIt@3gOU2*~q3vj$fnlVJZFE9uN8%0XROtcMBkWo} zx{PM-Th#G}sMG`%P~v8;Cn3_KoI~wc^x(Ufqbpe#x%$WA$d4LN7xlQViYpLA|0Vq{ zYC416sMTb9?TExxjMPjx!M2?s%oJY)Q(4E)an`uPayiOxNOREj3_Ib%R5;%m1DW&iIL$ZneOjrvMtVSn(79V=*|(&|4MP3 zF4Nw1V{iOU=oGa>*tg^8hJ2T5dA!QQM!pz{-whd_kZ@3 zN~N*Jq~F{s5}NK4+emZ~}3U6DJ8*M z<1sSkAHgE*#q}lM_mL=GlI-K?VoxLBRYMQieO=suW7bk9@rhv;C_b|ny!~W84&A!$ z!s|@i;Q3|UILKObw0Y`_B8LyN>|iqVs6;liBjGe9a1r637c$xA>nGC{w!NW!48N5u z0O&t+EgreX@tJx>R;l;(I1B!fJy`2!VHKBN}GeG2Lipg}y& zK6@pYuMMX~=)st*UF&+3=Jj=B+B4v?Np%z9q8x~$Lj5OC21Ib~PXArLf7`7cUR3(W z88MFMZ?p^jQ1k6aOqA^r@xcSZ9i>&*IaK{ImYYW^58B%Qr=|gf| zmMu=Emuv&h;mtAKb!OGoYP($Xo{o!m_u92m#8>G2r959Y{rhUkr_~(vrT2KWeqMq( zRd1(2*fYz2hm0*>qJEi9#>uQp^lb-cMSQXdcQb^Z?`NHuc&;`wXT})yLQ<_fYSXRv z!0GWLFBTHv_r>ddCC-)Sw8r8mqtHWVZ&7@CCUqsBD`6GVi!}C2d<`FMM`pA@iG{KV z3UmgulGvV57$R2dqIxVZ)0=2VRN$2MLE!v~2KUCCC8-$>Hr{*rGO(dzb9fe~DS6HsclLWRKU5 ze&Q}B^ahyVef}aYA{UuL%uijaVN)q8`};DSQVrEVE{Q5%xJbmASoOAOU9Y=7#rR=Jqbbc%q<6}*J!@1#p@ zK)0zW^1NKFu;)c0i0X!7Fhxl=jjWjm5K#pT0r+lhV@RLn#9l`c<?%0t?yz}F4Nq^1b5&rjWw0Xd@BA!s_}secsMDu3u)UU%?g0`RGhnOw zi@;nV8>w^+N=0YUY5g~!7_{bIPkKC#qEN36(IWpafE%rY#F1*40n756e>4Rq?sO`S zMo2X&vnBc;g?*z4Z$(ev@BT;v{Y-o;_#0-M$r}7T+r94cwf_xw4?J)PBr}%>2+bO} zi$0qZ`s3)y5~AneH#U6@YF$mp=4Ns{Y9!&{B}Aj3iN%FT6B7+ZQr{duX4U+)us0yb zo1E8hy_93Gs|w4=ph-@v?q5YT6P&2tA)CWCdTKN$n}iotPfLd#dD)L95@@@s0K3s* zp{&ld%Gr(Scln2*AboszxAc(vbF6!9!$lqTDzh%$hlH)L+|_SEFZ`%1OMuNK8QmO{ z309^gM;>boXXxsBTZ@nu*NTZ5jhPEifzCB)$f}Dp;cyF=Cr6jmPJ)}T#g0ri#Hs7= z6SH>aNanL-YRi3OoLEx~*O-FY%qMzo1H|6&&0T*_KhrGH_x;2Wk3Q5LhQ5g;JLg+3 z51v%*GaN^uS>yRB;G!9+^F<|SM!745 ztcd9s0f9LnT3=Hr|DAcQR1Abe4$}BQmJ#pQ+YGN6Lu~)P zR|4H6Xl8kA&~JK_6s~f3cIPrUnl7M-7c6L44OG{2ZQ%>XW*JpSfhJQMvL$Bp&ea8W zUPsWts9v4Uedi2$Kxoy}?_Xm%hEv&>kkQk~P9yu2Qs`YEI~S|oPS~a7{mu!}JgoK9 z>O3Hb1swwZ@TGM@cE5r@hHoeSZmi=z>Cx$j7SWKmAaHA`lG#DC{Z#MRhP0oJCcj4c zXhJjkxC^gq%kr;&f3fs5DAK~c(`FiGi{N=!w8;a=Rb_Hu9(e^!uzV(aVpe28OU_&? z5NED^G-Rt~Nk6IYw|t-D+T4I*(It?vr0u=7gpx*w5d&pLzDQSPhU3L@ZywHl4){+m z%s{#8ND;BWp1Y`!Zy7rAzYByuE^M2v>7 zj4?RSGUamQQHkiKs?^X{clrqxQVwtC&AkW#z@F%V7wRVbRPBS;0(}WAClm@0w~23} z_20B=T;ry_#nl`*iZ{h^(*;MOHsVu+IbR$Grxx9_UJT)JRgA->8MQdLIbMHe@oz8D zR1VmpI%Tsqo22@%xzBTSE)Ujg2O7#^@r|#5S=mFH8H1ua797YJB4nSJ&+b!PD^f04 zkP{@qZk*(9qI+Z`R+xF-?s7YW+{H3w83NCaJ1nvtp*9;32Hi7h&L~^cl@RzykZsx5 zi!V+n&Pq{$SD^GHxu7}Sa20DnEmXd~gSIX1N$RYWS%O2>4%@AWCXogWCN|vA^I9U#5 zQb!>F*Gt4Y@u33s0d>8mO->rwP7ltWM8^r@L2CUslkJ&NCTK-9TWyk*Xjhu!B`WEH z#H(Q#sl`L%pACNQHyU?~mFg{8QVZ+QXK!lb9A;?@3M7fk?X01RH{4>NlVU`2OCsMP zB&38m6jG{bp{Wpg>!oYU#f`9s8ZZ^m($rWAeG3cd!=cX^dG>qHWs+Kli^Q9~zXPb9 ziC1&X>7Z#GXg*-yD~1);#}!7+sXp#|ZxSfsn203tIA3&U?(AVk|?hUxE;r;cw^LUSi0@c0xlroX0U1xV@+Wra@U)Qy(d80}XY$g9*vfx;kLu^8qmE!EP}Z>1yK(c*UBYP;j%y zj59mjX=0q>(l5u%M+>^MH@_uQ$|QNSEzCT+Qu7Y+UR;)0w2cPJm!oppWUHXYMtNz~ zTWJhkU|}D2TGMo(K6y?9!kNGOcB_+YBv4%QyQS}H#=TBhs=6HDE=u}`PaoCn1DUkz zr;qFv(;q#KD^JdohXfR}GA9Xmc(yoz7*vy^9MYLU^&i<$qf+MHynOi!A|HkYRvWqM zK_Y!-DveekV%fzLI+f|3XYB~?l~BTD=?m5V3zyH#wsT6^s1 zm4oB)^8n8QdMrg8F~&?`b+sLSP#v!<{at4PpFj)(4%Ox8L}@^D8xEm;@5I)sliqRC zEhdbO8?7iT;T3eY0aSY^nllSvi%$)oCJIU#-w3w150f-P5Ur#965)LYoNNOMd?<=tY$aP^UJ;EVedG9! z8%9`JdNB0Zf8UU?JiRKCY40I5%+#~E70PO4Pe(IZ-kDTDY+$O^B8Z9Vz~)2ribRs~ zF;pkYg*a}nunb?s65iSWoq|&^yuJx$aCn}y4ECE%G*QomjSEuTh_W3D+W1pJq!7Y`v zE4+LxJW`tA0E)5d(I91~YqY%J4?>#($V)|G-Y-|x)HWeS&y18u0-cIc=ewLK=Ca7T zvWJFLQqI`v9w#VBLB;@TKGU)J?;`d-C`Rr3C!p7!(C zuOiMV+q!dNjP_+Rm;#Y8||t{Lc>U!NV=|5Dx*_Rz92F5f|HL8*8|(+lt>*} z5(vL4h%IiHOmJ#Ny~SWC<=?EQlrdk*>OnvJ(DUr0y`{XB2}rdCpv~2(CL4Q}pzX$z z&ZV7&M5n{|+E5q0b!E{sH%ARgllwW~Ktrxug1NhS%oDnNwoJK686|;(D&xM6wQ~cc#AK~|?X#&`6+}(Rrpq8LWZ{4}UnrI@ggD5fY)j;#L(fHc#KAI1 z*#_`1&-t_YNVKwFz^Xv@5~`Ltd`^ghl~U!r0gMs_G0TpS5!@VQ2L=egS^(s{j1oETtoOB`|5c?)nqYUEPw)Us3Qb8hxYE6JUVZqyP^cklRg*Upj zMMjcAxQyYxJPX0Lt3hZ*j5H-p<5n~A&vd2S<%%L{j>!wC5$;3?n`fV?K-mP%!Pm;n zQcjbT3Td>zinlEWQTj{w0dfrMw8mKAAttp2B?r^K4rzaRZDpzona?3pGOO^T$$(9R zlP{kE<1?DNKsKRdqjT9%+7%sue8@$~ANYkEYgTYysUtKs*8ju*l`7-8Olq#>*}0Bi zQ=`|vI!)u=lc8~10?>HbJol>N23tF&#HVb*KB%ld5EgsC8B1vV!>MfG8%8%Z(URMp z)Vc-=?yA*4+n*}LHHkbDqh>j3D-q(J`J+Xs){C8Of1Z24gJKJ}Y)d|<&W|(l7d$rV z|BJxuGEJ@v#!)RuBWR%&((wEs(<&%(@FlnW?@%NQj8kN9oIEI)kc-iSw;*|e8O2%J zW72CCDBVmKV2{AcKI`};Y=LrsvVHgDXA;8aWq<)+5pL!cg07-jzSRj(%7L?KnOXBz zU^sbo7kLEJF@+!aW!`hx!dJ##ZJo_n=r=>S9mTg^Ga!G&?IUEv0XUrPp$sI-kb;p? zIPB7EQiGC%u=L}>cx#Gu-y-#i1qc@oQD(;bT|!2GwoOWzSkSn9Vk*&&V5F~!)yaqe zU_=dJu1tsLu1cev212-2WRoc(R*3`7b)l2h+3?7dr)VpLU{k z6{{{fI_?W@r?+4e8|GV4!ibswHFQ_$A0Qu+K7U|2V6Xl7tK@Qsm7n;8UY7%uN;a51 zs}$kC!1I=H)`oa?OG^Xb_(B`L*0dy=Xg-IMx>U1grIN z)V)1|W=Eu~>@0t@GJN3so$lTetx0|ZJ z*eH;K`GcMeD_q_XSHw{1M)AJ$9yaYr+9#bPh~&^e`(Vkyx?uGLtZ0K?eII|qRa*r9 zgDnHVb{XN{XPo%fWXLaU(0RuJuLl63AxhJO!SB(aMj1YhC463~Slx_T-Tc;IlVf9h z5FQTjdTZe0vVBMUeESHxHBOR)kY2M1*CO~aq9JTY!qSA1~8td#)Eh?{dB8L*s$jOvoyj(c0Km$%GM z@hBo*I}n3tH&|txS-ZNg`EIEu<3`J%I??po2;z+b109t2HgddNF%3HE$0RpD{Y!44P{?w@f<5Nv7*LSLlJKyV#$ z1M4HXR+2s!C5Stz_zbL&Y<Ok#0kec8)<+JaA|lD1tR&L|D`w#?8fIyLMHVy+m&zor8!= zst^vqLtR_hu0f5Y^uxC@v$x_cokdW_?c^7r&rUi8Bg-OZC&1@O4H4q3AqMZsVSJ$j zbv-=Mb_LP-o;aQHd4$Obc2=BM!^5*zU)Io-GIa=JG42>JZ{F%14{2iGUFA{O`$4(ne;KV->raXbc4RJ zv@O!y+aK*R>t``LyYnNcVKL-!5DD8ky!>&%#@U>UgXOc_DM5Z(lQV<)v7o6e*vzmE zRTY;Kh_64<6@@?X)f7@NGOO zhf?WbGVU9s6&Xq@24toMdDQzH{a17zXAVN)O zhR){bNw02NM<=E*u(t)OYNzwlkzcWVQLB&mvHOP&n@UT3#hV8ehB;bsfBdvOkbabL z@df7Z#%J{WB2zv{`%E#h;s1D)p}`;3CyZ-|==v?=U6L4FiKjQQ7!p)fV!f*s8?}MM)Wr!F-)G#i2+I=*~OA98K*U^9gtRSgc!R*Q$|4Gs9^;qW+5&k{T=X) zVod0YP5%S37ve$#|6)iiPE%V&>OrWhGn}?iT^T{}3P|g#!2glSyd>xn zhd!i!Q(K4!^veFnZ2_P@{!+Rb?C?9>?*}y=WCtY}^%cMup=z+EuQO+)98`kmkXqD- z!~eAk^r za(8(XAzDxsj9u`v0yf`4O-rE3Jq#X6`}Ed&BFKUgoM!fR>nYk4BpXgxi9!L6tR!36 z2^tNE5(fi9FR###1TeM~lUA*^3?wQRU89(0Ax)^NgEA*~0?r9OR|HMLzoaWzCR=33 z3n&c01l2sD1fmd_TK46aW!^SfNFV# zFbML4IQ^Zg&+x>L--112JzMts(f3@D>7Gk4s@N`daJc>m=E`_N?F&XCAj=t^VOBm% z9!nmsqr-CYBQYcFuA#=pq=zYmi1A2u^&y3qw@a91(CKndDj6th0Kz(}SD~L=sD6MJ zWj71|wbe8T!~)1jPCy)7z0nq37Ysu$iBMqR3w(jin>5gWNHco}a^toHAW>MMZ5PUY z>aRn#hw8*zARX5k_tF>dbNC}e-GvyGlco%A=SSWyn0o%b;hezOv3i@)QF8bhlR?AQ zw^p38M(rs{3a6kNac^W>)iJSEr38D0nP@!Ygv}aI^r1ln*xpHgBcda5=wTp`vH)4cGfShi(60tIMHfK)%?7_k~C7-)B3n zsOZ2)pL?bH5beAbNCb~jS2<^g1gAhQ?9b|~bXc*Hr<{^-*ptEl*$*$Jz7`G_ulJ|G zA;YO3cK-Xuk~}y&v(eFquB0G)tbJkx8SLKqhpx=pf9v7kA~+d;#@PaWdRmM z`Im*HTt43bp|`|nQD-_j0|Inm2c|wF(s;0{(gU;Rg=V^OsmqfjxE|SXie|>J#}I8+)1O zJjbBIO8YlRENVB0{TgyZnY~2wTVUI&*#TD%dUy+PzQ>L+(_cSGi>Lr8g&Znd1V2z6 z^^V|_*jeZ3A{2`$wpHS)LAWgc7c z;e?r$PVkagzzBxIZZ@7VI+!wLKLv6rk|Hp7l8~)z@a@o{>dZQIa1x~UDZU7Ap*K4G zQr-OX#Sea-Lu*6a0rN9n;32+B(o`^4vY&%1D;G#Y7$#@p=+LiD4Ys|7pjsBNBnb|h zjhKrJ(QJ?0aZ6v3K2G4?)xdM>#6Q3-L3|a zt?1(`RvQm5Ue4AZTC3P z$C|>&;Pb*%WwN4BDx0!W$;s92)XkGLu-Ul#LjsNB9jY9EHekze7i>bLnfI|AA5ZD` z2!Qet@gB07an&N|kyEa_vLGP07FASwI;jfEDD$h~dWAPXK(r&&MDerzKg&6FJq!sc z@|FHIw6w)Y4gpAZXePMVZ=7#2nI@tJmfrkR?EOJ%v5FCUtfQp|Z12{sOg{DC|5Q({ zh@k3&NC9h`|CgAdlC)9hNC&guXwosiZI z4b#ARtD?OoC*dcU1>;! z4G&in@;IR$pH=cC7~w!%)QSB-QpgsYXrbK-$WUw*Jw-zaeaR{u57kgzkHI7cKI^ z+U#(u44n2jq+%czrTZ(99-v>(zc?5!)d*K*r3usO69Ck%BM*jjHTZ0BLg#}*mx#%b zga%|s-63=>2>rkDD`t8Am(rcj3I%1Mf`h3;_;VwL<^@Q;Ca)mAAJgW;?5_g=vah^A>K*rc6Iepv*wA{IJfH zKwrln$^&lV?kEoZaK2Czp1i<2eo><>ys~MqRT?7tJ^INu;G-|t@<+z+;JM>{b z9IMNkNynGgmVL)jm#R=!8#Q1cGDY<(5oQ;%JK^$qJug*!6ULLf@Yt z79%M}GN#YMX&BG9d=i~rfeP}UwBlbuu-RsYFGL{G0@a+@Tec*+mw2ASv9fQFn;IH$ z25aO9ToouNZ!Vx){Sf=dlmeo}Is&M_!5=6tXQ+_W?5M%Dz_-%v#FXrHCC;VAu3nhZ z4Ko+FJelNfm5pdcISlUb3m6?uzIREcO^a}WI|dyE93uKQ#O;EFG{9>~?8=!WGGeIA7d@JS!3>XG0t z&!#3FtcsANAEd4vbf0`vXcMo!WyVxwSXN~sBuc`);?n}dX3ZcC++)w`S`%bf@u2E{ zwwi1efLv>W)_C705T*4~Ga7u3FyV?qcw~jpEg9da1ca|Z-TVuY+(yE1u#XbX{!@Pu zxhPDYNeDdRnb&jn;Od3Snpm#SjdK(8GH^!(ln3zqT`T4;E85E=WzDQslDiSPB@na@ zCMaGgIa6dOz6Nz(>~o_x#6tkM@H`PIL4SLKvUq|7f1yf^lZ~ueP!Eq7IGdK#_9JY* zVN016K)fSs2i$l>1Wcqll08d4S!0~M&^@CY{jpb0jCrs|3LJ(8&z-$QjF@jV$3hj2t1K6PUNhZI`Q;$nOUp9%k2m@>c$ z=_G}w*@wa=&p!Fitw2v*L$`l1#y;Ef&XYI3K*M5|p!uWcc04iux7*W7 zCMR?OjvYt*<*X>KxnuDBy0F#IUM4zxdrY3F6NUztG*qM;!H{$=b!b{Udn3FDA-}-P zC=EEIi@E3!qm^K#Z4MC-#0@uSGG+pn_tYY#!m`HOBK0$mNypeQwMc#AOH_x%!153Sxv&VoyrOZ-_3+~Edt%n6 zP_Wv9aM(v&Rus&3U6X>;D^jITAn6x(e5-s-j0CAayRI&s9(E04MTpVIn-G*WBnDnQ z19Z>dw~IR%)(HAyiZ6jLNjMGkgv6DeIFT3Ib> zpP5yV8ZVAZI49`l^x+E=rPnhWX=wpec11K!F;Fc_)|ar1Z;CMR;`y$Dhpxy15c-uh zj7BnT;O0V8Eyu(4p$VQ{EpOp+$LT`F)5KGP9#9+nm=KnwS)BRnb|bj0p3)>%i+2GK z{yAFqdT15PZ5t7=<9NJOg`fF)kq?xw=X%(A+uTHM`OE#~yV#XEt{}FPm_vEcrgcy0 zTcCu_!^X`(eOh_>45?dQp!h1EF0cxw&AQeK?j2fUZ?V`_O}qRax1+^6&iG48WIu1z zRP($#m&^HQ%)J?HI)}%dN{KAcRpIlqmQolJO`;l&s{T)*q1K=e7Bs0!xXlx$(ry0k z1a4LYiN^*#&_9O&FM91ecjZO<8rToT6-5SW(7(Rz(MG61A3(HtZ9@plh=;2qo^(^b zk3W8mFS*d{^0f1z;9k>{YJ2nJO!;wPKt4i;LoBNq1v1zwT^%(dYX^O6h}&iccORBL zI}A*7?h`$bidf_=b_g4ZgVFaSqE-`b3|gRyO%nLaE=_Sxni? z?h0nqviirLchxfEKo}^)^4#~JItLk}954Gh(#nrKofZbUpb6kk2&Lee*kdRqH>#jj@1=wN(+H1}Kz3h>oa1ICa-$*C}! zI?8|e+SO;W5_Z^Z5%CXeh>Ot*L3-0jDkzqyw4Pv3OkZDcsb?LNZ?xvIf7BL*E5hJW z^NDTq)E=B6-F5$I3=cfP#?1E3u?O$oRT?C0h8)VdzA%{<+Th0?I@g!Wgb#XvasUNi z0xf@*Zze$HL^Dp3Qa7d*e~Sf-A)VA;$&K-!x&&zSI#eqs2fQgt{0Xc8GS)BV=f=`2 z5V5XN`}~(n6iLMV>Dx>~sma;Nz`=&~oa)jIR3E5J2xS=~OG_KR+Cb7meFpx1t|WYD z|6ZK{oW!eqIW!wSg2~2;<23xkJX1nG#i}c)d2@z}w)u3b-2BolXRRcqdt9(Bjhl5F zQ2{g_CXBD zRc(hX0Uz1n^z%(EkG+{rHX#9i7ZmI!lfahf1l*emDTJ+%cEu1XZ!O3s}i0i7Pl8@YHaeEO%I8vF2hu-ZC~*c0&Q_OxWaQ9&R$*!#|8S8 z{T|_WALA=|!%KMQ6Uow7?@^qK3jH5ytuoC)VZtA=w1lNPi>OKBOv$`I;tkl6fRby5 zMGZmpPa+3FLinRS^B)${`r##OSi}tU%T`-ZmJpCR`SE()94+820~0Q+webUcV&{dk zc_6B!#x2DoAYnr>QBka>AV1+rku=m}{?=EDN#V^1PhwQ5NP%UUF@rH+rnc7d*!silH=xLxFeC-i&PT>k0v{#2@Ayw=|1slb&^3W6mYX1O*A{v*Sb1_Me4(*xv z063R}C^Z-(=0m6#t&xX$r$hKr zjhS@m#nB3*f*|+t3vS6{Av1Vt_uGTrng*fckC>j&F6f`LCv!HmUfOr5);!t`nlM^B z^Qt=($A$YvPUbh6pNmrH5&$_s#=p#QJ;K=^F5*Wy9eFARaHaWnP*H?KL;N;w<=%q5 z9#g6JAmwkJzeA$NS${Bc>cGe7trYRwcDtur53sE@96w zJ=+iP@g#nH&tilar~3;Gf>?KTVm_e~|GW8vJ+)`x9k+X`G?3Vhd+L>7P6%NXYe?=6 z!QC!X9rP;VwcPBGt|e8zg%<0@9y;aJny`d61=|U?Xr9W<8b=}P<$(>`gO&G=4kFX( zK_Qh!bZSo@MVZ~vecXUELz)90bQc7{$Xsn<#7?l4X|x1irb*@)bF7w83m=J!Wq#qxefF6b=c2%flTd+p2)h1jL4b zYy?$Igg7r>{;rU%Qo@EaLb6S8rAs^%=-lX{^#^YpEE=cZFU!&p&IlD+x=_HdB(9xUS; z!Skdi3^j`a?7aronLhn0UQcUX?RJYB>6NJlc<7j`>;ov$&(@XVI+DnG^((nt;0|LN z|1kR5W4P7~l9&1*`Dg_m=l(XGPTXx%P**KA0MW!CQ6C(mbDFw9$TuT_B1BrJEnbuk zfb2OOJ~IsqZwH882fD!igZ%dpH-S<@`~P=XA?VRJS0KOe->nj*LWjg->NkIs2=O z#;yp~_)B=6^vl1Mhod2CI5}%L!nWHIcnC4}Izg)2V&$7|=ZZBkLC&5{gD^7k78c!d z=;209&A0zg7r_lY&Z(npZT0RLTIE)(aYzs8EgPm~@u3r@pbz~z#V{Ep6dx@o?9*wv zCL;agnU`2)#-4nck?2U++eHq^cqiu^0@|5p#o}XegqJTf+JK|lGn=eNhog8Y=b>x> z86y6q^l}(_@uUl9|1M7PiVx84cI;jPV9zkQe&slnKd&PMop_t_>@W}brrb*l%GD&95xX_bV#3~aXN5i92` z+@D>WQ3Mp7!*(A>$arS%>r9Bz5;3AM~{+0gDN23ETA;(oJZj`n)b*wTY^<5Rgn;> zL{#{{;BY)NY*>&3K&ccC9Fss+3OF)J%a%VbF=wO-uH!(*FPIcS>E^{Ei@Z@dQ;_#e z)k1Zr&Gks|3(<5ArwawhuR*H#W)Sz!y4mLWTZ{DQzR#uOe_1LRCR8GPr}gWZ#XOWw zq7s{r1dV1=Hce-%c|ZrQzWKu+5_a|Jz~NIx7kqU_wwEy|LT6fE!9nZO(i0=?1G0Xu z4-Iqq)l>e9dGE){Vgxz{3_>7y@ptXkHcMvNCaMf8<(8X#7IqBVG71%L_G*}0R{V<$ zud7X^3wt@+57g>Sn3LhNA}Gyr{!f8pv?3q0DaCQ7aCnbY~m5zDgsSiuy0i0XGZ$Wvr9Z zdZ0%JeCB$vx>ldCNvQ4$g^x_G`u(DQ9B@zHs#c?lhB`XtX2ciNx^J5|G-p#sFK6GF zJX!lMg8-TmrRm8)ET)zUDXQp8fL2ctCFZKYfeck-pvHGSH)U|Z$bkPI>#Pe27W|pO znH!-FBvBXPgCvC;FcoCA=dX3;@ox2P-1nRW23(s_oFUwLZQV8O3k1CM#YY~wkxfzp zhA0M?4qZE|jir>V-mbQ4ylYHLJx9GXnK`HU|EfbPo~swkne7{0*SRx@+~w~=70XjZ zj#ZGf8#7Of=unxdXM6AjNC8)0e^H*JDY5dGDP7j$Bg*slX1jksTgq8Dji$CtJPHP3 z`A?q*hUi=lWtHhVl2e)jPP`CF`kmXCoElfYi6!ap0MgqdMqxrY)*5u1ofQ4ddj1jt zUqs#|kg*C~M1NFiuCFvh1U=Pa<_mt?FI4o4`Z`HU=>FZvU2=4u7Nbd5^wsd=yK@{Jzsvko)fdsAUDx;_@9E(t~B5 zSK^3K=9Kfcy!an&*;&>Z5Okto;rgx)mv}{$G#pkp1)Fdpx7(+i9;1%}rO_X-42r(k z7UH*NY*r~WVr1apxCJxiaoO>KgEX%@!{f{Rl(_Yl@lb(AV~TxtZrs%x+fwQAKq!j_ z-ceh`T3+Yj$>7;lP(!ouq)h6BtuMbN&SIFU#R@hp;9f|x21aM1sDOmPJNd5pIv#MK z_|2#S#}Q*plsc^nzq*_d{=AwF>N}gDBNS#Fl>)T5Unsq52j84i8xpNThPg3IG0S;F zMpD42kVgyiZN4c-HFk8aA6{}alqC%rG5PRqnC8*Uq~SUik($~Pc!k&Tw(+BQ$6*Fq zZxS__Li#s*JC27d(blH6J<9-NX6YWuJ&7RYZvG!}GbG45-WEPx(uB=D@YfU@DLKQ< z5)T5V2;dl&N#a;) z>*3HsH*n~*Tm<4cF3|+dWEk-5mo&Dx>Cmy;Z~aQ-sCLrO#}_=%st3!f=hbR=5$~;O zW(hv!-p;t)ZC&w#%N2NTH;3-US}w%iurI|!wZLr_6}Kehk+=I$-W0(}sZCr#e3iCT zW~0>eRoG3OG?8;BbpeuS=?mypPwCPk?2yi_D-{iMsnMP{q3>5SKfnILNH0da6}O3A zh`k?GHKNLm5SIVqK6BASYQTOBU0K&mv{a*C^gA_?a#Y5~Oj-!;ewBF*+P?ns;H-dv z1?(*Z7aFS-cEYA%s!u{DnMP%8Lco-x)RMe@a{KG|c1$&KbE38wd z5ZEvL-`7XKfa%P7SV>7$o&)Ggmb3-)7yjsxS#NqG+N%TuA{`xCc|)Xt1d|ugfW|8R9;V@8%p*d(-k+SG4k7zE{~*#h9~g_Hhyc8u+s%x+2t{G zufC1D^NnduenMd;BL=njl#ErU^D(oTAM~&ys7TejoOjks>gpS3n9I;1X`JKv!5_Cy zoL;cooPbDGAzeTIsFP*-6uRe&N{0FaPoo(=Q0$lR( z-@y1GXti@HW7EspU2IiRdSOFe4UT%}T2_vG^MmmNnJ$o_N0L1Wlx5w; z$G%pj8-bDSBn3h9m+3bC-C;BcMT211A*MBk84X&}+n7SC3Npy)`w~{&x>563yV-JM zY0k0JgaOa>qLHW#Jj@mqeW_~xhlFw&HY!SX51#Z_#Y`?3$rEoz#-Nkp52p+}v7Q0Nx<=ihlU*%Lf} z)G``3#A%NAyRoTB1E4<{tArx`?iU!h#^oFNlm()7*|TX?XpD*q>2Dh>n2YiSpJS`( ziI0T1%SKN%vyuVQPvd;lM=6HGgG?UUt3?J}9ig*a%96(9-?vU;d1MJCRxNvY85Yy< z3aWQrcW7zzl8on#p8sX zLmLqb6A4SHavu@l!$P%^*);-I0I&O;%NH>IosIy-c=NY?-2$G51-%F6a|f5h$)Nc< zf^`!qDa!>cjYdtfKYAerj|yYznCU_hs?8&3_I=26q*3-yfX^^f76l8eXVN7fz7_ih z?M8k;HC~BLQ&cX0`W^0D6b`R}keK#Y%6`Zay8X2^P;{laa_&?y1d0Y%ymS?}+jQb| zg+39j4jAoE#RcfOgB`_1yjFeETlOz%(I*>%3GtI?ZED> zXagqpI39u-9UFkxE#0&(7!+PC3K~R#{^HvY(<$CfKZ`WT+{FZZh#-8;^PLLc$3)E(-Tm$Ml&C~Y_a!oIR`JYFc* z04%X9Y8~SHzAy7#-Ry|0YUmdcabie2`{WXsf;s&i4Nx4v0iZ@U4pexOo#H8a zsLC7rZyDbHFs#Pha!k&IH-=FCD_h=)`WtG*IdhD5)=yRbFH`~ zgfhQuPT3}SpnYMZoF3(0e*?H6+%k&MyR)?+D>vFFn~`-DLMFEi?Uyb(>o}fjw+pTH zkrs}9BE7aP*T`Zs)puZOwW#9PuX3}$dh+DuM7N%rihm@4Kp{}OOf-q!zT?GEWu? z5aX+wzwVx_-3O`t8*N2_t8mE`=*8f`!e5a5<1J$eh*ATIeGWz+l)CLWQk8(@G+H+3 zl|QGajb3c2Zww|a$_dsIh`w>)LWc?&b(P3EQiSNgIHbFu=sfTHP@7@JQ4rs`=@T~X4p zB^{VFi~m6j@uIjp?|;CP`OCbFj2LH7S#~0{P^_hH2q&DOP5#I z7>2C`lz)^~S@}aLMMI5gDifJq=Zq-cE8;@C)nie7Hn&HMH2Aj!ofH()Lp6qU&><&Gt^s1XHkzI|}sheH6U?$T; z(*NDcJ_Y0p*VqNZT~O~(L3-}>XgL=v+cq;jQ)CvuT~=+i$O$aNp9|!W#+%BYFB%;Uy>9Up-+^y^M0M>^ayk;B z2kD)bv+cxyA$CPrVX!JMx1+IYIIOqv>;6y$MH_P zTmPS76Cg6BP!sKqmw0sFhnIE*r*)B5(ZMybpox%lV`40~F6tX~UKA=E(9s+nK3!5Q z8c;T6H~*=eHgPOu|1pbRW8}SOw_me`S)^X7$IOe0{*Ypt2CJc2(WHTLq3Tuwx2(J6 z>12*nUiV$GL1cn4W)@||hW9fqaIg@9>}2E^_|m~}^Uz@mEH3Q`s-x!T#y?QwLVF;j zJyp7PP)4@%RP0(>Er*sHLoMTnv>)wU)EHxy6rXq{4S}L{`A@ptfH6sb9a|G%bp9 z@Kb3-SffjoL$rrzp^}P}GUI^$3500h5Hvj6atsD$t*xH7??JBz>7#|A@PYqiIsqjQ z8s?Xv1u^6a%~dOi2`ExBGTwI*Fx!|T>x8&!rD50ymJ{(5j70&aC|CMqTyDtmt`93x z_?tuSBjeZn%WvI;jOtgszBEP30&`gB=dM;#(2O7OV&Vm)wYMoUe`TL&U2SrFkn@%%@J8XXKlwUlfI&KIUE(earo^rqyZ2cbN?c65fa~%oYV0V??f3tsxy_kp+9oiHIAD&Ce7ruXEMdrF(%1B%F zCr1-C-O*GOr6OSlOkj=+kfHWGx*0ESrtLVB&S>03Gu%q^-^|eYvK;&v;>7NV zVUOZ;kfERDX(n&7?R|Ev+-8S=yF(LRWFS2MusvJxp!bBGW0EENm(Xpy8WY$|lWF=Q z>%-oOTAv+Y4E4ykNlb{h8u}(4%dS@db%Y7wDmiEli#YbanLhHmoUZ$ED3D{ncQ__ptwRTq+#C7 ziZCspnwf7v`Wj1`>p_AK18QjQJ97Ad*U&V$J=3?9)6!n;?KnJ{ z%t*Cn+xH1;F&%M`1%09FG{J^?pCfqcpna-Z!d^a1uHxRqz7EC_wp)*1@?)o%fPLx> zNR@0IW4efoLb}QA3|%5}+aWA=>>nT(CyVYf$CU$+@-_2oLK^i!<9^x_{O(OH*=bhX8S zU{&`sGJ5h+$V!ZTJ;&=IX62)-(ZL5vrUUe&W9f1)#`Tl>Ii^Oa?qZafX2^d&L=9WA zphS;|dQjKuu5On2nIfGxABa`=c+-(1>GcoNM(GpHCYL=LE;N;}U%rA18;1SeT-)mNrEN0lO0VC)&AXPtTpn^w&Z9gJ}{{pcj94 zq|4MHA>Z?2Aq>5+Uw#z_kZp|z&)0Nr4R$jt)z(>^M0TorRiVFSc#+SNR!rHE{X~sF zj1C=ap$a&K;ddi;2h2jq<{mG-?D~N|suY2y$DH}Q3$3?X-t?oM;;6cDt6O+Hibt%0 zMv~-_%VkwjOUb9GHAwCVI(-nkJe)+n&@6^{?DwNTmZgb=9tJ83MnFSBRqu zi`Qw0Q!SLr1A$vAT4CC`DZ4!sw!$_`WHs`S;AaD&RJDAGxe_p#W?8gdYzrb-noKoU z-#+KpYFiIu9dp)Yz%H`aAL5LgzOTK#lHWYE*g;k&`YQiqx3v$_IDrw_oi9cf4;f|2 zr=CaXbC;a+G{TI4g>RtGKi4NL01zUx0z>FA;2o~on}!Bjn?1s4-21_ud2`E@?;}d* z4oa9$oC}vA2H8V-iAg2`>vc5PJO#s5IvFte;tx&)QT0B~?)kh&jV0NEMD}DucLWdO zpQ<_>p85+2Y<&vI9bl4xT{|qyXqXluGSa*LnCs^USO}*B}W>vm~w|ZF}P}D}e6?bghWGOE9Qdho%6-A8|pi zPCT(xsj@+s0_cQNOBK@CUEaoP$x+K`oW~VXb~d&D*bugo;f0#bHCyH^@W2|KfBHV& z#k`6~k8zs^p#|j&lfF>BT3=ln52s;qrSL{VLFl_}RS0&tOvzBjAz0Ne$#vbZp=(Pt z(jSVWS4KfDA3Ab4ieIni8K_ANIHRa1mPF*)wcqC=!5NZ& zG#+Al8S;(Z?<6zJue*vKI*$cNjX;`j8@_RNJ#hD4hl~p?FjbL4P#kq-sXXhq<`fG6 z_SOfM3OkMiP4VPw^6-Q8*9(39G>HgLhAsDy{dfw;KVSqPf;&XYZmm|u^4*H=4qko6 z{R!T!P@WJuU>lL=$tYrp*M3omSp+A`DK^*94xIpUntD^Bk3>%GA>L_RHur=8wJnVZ?pLGZHs?>osO>0v6oRePT%2|W+| z!LPr$<4*_t$K#t2;eVE6iTYP$=|JW;6i@lEH}P-)T@G}42BFiLJGCp@ zJqitheCvE?d80qWy|-QKGXhQOPAAxzWtcZWCQVv{1E=`a-(Pf1{oMv5#^E%A_|o|B zP=UHiM=!W{O1cf0V;;8N=)KuV(Mi7C9xA6bk0EMYOV6y2-`j6~DNpKKZ*}x%r&l&VD>OhT&nvwhf%{-19*aY% znTiP2^k|B`vRKhJJD+Pj2?9^H%ozvqWlFe|_*$7)v6Tg~*}3#aN(^m$z?6~TK7Lec zz*J-l>#PV5fC0nUBe^a;R)TFXlbcQFG$dkt0P6I=8hKT;=Sz}ZF@FD;{S9sv^eE9D zz^2zn&lF@aH@K5Q7WK_tBRP8}t_4)CI7pLC|i_Ry{&nZhv}eYtHxp+0_tq+ZPC&Us=#L zzxbst5tvhq81cxJ8#!TUzjSnwo|+w&y|%nRhX_fZ`2J42*YYu}or{G=8EKLcWC385 z5_f|R;s#Q_STl0v#PC)hZiWc#Lpl7Mb|f{1AoqI9rbFlC%G*l8tJ9Q84p%~U-)+{b zkB$RJ95zf~)h>yY+SEvg8*yc!ebG7`%Gnkl3%$E}FLv z%x{1^C^)6hHvvu#LAI=D4C;1_a%^>DvU7A}hQnysZA}Fa52-wUVQ|O-fl{nw2hOO9 z^lUlm|NlbRXXfyomy>$^?$IiG&_N%}FDXXt3hEQfg_h2_wotn_n=-4}wUxEj4`~IK6>ZZ+U0e{!-=Bp~$1M;XE5TU0dG~A``wDZKMyn8`J0pq8dvVu}*ZXbmP*s zP~MH^n!+J2ln*r6CuLaH<{$v*1y#2e&+JD3uM^2$tuk}x`d9hX4<81rrFIoe{z#=U z6kII`i$An3{DMlfq(v=UMt}-12`mMKB^{OMZx{P9k>Gxv(K;*ta^oQqKQGw% zDhX@kJ*N)bR$p44xIZ2t{{A&*v#)-SElnR(44zvoK~_Ko^XxX1=I(rIw8UT3sr@eV zeY&0bh)QojfRLz`bRG1WgG_42>`8xZIUu}nwo1IvZ8MOkGr@OF>;D^z*!*xB9 zM)h*)19bCw=7b)ld`M$Yk(Tl6-Ha}~Uk}0l&cBC`Du#fBI%kkn@j#*h zBk3kQAiXVGzKmT5-U-RvaYxT>!x%3h&7(c`jqULbu|(b?*C?cFZ%=+$he%#gke;MW z1AL_ymCS6qg+#$@y+r`(wg0FAbidQk7k>?6cjpm`^p?6zr}=XNQdp<_{ofK3!mh7u zo;^`qQ^LQO8f6aSvq!bjP~i&B(KL3e`lfeFGuaIHhUh%^Pr z_j!js+-h;~Xhe{k_WhEV2hf zFjkMfJ0pIRU0Yo-A46$zhKilsn>t1Z-UYm(%|uM=Ss0@}i_Dk;c0w1E$+cK2r3a74 z+=L4UP|ST=1C*qE>A*MTx(-U^#12&Y7*5IIJ|WhM)N0kT*!x%@Yjvx8he~y2 zV52#~mgl9hGWb?D0Rd^L8QS9xtMGR{N=;IjWrTPVuAxa6cdrHyX_-DD(}CN9 zp41NeM)xK-XkI3P%5f!rA6tRRiwc!s4R3==)sd*7G;&uik*g0H3}M1<-Vs0A{Ty)6 z+v$5gd{WeDO zuM}%{j(ri31A^x8gE0%X()f%W@Vh`3Bwo;~cb}dj%vZohiKCJev}&atxlAH?#xcYt zK7<){W=4v(6&8#hDyopBQAxU=oY$(HEuXs34;XI<(t5s1BvV2eI1_>5qsM%>H+Q(~ zFvOXw`gEP6*W(l{8F~aQD z7{B`BcZ#F^uhMjl#-~1&IVjY&zlD}q-Asc$1!Mlx{&=VSq}>jOiWHj%^_RCoJpl@9 zdj)E8zW0`^Gz28T@mrQJ+9JDfu~Jl4=HK@0kdhaeqHGKSu`i0w5Uu$E!StVIET$G6ZRD#q%#%>FltB_=lQ`Sz& z2X4eUW^jKCUY=kX>JCK}bH++x7ss}P)ASILig0bqUZyPAS}*7k%~+ojolOPHm5E3EA{< z_tbRyNFH6X6qiv*s=Lum_%+=i+ffFFPpu5zi@_Zz-F=+gz>e-gKQ2u>Yg=|6g7qz| z_;I}RPtfK z7!PcB!++5pT~R;YV0IqrjfDgacE-O5v!|iP$9u(i5G3a}Ls#AVQ1zZj+Y-3ru9Ve6 zN>hDUr;xoe21JRv1}^m>G|R>BE*!YZ29GcZIcKO`riUo%f9NR!3n!Wg*auZe4e8Jz zQ5bS`$W~J+{dvgx4<_76h<@sTbsQ$z6Gv;FILBi_?Bg6{Wq3clb*hd%TjMSpsMjS| zWySr`xZ+i|63waYpOjZ?2zv*wke?SA=t4oQPk_=5EC=4?Uh%ZlEf1zTmiHtKkMdK< zvUN6{4X9p5D;Xi6)qk^y|Ftmc?(3sQ>-@_up?FhuKV}Ppdjw~pkmeK}7$y$`BIw3~ zxM5Q_^Y$6MBqrH%#p@$##C{d^kOMqVG2H1|0NR!a=S5JN)rjhWpR;m)%6Re{%b3ah zp3a~x2p)G{6U|B=c@9Wgmr1jBb#Yy3in9T9VrE*%hd~!^wB0YK&0mAZh}H%mgD{NC z0!3>Ql8==}$aM-fDX6YRu?8Wu{gF}Nvg%A|3UXwWB1_!&s4V%e@P7%`BAr0;BRjo) z$Ua**4)rXB!y!)Fp5T^SA|;^DDhmF8geTea5+qx2 zBeCFpGnPqZq`-P%6_E!U6v5E^pek{2Zo-~Lq#Ot4(^c_;^cO8oKQ>cZS^8C>T_;Mh zPIUxDQ087bxB_JPgp=IjnKbt-%}5%2f(F=1n`Pe(9qSrAzKSH*wimY@Tc;eY7xQFf z;jDS(Q3qe#0AVbO7>ZfHEL+w+qghcN;D8*{I-6wf5qED2Pr-RwxVX&Po~3k33M+XJ zp#LC$L@~S-5MaaqjWP9IJ@{Tnl_$s)=`fq$6Yk!%(M&QKiA*Wl6|0{05iR#@v+ z^LhHwMd)N4m+KK0#c=vPV&m^Y#kP-NXRb zkQ-)o1o(+c`BTxB*UjjcBOl|j_zu24karO|J)Oo`1f0ylI6qEKE(g+hBQGvhDsS*6 zTybIDJOFY=m(l#}S{%&zj6@N{w8Gj;BIoxSarm}XVU{dW7yKiP6~47=P}5gl!N`9c z4RBj6a2UvGgoI(I431%;IbemC>kqUyZdLTE$&<8BiV7Ht)~d8hX*ORvt8jN#%l+8Q z7z2M@v+IfQkn9>YArBB&0@(to9%%=%sB)}#i8_2rdR#LMrsgV+y+$&Kg}EVb0@tWj zrYjbQ!EJv|07PkC5`YBDQ4lfPz+LN|@;Xwx{v0p_Z7U)Ts`5hsG$ihApl&I%*DUor)>#Wd?ZN# zbz+DMt zJ!D^9&v7F>`DPr24FyD(5xGlfFvS|nT0Tk8w2%QAR)-sboP@^hs)E2>Ed)5mcC#CE8t_y z+=XDFy-^Zs;)U%%o9gN%`zybbg^b@S5bEC>D$v(T4$EHLPC3ZM@SS0SX$Ru!*J*%x zs34RT0(~alqpzfp$di+wF29lsz(vC`-EyOFg*|Job!o}k-a=+$Tc2B~lWHx{NdD_Z zZ%*eA^!@s?tUGZ5=nLQW`9N@gXtfQRUJP^7!CZ`GaXMvjI(U@G6_3ge~&p!>Q z=fpjC1i^s(1@F4DqXdyWKrf<6D=bS{q zas&BvGSv8Lmgp|h)_&C9wO#qBg2m0=&8_e12@dklO+2@2CWajczh-R(lf+9%yVi=& z4$lTi*fy9@R@ODYeTw5O31jFBkj8PX-X3*T?huBwNG+FG122QtWnA1|Ub?J)F!-7h zO4x8ng>dEAzoO zNB+CH)zdB>Llb~h)VJ!?{gCk9N;yl9DKa&LK3))~~i4Q|zVW}=)cZZ2}?7opw8$6O1yyM+r&>!Ns z0-huSNaDh;nLlO0rd7x_nRaB5S)5 zeKg&7J(=ma2fVzSE51~Q+X);2?+YUMx&IZPwqs)XAGVG;AWzMeDjITUKLxayZBm@N zyt`??F!MPITCV%s<=)+CQpqL=W;@=FYQTuOC{nd)!M&QTQ7tR?W57d~=HZNb zvhR90z{N3sUH>!_ZZ{pPoi+QlHP;U@Nzq(3T0R0cr5%BN3sl3sp#h|F($WfKF>7v$ zKB|tkd;#a^bBwwo>I1ng&;VJ#ques^0Dnd=42OW&$)T%eZH=*fIdfbM*k%XTe~o=~imlo#8<#%hZU82K z=Q?FzW?;XhEsjFgdhcJYo)+Nuc(LY=8i)0hVYcU&?dx~m(yp(|5=X(bdaR7PYz_E& z3v5_w(Dt-E*L*1ZQ7jo&C7RD6qlnCL>>4CO#~qZso>qE3NhmSkUcTmmzTF1(6Q;vv z*%R$#Gk2px<#58^7ko?RA*`xl1shx)7q6)~q>mt%pgUrI*Tquc%CC)#`Nb%_qK0WM z4J?IrRIO}a5(H-BQduy>Cv5(C7dFmma8AEFFHlY>Fx^$WvqJyomg)* z$a&^igm|5KRRk7!rVT?y$0WT4XJs4JGkn`^J*OO}%3~ouE2xN&d~P9f%dMDq;aYGr z4f@_mmvR**;qrDDw@nt|%;X74IP&NBP^~X0JbHPJD@l=@ti4A%RIkry`8Ve^nN0=LRkRjR&(ux`AmUciHbMJ|-Qg@3& zZ9aX?b8Xh1HKUMF3{ZvTCRX?$(P5epk0!u9l)>4!7{mgYtFy)QMx0ur6h?*2Yv9N2 ztUcszCNlwGLFLm1hSG_3m_%q%R8sEFs(G#6@NrpRn;<(#fsk6D3>%ACaM#sHM`4$V zIhF2Syci5t>dqrl@x1%TNiShYOB|Pb&t`!yCrZRoBzpWCwt4)w#PwFYLgo6v_lXS< zp1{MVZ(TIZvH$<#kkJ#`H&_v+X0KU<0`|-rRD7geI$M8^lglFnIE`VZhm(fGWiT0(TEEPUTLMBL24Pj(P4$dMWNwH0UC-BF>`Er5B)T?5n%0yKaBWHJInZ zVt-FZjodh27{ZHY{_q8{_O)ikE#X-Ng=FAp8pXs`wL{1Orh0gtn0Iqtbgm_5j3M%H z;8g#YsMw4xmw+g3UUJ56yqQi*IhDBH3q!vsHu_Bo_=9qsVgA>|kc)288>6@fED7%S z1+Lw=OG*=PhYFPZ+^d=$b#x`-H1?}1WJ2kHW!y3nAA$F`FsWZe!SUfqNHpb%G-WSL zR|q0{%#S+-_f$51JAMZQjoo{ePfeHeEX!)oEj;y{Vg~|7L&~HExM6z!Cau4O)#>DJ zg?FXb+S+x)@@i6PDMmoS+4~1vO7_RkR4ir+HgGSpxM;ddojFnMuYhN;BrT-p9Daz3 z$o+)rP!kC<-UzJhk;r&~%7WN3c*n1z#{jpu-t$YoF~90#(-`6r)i< z6%MjLo4)VvQ6bVX2rWv9++HgA6D|6wvnGgE&Din8v$~>Cfh7ByeMKX7qoY6wa_Jojrvi^ zI0}F~LXgKB^i)!*eeDb?E;V?iZGq=dy@n^sRX>Sn7N4G5!cNy;8R8)%ZqtpV?!u5Q z%`52WKu$lRK}N!&pcY(z(Z{3rlHrEuTgG@Q?WB)coGY~=eFTkVoPtB%oz6M4W7qMS z+~?~f=TqJKgWOKIav9J)8h~c%v_wf_E)#32y9H6i)b`71xjg~_Ls4yL0LWn~D ztQTri$?w0P&nD0ML%9J{w<8-=%@yr7qK7K?9VoRc=J<{~3k1cEx&n@xGKcO9UwF0FY*P244lR!={X4D+8ZNauYr&Vn>3 z)|@0igl6R8+8fZi2VYTUBtn9|9wqubQ5RYM8^zWa_)v(dY1jRS$>=*>JJ0=FU=~`1 z{rAox11NdhjWX;2{w!^$X!w{)u>ks8vLi@5a%!OJ2)IymBBz0Q*e<%+&T=gfwjdOJ zOTp0FaMz?P_e=2kV8MI@5Uv|Ci#YP5(6vk9vdgxv6nv-uBP|I2 zvcCfXnTTkP_Q_Uy)YH}u9-%kuw}id|g^|N4B6;UnzSacJYr+3y=JZ`}pf4x9O()_N zAG*|HwGtzC{dnGN1K%+oB$~%6Ho8}tST;cr*{=raRtS>r441Hx2tc(H*iC71-Pp%e z<&uO5=Y2oM38vFRCESWCoLwrM8<}~mwL(hRq__&r)z#um70?js+Q+{(Bsn@^c9l1g zdMCNRB|xm65Gz%%y0?>$KNb^@sdqWSovl%XZLhq^Nb!krqGUEpcNO`0Qd;h=aiRMm zjYa(S`@ySG#^CFJ&?!7{Ew%I(SDDTW3qHGPN6Ov3MPT#vnm>}lDi(&*@^Sx+AnTO&Wn|p_J_cc{&0iD2C_#MIPLGTop~YyAv0aF zxN`LDA5rT&F{{%5e;A#8wCUAiY-yH)!lp~JKnd3kJ(FodvQcoSHTSCX>{f9=*BtC- zsPnOE@DdfKSpxBKtapGEAYIVR_*qsSvG^h8BgZgx0b%MdP zBsjJFR+P?o%?^!_!$wadNrGT=>ZbQe3E^vf)l_iiduyV28y15t2o!G72Wr)eO%XIN zz8dJr$zU2xaB^H7sp3058^OZE#LvO6uDr#e;Ixo9iRG48wj8~;^*De&g*bnBuEXAQ zqDNWJZGyz~PY5i|=qLxI^aA0PY^$8J5yzO(uCVv&>_a*7`IBraWdtWsN!Mk!Fwdh# zY2qXCEk?ou6~312)ywb!tI^p&37}jw@dl=01yXVv3JpKd$EYQRK_Pp0$~mj^x=Bpb-6@**x2U z#!fR?OPY7lmY7FtNyEon2AOQLMeYL<0|b(M{2m|T?}3x)q&y%X4Dni7(6@I`#xc}H z@I;`7c~{28`re84uD~?<^k{&E6_6ndm~C0{6CyD4oP5i`qqb-3uIX5n?%v}Twu8p1 zlpk`oxN)7sxSbq&mBRC3N0SbV(Vpc~3}z0B6pv{1m%%kd!bR>AW!nDID!!nR*cKHm zb9D5SwftI8Z6Y0g_?MC=B2wsYU8z<7?u&qe5TF9|9MHuhE{<{41gWl$tx_>*wtR?Z z{NAH5KhN8tU0Tes?radNDz}a2?z$FlAj-EQU2_ZK3?IbZlPicQUW0FK<)HB3POrW+ zpG-{|4c%K}`D-8=tr9Bd8JykamtyJE(mxH8x`y6p1?Tp+rQ*&ThYFL#sja)d?t4FgZam>rV4$mLNE!`r zk<+Q5r5{I#Z&=4w5v5EgDZ@D)oCtUiJ)lyr(g+I1zIaSX{bRoomF%}1ERgO3nB_9M zw}CY)bm(OB#oQ$VGkm5Ea6LEv0o6-WoX)#%I8i_f~cSM zBY^cyU<8Xc_rk*K3fhT=^yI(0O?j5;uMXODG8c0LSHGNs)bx|ZFFL3etbjzcsU@P3 zzr!gPu)8s;C3yEEQ7*7Mb5C*K2f^+~HL;dbzoX|(bOKqzthi4Xkx5o{nU>bfNTSL0k)sRJd!|I?p(Y6|hO}?*2=&*)bb>A7hM&pffD_Ef4;~H*rCEU%xf*)Q zA56$57(qTo8M85pL4LP@Bi_0HQ}kiA{nM{`)~4h;4{E;(Lcbdik*kl@Q22+CIs$wO zLySeg?EICV0~kB7A>L%}TK#TrlS|vmPy2F!wIv7CtD~iCNrQR8whR{$a02JqW%PJW zhhXMyE1NdKKe^I~Qk$nm`r=ZZ^;f3N$XR<1++C{lc#?w3gD%+ph#V!_o}wN;f7~FM z&Q$Rgb$${1P9Ioadg>!bBHUQRVs{(MBcgTmG-m`@-#qhJl+pPtn{Xg=Evh8RC}f}2 ze30RFOXZ5)x0})$k7z;c$^#Uaw03faeUD;=S9w2aMPS2?t}QtA!Aj^9P|wUQ>umVO z;a1**D{4Z zo(0bb+RiCnkVkGD3HB#@gVr$Pj*64t(ay6>mv~c_BlJZAa?cA=OZu8}3*vUY`KF~u z>tgS{c4fg!kzIGNUWK|-Cy`^JU%&hlC?oZzPdifOMYzD1&XIKbd-r2ebOO`FgFFau z4&h<4Tge;3>nu7`ZLT<^o(deep8e-t0#wOlfwu_#5WCAMnsKq~G1_DS9rTr4Jn zAl9kGzAm2gdinYyVqmST?z|e82laiKJZWCwnu~Hb=7)(*5OgXjg7|m9acP)mI|dA@7+njGOCAZ3!Q8JN(wd z@NnmJc6E&mKl5TO!=AwyBEfab*X{TW8!WH1bv@Q^{)}x6|fB4 zORd0lI*^MGtphVcNTp*}6FQfW;fel4eG(t&`c>BEM77UqVW#pbE=%lz%qYiCI5eMW z;WQ4O_@ET|NzHEQQAOw!rd!l#g=&tFDLu~>+gTOTWA%G!cp#}a8OzH70mgFC;&LeH zbkxS=2ny~z!6ic=uiyyRjaRm7e{rCg?I(=1c2dy5pB?}C;-@csu0KYm z4gz$rpvj;gV?TNnsF^@Sq|46%@k*`P{fa%IE$Arj5N9x*;2p;n;8XODbl%`{*H29w zlOU1GCNg6%iL2P!C>%+SS5ZuH?mJ2b?5Pcw(w_#+t)q0piCX=*qOfvaA(Ei2|gs4W3U3~viKnL_lE}Aa;|yaZiWwmo~=Kx(KEwo zeptshM?r+vJh&$wIT-N!`~x0o#|%x!U%a3_nq=bPJHg7nHfn{u8L$Aa)j%j#M_0@hUSc(Zrl7t>Z+b#hS$i;Gj zg!8x$&0@&(fiCpgA&%`OScJfQycx^V||ZV&4;PRr~b)2_N*DWUrF zx7QYrs!|UCCUCXDhB5lww}QSk7%9Z{&ZPwv(VNwZdKe_kEX$&bDxBGkwXFBivA^lQ zKSPP3llMEbPyON7vXrAUZw|Hkvd`_lv$@h*^8)$nI>!u+zdD*tt>}HDH#b|niK}P> ziS)3HV#12yHkHAXN9+S=QkOf}HdGh~qsKUCUg+N6ef+UUGvG+Dq=i=|G1xUkU_xHnqo8%|6XgV+fqk`3M_(= zK(jXmZ61rwc@H+D;Ak<6qw7WFSW;*Cy}2+CHI$sL_+0OHFIv3{Q&v6 zN*BkSr&cmT@+^-EeM}pi48z|Bw2@Q@QEG8iotKe?l6gccozVA>fXuI~`%G_cByD0MBc$ETcu4tH4J<|I@f(5Rt`e+v`!r&RR+Bn!8o*OH{ zLeQ?vm<03g0mKPl$MsFHISJfS<$vY*?>Dd0L@#kHRrrAVZr!@*HgWMV> z59)^<0^OF4ae{T?D$Dd*Og;<>I$S~MxQALv(P0LqgNooZawt3Rqq(TtKp84>_6KV< z_i`RP0}ls@ZtDtHz9R6lDHT{((X)7K@hhh)Pa;gY^@?&1r1dsPcU=ZZFjl?LW(fX) zfVn*L1|<43wWSgX)IRk6)e(FxgBs**Yg4(=cjK>neAA%JV-f`DodpAN*;h?iHN~DW zncffv{>4tlW16;rV@=0uV!t0ku$Yn=k}mzYq(5ZJv&+16OhLSdX0HWS?oX@d{zfTb zW#Vs+K}~Gc!DI~2U@9-HwCw8iS;w4672WcE0M|K0M}I;(9AH*w_(`x%HjDF?FeB*+ zxdu`2D-cyTEf6}|OD;Ly74*$C;=oof;mNhq8yyp)6QuCrTvNHnnmj`Rr$r1ofSwxN zsr(DihT6%E^|X7@WmT~0;kKkwn+vYP>|G#34V%t*?@zCGWxB*E-qK`(rHKAT7NcgA z;iwE4!`i08SP1uo%-nlJ3t@7g>Xh7d>OTCi*&4ORN=OL4L)?Y$S<~=`@HSw8QlZj+ zfooCN>$Ba(87cO^Nb~SwXyI<-nX_N{sVQ8@4a;%9SvsQ{918eA`e8f7>g$2K8HFQ@ z=v3oMm=#f?x{hbb=H6c(zrZLT#LOcYV7ps|!FpdA_wF)+3EDIC!;Z)vXDb~$edd8A zrJ89O@O%cNf64p`H9o>ARPU5CI|=@Mm}df6G(3L>{yTziu2tOwQFkzP{&=CyakLhrKEw=4~uBFzdi;3zI%7)7N2XHSSB4KOvwm#kE5VLnO^Bw$kS zTouo4?J~$B+|jJJq7QCj>7MeU>>XE=}HA{f<4@!|dYLYfxc#^kdF-m9j=nbL!ZON&(= zJQ@-iNxo|LNkrCT;ibby)|`4NJSv|yXh`lpfypFVaSy}i{EyM@bIZv;GPecd0D!%5n0Hb<<`0sGR94DQ9oJ6h$ zSUp^FXupMHm@<%}(@&9>7Ir&#Fx5}fnCe84;k2t25_)(=-zRPhr? zID>l0-tkvaTAwNO3n_tp%26<$v)eIy2~kTi!C%iDuMm-fBvMgnNc4Q!))Z*gVj*$a zghp@qf}xm%J>s6*O+_Fk*>`0rZ<7IYGL~bp{2+O}g~HZnR>jS0cDNN{bdu@CN&Or- z0jcI4HV2ndoMyHRa(Y2B&x#uZ_>Dr zTl!!c_^;~)rd&qxT}gNjTpvz%BWP}5k2pZsN@f7JrH~&=h8xjgMA5+fDQId(9n8UY zk?xp&t2k?m>Uzu|O|W`!VS$SyQ-IMmvYGc1yt#-17olKTS&w_esQ=G~qAJun5@7YW z#YPd9y&BERkGFzR*v4U^T{0;Vv}v82a#6Gd{HqvFO`|gb;|Z*VMT7RGtVW_Zo83JU z%^(neP3|GoMrj(?Ui6(VGEXXcbgp;~#dla}MDro-mkvEdVjX0>H!@vfIjFb!B5%B% z8>{}l@6x`Doh7!Pgp za^`zV!1PzcrMB5jU0@InC|Dak;t!0JbKvyIwEd1ESn^N=e{mV2wepc-9~p_=u4xhD zJ`!c$gB@~)wc!J}|7D_Xxg)Ltoxe9{tLIu*FxGc}|Fs!#AG%)%0+ysbn-YnJC4a3P z@E&8c+&!Ov!OQ5X}X)1xygBh6gIe+|P`5bZ}ymhL(5FM9PdT=r8h))u}#kd*7niAorQv!}e41_V2dW zVK{{%&^~h+u)H>~w6{$VqBL^0TwmULgUtgk?_aMS+#XJ%B8}`tFPFozs`Ap2fUE8u z>Qql2TwUg({ToSh>_ zBCC2R5g+hFx!QNyKoicB5!y8^P2FQNkqO%a-ai8W0XeFClB)K$vcro^E8uj?n}Yc5 z9^}L4OCOrSX=Y&SE>U$94QgfEs{BJT4`((m8xMGGo^$O5Zd{&x!Kb8a0j~IzWf;c~ zNxvqnRq1v|`H-Wad1M{O4&(^e2V*^KzP~)1EWK;s=SX<$N@Hm=ur_@QWB^U=FWnab z4TrR@ngijDipC;u$h*erJS*%HizCRa3k`p$`mmw;9r>m z?2+`Kk-+_QXy=U6tQ8ENyF~YyB*#iaE9y5cnu|xLQVC;;SASO$Q%+Uqr6#H(A+w?L z`7y9MCzUC0wO-?VwB`3@@LEJ(Fj4mEpgz20xdc3)867vkF`!Mit*MD+wUI|@A#vLI z^6x#%RPX|<1`o@0VZ-33w&2Pw=ziQ+W*by$E*82hux+>j;yC9{>dk=qRcs*M2cYH~ z)r~wos>b0PnG+lFV7j6k^-Sl)Zjc-9)-R38C8Q`pu&ysFeG}POPLTJ97#R8m=`Q5G zqwJvc)p-5H1@O4_pg+3RL$f@MVo%^Tgaq*XjmgdL7x`6M0m?(sEKh7+&_^>Xao1)A4XmV4t% zZ(*kuI3}^baRFVw{6hVe6-0m)%tW)^H-Gcz`L?TbC;Xs&M7{VK0 zT7)&si~GxZ%3Vf&=`pqN)LT!Ak3vW%okMwAYtc3sy^KU-;{)|dp zr^GjJ^ChQMM-e!F;F63rfE~OpLW|*EWFv60C8V>wEACfH#fr)?vp{*v>3%Z}$sA#*ugM?5j+Q|vleF%dZngh@H>!>&}peti^#BQ^y*j`@i&H|CR-3^(dYocG12~Uaw0(+KG4Qi%3oA@ymyAp}p=taqD zzjdTo2k2MBS8Lk=5%7FVl;=c&jzop=RkM=BIDIil*f9_)H()u3;~m}W?5@CdT9_6{~FG#`e z5JIKYYoorse!b`D@78Wwv7H%$wH|(07mA*Y-rw!1D`WwGOPH>*{}c$6n4lhc`0$N_ zk*j!h6iz1q|3+D-^{ds;YQDx#K5ioVI{B0xei#BsSs|hUR$m?DC97;9JP$Z4~d4oTw1p~Uo0cP;ohT>0RA_|k? zsEITN;+J_#xm!ujEf&>5#lhp_8FpS26pn}(4lT!Z;cd)@)!=`qG{oi!uI^w<6sZ}( zL>T40xK!I*Lt27B3ym*x#o;TqCLT$_On=m4c=6bAJyi8hKzp7{TmhhPl<6A`%01bRFG1;{=ZjUxs0;YaP_d-r%`R8aWgpF;cP3b1Wq+aqn45f;vP<0z z+?<~Hr4n`KjFkD+c8HmSPvlfxGzx&KEJa75-pZX(%j>seP7|;<4(xTS(_dU38-#`? zg!cY7{xuIZGE20TGkd;jG!iq7Cl&_h?ckSoG3U-bpOSRr-fxrfqD`57Q9q>4emcy+ zl0y;f+cm1uw%n!AOoEk)4WW#2B;edvsi=can(HXWn`PktqeY&Z$b$HF?k zm*C9m23t?P*JbXk!rZwvzN$Q%Qg3iJNWv8e=OPp#tk9$!ed=m%PVH%9<$o3Va?*u* zl?-Bd!iu8)*HL$EOs_~dK94nYP@qaC-PBX6@c&xf{CblaD1&a(m)pLuSD zM?c6L#OTHYYW3){N-<+xCW-Q_Rq zl~TGYVcfjF0#_Lw+xeDKqc<-X5bt|yJ6Oa`cL;W9PnJkW$cUGFzr9r zLAVW*VP7)W?EiTfKz+LYiBdeW*@_}vMZA7}>ITV+6c zZ?9lX6v&1AzoNjA` z4EW@nn)R+n^4)BHdX}ln46peXWvVIJ6+PF*Ai)B`E4|{FeMI{O-3x!H_Zy%a`A+q; z>Jfao_-ab^9=&ha@hkdf@Pj+xfxj{oy(6gL`uc%2<*B&PoNq@w&nv=NmWH5$c$dHg z`c#yoC~6wDt-keKY}|>G{j*RqCL#VQV0EJ0pVzIkkS~o62swA}N8iPy9Rl^Z4R%qy ztGMVzo^=3l=g&0oE!d0!r615?>@(S9c;P^X2eTEl)YLfCnlQ%9gYl|3ilM;M6=$qT z`fzBp#OUcMkQy^;2bdbIx4{E`h+*|))OGAkGKL2+NTRejwdkh;-3gaS@=Q~iPYA!} zsHRF~SEsV`rFo7_mMe$_fvHqC%k^`&rC{0 z$^lM9I!tdQsZZ+mba3GKQ?7|%H+xc?X|(Vg)AN+v?gOkAF5}hZZ8&a&%;Y+r<@|)1 zdvvJz+v9-LT9vEER!&t=o)I{6&Gv)FkRnHb(JCjB{w-UL91ek-hY`(f!75EYhbi6X zrdfeyw3(1wIk1rhSk!@f^gGKFs6sjESoC--8RQk0Y=kTJSI?BUNz;mqm|UaI@NR8o zXKyEY4-4TA`B_JFxTR9R8O!iV``ljO1=`8J%MzqmV9)q(D?@Zn$*>O;?ak3PWZeB+#>B7@DT!=UqxSdv<`5E`Fs9-%tp=> z%`cdG#7J?n$-KT@bR7Nk#zNo_z4Ywd0n@n(G|c{vsNX2TwciAw&I+NfjByN3M*C%2 zeCGYniQbFk6XoZf-(wzVnMyp6*0lwFOuJjsCp-)uO;)>MeSpt5;5a1oxZvbk{cujZ z-J5_JGnh}p4NdEtMuI*k?V%!WH{&pnjFi_CqZ7w8lZNkko2o+%IUOo5-K5DbdT1rT z3lBHILCVXG@e=HfQ_~X0!>y zVK^}?N@;Dq%~+6c27WlrysW+}Vf=fj#zip%KN`nfAvhWxdGyF9h^OEXzv}MW`oF|> z2GEL;Y`UkP&FboMJNLPWUMqZTIfKmFJ)SBrJHMc~>gwt>ZWs|n$A-!ZC~d+=7A)C| zBCPFOIQ+kB_(2qHhH0jXYfv9}F`f1pWmOf)+qh222d-!~wmC00>|{pZC9`gSHtv`g zrzQi$-)U+ZWiM7sxSO$@3(Tig*ZO28M{)vOZcIP%0^0fCxeC*`7UGo8tS5kKf9k`t z2_uYrS>88jc@!Q-7o>&9;gfqwNRLS%ta#zK64g{jd!!(&%2fs* z-1Kco(WO(|#bEijBGr4%qE9QqBKLz$)NcRQ59-Rmjl0lG>|zW%aIk3F|9^vBYX8GQFq%daF~wntl_cP+?Z2M6u$SocINXFZ z&_zhOfLX$CoTyu5`!*_|0$ExG{rSKc<68{8RJa4%<+F1jv6*B2X#5Cs`{7P%lQ{D% zG$DOMsq!!C3|0^ksIHRgJ zslXNYkjbZGswOW@*xR6eeOQXE0Mte^DJnlCW=<@s#nc#mJt7;%_W|7g zP3ziq*s^G!O_X@vsh#%=II2NBF$SI@V>Ybra# zy4)%AQ|K*ywa{!&YuLNFK=Q9_%0x99P1LDHM#t1Qk+Be4R2_hapQsSIT$&22THv08 z>^|TQ2-q;okJfC#TXPMCC{QY>v5uH!XAUNb-;iMONBYI12H*Ili@k`FLJ|-y#PE=Y zP`5S8baPM+rzWDJPlW{$4@u*^rrF8i#~DrpI-D6{PD7EVy@9m#j8h$Ff^#>&umM?P zC6hTEfs=ra8CVGfxzUC8S{^S|fND_0qYo9paSmn4#wHLQ?nnvBM7JCp!scMHho@36=>9U5flvN^z#~BAJMcrF7@(LPE{)B7u(r zZbeYUzvZlfkSj`~j6x;6)XOur85Dx@?ZL#T_G;HhkX&D5esK!Ci}on+(39W(`9ask z^T%8S$#G}O?Xh}qXj|KmI+!S#<8$H!511S{dyD;BIO>fcWmFOzVRPaJ#%gVV zqPF5Tznj?!j@)e=#c46vY74HF*+2xHL_^UFg^8t^K}$!|aNX@#((8-Zv6ki}-p)c@;#fMq`*S zQ@YgIod94CzbCV&q=e-g&b5f*)`n*T0b+`hVXwN10M_u;Q2e?I1|@n7W6|{by0QN7 zwzaP?9Fa6O;c}AHk^f*YY8j>Gi_pbN(s*T^NUb}8Z`fZdL|r$Y%a?YU#-6Y*{@}m)9@r`SDTP83HOaiYJW0dSM3daiqeUQ1O38gI zyWK^#l=^ZM5-e|)dWP0lymPVZ;kk)?5+=4~EEDb!z*pgW@IUb~LzWZ+w67zH^|KI3 z(WD;pal9oj_wBAGlVm^NuSBbvTT^H{2U#=!aRFX%PfT80-(j>rB!JsqF&-AP@cZVd zgDtxoF>{o{(o2*Jgq0kd=gCq}%KJEz=)5!FwttKinVvdErSpX&rwsj1`==ApK#O}2 z`Fo82cxNk_Mvg4SXoO&*_2N?e6C|{FT!0!T5C;Vz&_s%|5%Q%RWPQS)TiosOXY)vy zZ%9eZW53GqK(Dfo^-Hi+dk7rQHRaT}8tp56auYetPI9`9im*dcO7hf5KURc)gvUU@ zsI1Bw+)3aU-q4G3jd6M_vlM_}?9T8|`@0G31h(%CKGhhCV(-dhcOS*8IJ3J?h!K$) zMO-T6{v?ELp@lI%;L74=e2* zf%lX@(FlA09$OSx7UrC~S`R^YsSA`pGN-o51||i_(bBeo;d!X&zz*WRuJzm3r8;=1 z^1C(G`R%J@VEBP9HD2Pvi_8pQ@7FS}G!4-=pfgS41U5Zo?@%FV`dvLx8O&m~t|RPm z@Ng41=PL1|?njGe7O*(ot}XzCRPy(eD$?tb@TU6!r5G>V-b=)?ZugF)&L-ynkU`{@ z$eHXH>p0s_q`KdZYCG?CUsdL;y$ghDbmO+y3(NcwW67az?>P^AMCrYH$B z1B(7!ef_#;T&oTM=c+4}VmIf}!A%v#82(gRuip9!bfiITVzDA88*yu<9ut#S;f7ePgZh%5`H-n;laIm$&MF+{#vjxVBwX$9oCWFR$adB#s{dC#`oAL2Xu3CX z_P0)A{ICcB@}PM4!G!aU(SadeCMyg4a%Xx=f?wJKzF>EX&-K~Z5-IvCEt;^ZA;0Z^ zok;`)$*e&$7%5^cuSyRjbM1fXqFJAWz}{6gT2LW27Du*pwYH#8e+d`5!SNW@yf;8P5;=#)ELjEaz z$N)Kp;6Av>nd<#tq89#UHKRrLlx-e#R~(_%@qg{8^NTpB9y&yDhs#mjUq225N1644 z!ynT^KK?tS!~>}IL^AB6-UP(DKe&%D7yA1l{$jWvFTPP>6#@=O3{;8rUR5v>LxKs~ z(Kv#*RxAF`H9}KdHVun32uYbRa;{_3D7X&$~=_)qOWZaa(`% z$l*EBK7zuj|LlTL#!xp4B$x!s-r)6elEIqS@&zSf-Jr@*s99+;Sst&c=Yal;COe2D zof{Bl-XhZvyRO||fQQZzkk4I1#XJx&st z`ekqeCKJPjD2MxPbP}XtsbT4Ix;v;d#y2w#c{@~hg)GE}G_XAQV(CVVxs0Wry~Mo+ z>DbSzGYL?SzK4}^#uzEL!8ht=LVi$QyD@U^`XhFpV`Hs?t-nXObAO|v|0|jZ(Gjw6 zwg8J4#@E?a)?i1H*c(X;g&Xmun7A__np4NQRmI2qr*!~w6Xe_5%OO@|3ccIM*kD`c z7zxF3$Tt$#xQCBp7$%2T z?k^-7=atEn*k@PgNgbn~Qxxx+PKRMKW8c7a$4b0s9~GyEBpqyrVV1rjW=z8zLgF5i zS|5^Xrrp_d)RE9BUQu#E6sLu_(u6V8%B~PN0qu?e;d{rF`wKWEV2>=@Mr@q3ky4kY z50!?mc)bcMStaBlxmN2gPb-)=?S$&)PTv_b&C<~Egg3(`dg_A!VqCTSld<^y-)Tv; zFG%<4OepJ$N~nE0ZX-R$dHd4l^CEKtcYNNl1C1k!Ce%EvvwfC~xlN&n$RTRa&KIt#ur-t1Zts9S?ZzAZ9ji{w1mtGT+jM8$(Z*p@-QkKO}>P&Wdkh&O6S zu%nNs?z#xS(Z_TQvX8szbxI&W*=VOUTBlQimR}+yWQL?=JU(rLjepyd)*_8qF0wZ# zM2Tu67Ta`mH5{cF!vf)*HVPCndZP|ctgbR7Nx@k%z>dQtKgT1SILb#equLEO1X~w7 z-o%x0z&X=!e8IAUSPx{}R%$JWCePKtCiRq=FB2QArwHS?B895WDOHH0=R|B!9X)d~ zYs*w;`pl^DnHPx;2zCgbqM?eE0l%Th^D@4sSYmkH+yY?T@b%M5uRX=YEgXMG1F+Z2 zN#FqLfAj4$;cTo+th5;1V5e&6&Vo}9#2tdaHrCr;kYvTiY_Jip*s8n$hY^!B5#-QT zU^>Qw1)dpnr%`9Hs>s4ebRPc{Tn0egLv4L`oDQqNv-=lxew$|uy8$%&{I31ej&RJ- zfpR&UbNnQKUR)-NqWdKum<1-d3<*oe%LJWnie%k*|0Xle24^HIq~XDs##d6Ym{^*Q zpjnoS>9gEem^tOQINyUGS<_daGxPTE;cf$Vx+4cBpPmrYIR%9iuda+sK{&s+#pm9VF}atG zOS`JH1+ehhT1FBd>_Q9PB8T!z8Zrt#$3s78*k{y&(@KT4sLCsT(qAh4XvZph#glDq zQf0BUQgP^X{i;)vVW>J&$-n~o+I$zN(I;?VerU~g(TC%cqVX@Jd)zeER5Cttfhx`1 zW3y`Ucbw5M7MDqiCHo0>k7~E~$|vMpP9_9=F6-x0<+fA2c|(887v70IZDYz%-cpLo z3`aon;%l)3K=s2{P&Y7CUCleEb$%a7Zh8E>R>~h(dEb$v+8jSKqeeooJgO0Xn>>lcT_q zo-FW1w2xFwuahX!2Vt1Y8_VvL+w<06m+XLTW?8gPL4~#Sy95xFHMw?oRF2RcXX?Mk zC4<$t_*kli4^YWm$TEFT1#pa$%4%J($-3qK>%Ja0Dx$wB; zLa|K!enHzC;e?MC{4_tUJOZX*nd=W{xqXxvb$`@&0mN+y3Ar|xMdRrtcW1q5FK{nN zhQ$oHE5GIZNcfk^+;NrR5KA4sWUrK{XVRXdtxR|&sysYKOJ?q-!0yd;(H7f9G|sfH zHCOU1qgB<@9#}fq+Dx#QEOz0h- zd)X%yZ?Xu0#{u8uv~?+7{q#T*pzCME7?*uwX9lkiA?kqb4b?tD-BOYA^|sj?&Ca9G z1z1`H4}6E;`@CYj0%I8-uA(xM>{ww3C_}+^M6_@z$HJ6pAlxm3(g7a8b@|IL+%{{& z+IXz#lx1JF(XBn7Id#ex4K8?B!@cmT5{Znf&-(U}A{bz)+)KNcFyP<6{5a(ey&mF4 zQmID7crsZqIc!AQ!djP{~gaIj><-uP3jYf8 zAu?IbL*)d+W-4^Jmcn!F<^#qP{4b;IhioQ}o_1K?_ZY~o=V`@5gAM}g(LsnTxUrC` zl(7e@PDO-`qo7^7>xd7s-k*J+WW|Nlr5fAkm44Kkxq!Korq zg6RTidF$HHly}n5ykg~k|87)njZLB-dCEAes7EJBPB7ZCGa{PjK%)4o?_j}BE#Z_2 zS;S3RpqUHWq)p0~H&R-`YBVSW8iDb~)`t30ljiDL-g#zX};A)gZ zYp50QS}PTGP7C{(bgJo;49~H;1SR9Z=PKEyGkRa;qTXw&2w>iOZvC@^R8BVv5@9d< z>7yHYU3*m1Y=2maE!vkl7M9zkyj3GmgS)Qsn?7Z)-KbyDcjf3j;aC9qTnLW&DaYAJ^3%fhKi|rjd87^UwRD70 z>Y{Ywainbw;VD_6M?kxHLztV>VaD~&G#oorLgHpf%NHVpi9z$U4QRC5naTNi>@j=M zwoO>7O;W~~B1vW$S?LUBUSz)DwM22ZxdTY(5KKY9QiT!gZ^TG1zshfKcT_||NW0;^Sld_+X|4A+DZJkDj2<@r~JmEgdtnX zXgx&(=*}t`!X35r@EtS@iIGp}ORcqKQ9bJRhl9+ak*qvTP1^bV5nKq>V z5gK4@WdS7efD7>z3%yKOnQvrt?1~vpE%l*=$wr^112j2_+!U>PFPKVgE$MD6fW~K$ zrZpQn9XXd@jL}nSz2a1Xcb$(;XS?0G^eJie0H;wqjJVI0mY6P;JA?+!TIpE^~K}e+w-?!T& zDxqfP1lCXih&I*hNMgjP!^74^k}bug1C>68=w=z#(a{9Ta~8vQSXIE(ZoLzy%0Qm;+@W-IJL%j7r?N}K*(ACID$ zHgujj_L;0l5_}|X(O3okPg}?A&e;^Qh7U1BPbq9tbYuFCP2Mv5r2Al`oUZLAFYbW= zZg^=CovOCK9-b*(0$nU-zV)SgWJILulU9hs>yTr9c1i7eU&QFn(zRGJ_yBNrL=W>G zQ<1s*J)hn=0hM=bl>GG{pyv^an>HxH^nLfrHHL&G)kU{$cqb70L;%9Bzz+!${E1Ey zcW-tz!y@0m<1DDEmf}f`$56#q7%_k}V{*oW8dZ4vg|hR*mliYZJX{>O{OB-HL)S! zznO{M>m7Z3resYUfZ{Y2`t`5l{nRTqWm<^Ge~*t!Z#P@_U94yFo|)@MR@p?qVJzl> z_1`+#jC z{S>SwZP1O(INVuJrCi&-L~+PoT=r!82@w-DZcXIooLVCSUf|Z2#f&Q5Q5i#N^RrP`dBaese3D2qky&jH>lT|{a|7o$B#PilSw(Gc%*QX2+F zu`9UM?->}U_lzoZria1-7>@PsYsrFu^;x{1D&jx(=zpumD5AQi^hbl-t4BvZUWm$vG8Q{S`V6@Uzelyh?#Yu| z4GKnIhzRta{AUo|#=5SMfjw1f>U_fsd81adqCNyH;`z?Qm&iJmrv`OaZ~Qx{h?BKQ z!Aq;EsWs(T@Q^e5jtBnv!>EAr2+If#z;iFZVsyzmQJ?={@-3D0XB4#5!FjUCHh*%^ zSzWKgL6etnNqaN&&k=2q)1X_Bs$z}`U-xT%XT>D3e{)tZTy@Y9er8*7C zsz@1UsFeyuXdN2&Y_lYyc2Nya=4bb|!W|a|oL?EQ3Mgoy!*U8pw$4SD74b_Vq1R&~ ztica(Sh~XP&oYbP@5^%#DJ8hFkw9h1fpvWo^n=dQo*BT}56B?qOhz*J3J5U$Q}5Cf zu-lKzw$wVp6hHM(2`0}jhH`nJpB>Di5aWlk#p~i$?0uxR?bw9;D+|GGgG2cD9xZKR z2X3@U_vO2WYCIcCy*VLBdgBZjODi3a2-ujQpG8K`fW)`J<%#Sx;Atvg3hl7}u7uM# zw$~nlIOG;Zv8#>cMKdp?Q2JyxQ==O9l%7MhZ4Zql%C=e%^nkv3i9Uc%Qb2G%oIl?t z=GJ^HIG&A7?rS$m3`PVvYmU~n&x?%9U)z*gK zqC1z2=SY#luBY!s(v|u&H$?0}G$M#b-JpFD8ACuYL(KVAYl2JNe4meNtmvC|h->lS zK^O<6>X3~(Zh$EGS^T)o@RV4<0Wm=J?-d*Aq?^bBPE?Os5SA1ki7ZH7qjZk{kcEd} zZw5Xt-qeK@DoZB+9R28xM3Xyb-k>AQ4(P&KUsJ;NxnnFs!>dj0mCWdK+V(CZ=P9hF zzhR(2M6s{%5EFKRl{D0@a~)O58r!WiF)sV;(17hTm7qqSpo3m!Hm`1jBw>RCd+5h z|K=VCYxlYmtm<8t&M>SP!Wvvmy!jwv!%Qg36h=~EKq+Zrgc`^T(G}SSfu4juJr(8b ze-cwcrq^br5i{j%!|9b^i;8|EN`v9wlg@q*W?9d4lZYrP?GhD~fM8QvB6k!S!fT6w z77Ug{LXwiWnf#6S@iC63bH473tBY-L#`e?vunMoYXD1hT7I7BG5ybNUm;{gK5?QTh zHU~CgRMjIFZHo%68uaYHK6M2}`I`j!={`#f$T~cJHGP<6$qamsUFhz#CPxHy0H-G2 zY9>+{QOx})96`Tl-WPI69dI3Yhmo_+&7UCA@*;A;f=wx?&(l7@jszfGb_)XD+1fS7 zJ=#BtyI4QPos<#=j$QzoPc;?Wp(pD3p;RFg54@#CHYgke=in}k3dL{RZ-qw*AL4QQKvt@06d`2UneZnq%5NO$ zLdaOD?XBmW48q}OeAYehDR`<@R^c!ZAd#p@cnVW)oO$}|2gTi z<;hiv{xK>tQUrU_zVRD`MT@<10vU=ZjeA{Et)dYdCK$S8Rm}gIeM7h2)X`o{)v8TF zME?P*WS5vleqT7lq{tp?_>#F%Vasq6y(Gq$mN-@wx?1brG5^!kQ;~nu$lRN({8GW+ zztQkP_m)Wp+hhdC4QwfwKtUz$aAw(V(V3H(bZyCZ#nuz4!$TRgYr_5dZd z3Oi?fCz*S+2hV|S(?s-Y=nmW~rW>mT;ECUMmcU)qMnv9U4qj#CaP|8o<0*?iu0R-x zo^rgAP9=OczA$g>1D(z%OPCj;7*w{gRS)db-{^9EzAZ-$(sYeBU}t zX#8FPK_d6?^TBr5rP(>awVx&XHs0q)zrehlaZuMT3Z>Ig@XTWQQ}OX}g`8F;ejIeX z%j@7bu<+!(8RwJe*Hg6flFco+JF>`Ze*3wXlehUfoj%w@E>@wrG{Bg|9Ru>x+1*GI zTkZ`@C`@NZD(^5jkYr{Vu8%@%jEzO)X5>Z_AgA-wDidkB&}v=NhRNl~-Qx91Fy-8bkaewP#I_&D_*G2#UfeQvi=M zmtowmE>8?lQ&Bg|Ud|Z1gH3XpS|s;4V|HMlGhcO!`=*JZhSMIzmWd*|3zV%&#C}}@ z6_z{FN2SS8cw5I`1md*P8mG(H6O~%`iV#%XZ1jsJ zsO_+fjx$cIW-uWQue!)Dv|L2>gs~%1AjMrTBui6v-N5oC+fq3KsQGrR2+tO8yQ!-mU?z`|GY0j;q>9@4-}M;*jh~gUYKN0>gv%7 zI%U3nV+zJP)S6gugSR2rXRd35raD-<@3N>?R5Pp|UU03~WR$_OS45p%SQ8mw`Gg;}Yrcjq0-lrw8+*SWX- z)2qNxyS0|2o*vnXQ8o17&Xf;aY)?XVqkZ{^bup@rN7^el7{U*U9uGCcMu!9kjAq?n zAs1?%%|F^pQ&mb_m)!BrToEj_k@xa|gEdKsBmAW^{j(6{Hz>u;`9+VcbLZ=<+;U-- z71EI4r~d640HtMQqOQ()P^9FHm@rexKsF`Qc@X7g&_`-MA^SzwXGNgzbln9cq$z)P z*QF&8_JrDmo+N!MnF7sZ4DhR>bHX6dM(f%rDL_0jnXmmE{KZnqCnpj`ocue(o>~t$ zRDz#KP&)(s)UaunL zP8Cy7e$6OQ;v>($25f{1iG(^UQ1yG3n+1fuQ0L2#7qx~RuS?JS1)hDHPv^qXf} zIfU)!Fej&u5?tIoR*FRo2fp4LxWKRlV2lG;2S|t9lmJgc=KzgJ1VDMsd(VZ{iR1Q2l#`#z-vPvzA) zLx3NxsbVr})nC~of>gpy*I{;(A98mrdz)Qvit@!{tQCrHzVHwaMczD_sp=jL-R*8; z1!-m2@DGDldUfuC5yyC=7Ru7-=~bC50F;LgS_ZR zmo5#^45RSWm0GvlX|j~ao_ewJ{8@PjI?VkDdn8M?Y0z= zgb>#h z1mGa}Tc_*2EW-J__mZB*9`!;vL#Zb%qbPSBfrW2t7tn!)y`t0}+Ho2kGK47}5JA!QW2DedFAzn1sCYhAquozt+>(5bS= z(qDYKoEcuQ)gQ=Y$alh~2_6nL|D(VOsL-KMJ7*b`eq8W2%khJDb2JODp1JY2IXAD@ zyd7lV5aesxB}mFIb?WoQnvM)RvV0E*$Vm36$ksr^7QiD1(Ex$B8wCt25D*Z9Mom`X)G z3QOH+ow>j5GF0}>d)g=avuB*SQLKN~+ez)LIgpKOxfBfPYF)0Y2JbjWK~$RktP-nL zUCd^j;p{?O%DW{9{3MF*M5k7>1Yd?fAS|8_CI{;(S5W-W@h5DO+Ki*AA&>`Aom}BN zl0Hlp+r~P77vdm_$wqq3o^1_UiVsuY5Xmg6;xl%t6SDDlB)-GI9=wEQa&6`*Br^i0 z_I7TU>mgk51dwY#WO5p>2dA<}l(%je}(nomA z3hUCwsMOQSd}Xckb6gz_b#z1>Yh`SYL?GyfD=jyhgE{GlB`j6xlgCyFrF(}Vq*~?P zvpgB4*hrV74We4s&gc2;Z>WbhC%H8Bh7=o!mY`lxiw(jj@C-X$^S5nvx@D^3ECH|-p6|KF3wdSD1A-dkRz zMdXVrFY+d@3srBXKF=iUZfyJJg}si*%X;8!8)!dGQ!N&8B2`%}vf$8H*DJE*e5`%D2iKP8T-ca-qxR@3Oh&I zy@r3pS3HH%`e)EDla=Os2!7J34`b`}?+~mqL9-(;H+Z%rV2SPmg|Vt#(p*?y)8!2a ziwJ|dBPO|q?P_YTMx6+U&B`o=Yh5pc{9FL!_#6S~yf^Muyj9kGvP3gs!6EYL#Vb%| zzY7Qd?#L^kOex6lsjFOs(Y?n|wZPKQpxZgND}pko;b~1bx|J6bx_c(}sMVDH9tJH* z6V_J#X^5{b%GkorBP>wN*K9_HSw`l;D{>po9>+ZB7HTj3;6xdRWUz5H`q5!({SLf3 zMP@sOLp}*7P69H(*+zz0y<{?(cy!xAGP<)vJD007wmX3_g`|F+Ipl-rWunX zZNtbl1KKc-;-IC6zR5D#?`QuR)UXM|!vX%FWw!KY)IirS=b%J{7R2HxOVB^7582kKgC+F-X&%)XHOM&o7tpb^Fx4jI4v5|;Y_c!6?;acX2@D%_otVk?;$w#u z>N})Q_Vc?(MXx2G7qF&P-ykpeY+Cz$fur>N?($ec)Hw7EM|&^~pC_|T{*`8{6@j84 z9%})FKFRdb-64R2n3k{2xksVw#2+Xb@82Hie-E20g{pBv1+__&XRD;8RCd>)F_^^y z${^gq0zVR%RcAO|5L~V6@2e0?z0oGsp*oxb>w;b^nKpw^?EIxK21-zv>y|NZhcPyr z)8dSG*AlqoFOvRAL1b*f|LBCgnmBi<)X$_Et^8LxNQO=cPL6v*+R@;1 zt$4t4U+fcDOnQ~}OZ@Z~ct@jJd<6r@2d`HjwV5T-BBaZXwXyH-uOSkDJW$BTs1|xl zw*eZcQUlL{Xe%$sr0M9>fpUN~fUUi2VcC_LCc>#Ag_F|CM8$cAUODc;L%nn*hHZnK zRbs~VfdFtl#uOw8)K2!`ZH%#6%VtGuqlvf^iB29y`lzJ5MoFO}l6K5Pi+$-n&AM|2 zNB9(VK&+m3Nl={R=`YN?VhY{#)LRVjsErNfN|*u&nqY1ej8p5B( zl^PpJBmlN#Hk(CFMU2Fl+GHj@IAXdjFN8?>o&&E1iB8abHk{)6(8YHJRHZKi*FIm| z*<7S60LOE^koQ*|4-W9kc3%T#`U;Hb^05J_xrm=FU`u3WmU@)=kU%n5vPsUVtl8LI zCQ{zbEQ&Pu)p5Ux4lkAEo>`l0VVp-@$-pGT8EJzZhwcD(Z20W$j8E3ZhTyils9%kx zaZcD6K!!UGLdS+k2=QJ_ERQS<5J&S9(ca=xE4x6?_fSIF72#-6yG^(h03L|y<4P!S zoB&c1`Dvbh$Wm?EIq}KLI)x_u2HG?TOa{CYh2A0>`t+x_4^cT@53mMy^1ntYW*PrY zyry^UN&?KFRdDNXOFgU8IDGF_ld$u({WWPu(2a{_ z>(wNQg}v!Qie3(@Q<>eQwMw9;8hN4&($l|`VGxieAU&ZPSXmE#lW)x{VZK(Iva=)l zm}rUzE$mmfQhhvi1RJOHH+f9LO0&=&lSb&Pl&Y5qo=iV^|TO z>ai?ylsh-13VIeDjmsy7IWSDcK3m-`#LU_+DA>>%8s8_+H!EXsyY)3l2Y3^(1%C6C zdl0lms72KqambIXUu+k)lKOd+XvX7U6NH|;7+ZOjR*5@mtFfYL0W~;Sh%$2P2b1hBNi3ey$Q?+ofYZ6Vq8Ca~PL@;o_2gs? z4zNu3A0MHc;EhfWCWO^@4I{#271mnASXTy$g&EM*%=~fS#V6>un7nEd_`u_U0 ztNq(;6fFpNzknj4J?=>+i@n&>c#d)N!WA%JZ29uH$EX*AH{q)iQc(1xH_9VI&U6EX zwl=QbFa6NSbTMniyiwtM$$Waug~cI8dL@LuS-H!El(45(;rU!C9CbL$Ocd-jbW9n; ziK-nkdn(SGgub&sUCdJ(!^5T|PDVX9p6Cxbvz)+YGDP;aagO!slC2pPK^qs&VcrS` zp{n~M<3jWO1&a)!0)>8>Q`PYhh0oWKdY)0Cx*0#3>kTKI&Wl_?6|(>wi_{Utca)Yl z>I*n{8l?;XMQHYPpOF)(x9kW(Jpg!xxhB#)PGP%k^?|LEfhnD?+;v709vD0WP(`8xa z27bzg{zDFA0ZcLQz)bN=SS~m2w}>ciJJ+L;p;K0xiuETRO>iJ_xOsi@i!6tak;|LA zLbtBzrTNL@IT;h5Yr*B@4TDHi2+~JH9Y+;x=oNf7hM<;xhf3+}_yj4KG`kXE!?N>O zG;nvGLbY9Iv|F7;^aj_KwrtO49IM~@v++VbZtw|iFws$g@xyZ_g_!MJl@ zIr4y}Kon#cD?ot2`mXfUZaiW4ul>-WsOAcVvQjR7${P>yA!8Ugc?~W`Da)byfK!jR z&&l))hSx=RMU(RxG?xX{K_832Ti;3i041G^WyFgs4n5C0Mr~oTgTVE1{FO8-pYWC-rs$U z1bc+$X>L>R(WG{f>*(bQg76&%agh?V<5XMZId?@f`d+v^L?y}5rBSyNtSuks#3_4j zqa)(W8&hJmU}epsFpt;bqFoH?zO(wtSHtxTWFwYul_6lP>7aj^+V6Bz6oTKsUr0v< zBN`C{@>jY!5-)C|0_7;veruf{oM0-`xP?}D5bA*hqhrW(OC8Ozot^O#3Zjuf2i~|p zl|D#4*|=*nE1vbbIIbq%pHu$rTQ|et468qp_w^G>80*-1E}Xv+0~`{PdkDp0Y5)vE zI4lutJYOiL0AFaAF@Wf0cH@pcnWC5Gdn9__uuxI{7W&oMTx@4Hk)-#sZZf;+I}kikr?5EJhMU^H-=;2>0LuG_B5b`28nM{~U6 zTti3pk*YP$vDom;JF!bmb&I27uLT7!X{Xnl1h$b!R~lM-a-fS88m-fzX~ddEhW%$f zsZY^?9b3SLRS6yp!fGzQY{#sg2AtJm-^&Gg_M#h_!uVIDKc2(WBw(BcAD zB*{XxDdgbfBxj>Z8D4xpmAQ3SlNQ=_8_uH9zjLnYGEjK5r?_W_9rbmqgKT!JF7>Pr zs4Hs8I#oAu2_=}z44)3z@D8X7YogU;V(G#=ZDfW$;yD+Y@FiyD9`QLzkfC3d3S*8zv!PTv*3_1%J!3;V#q(U2kbDOBo zFn8{X0LKl&%2HmE`Mujeg5e-Y)j8`j5-h^JbJ5y;Yf&cU&IO2Pj~M& zd!0)J$3C}p8WlDcC%N%^G|pGFL0rH5B~kkLIv;2i+Zy>`5B=+Hd)LP(6BU^&)GN=v zQNiX`sd9#@l>y)E7zgKYaYVj;2G};zBpXeJVE`@| z7t2#&ux}&cFg{iULnx}pLqW_>2fMe51&W0)bNgE8bq+W3q;jJHE>)MG8JHLy2?i2? zhTT{kvy@(oCk_HFMuVrQR!C7vaU?=mr7_AwQ+iO=W<|wA-V>^X+FVm&e~*(suhv7e z3^EuAp0J5hzw7!V_$awZ(yrX-@xg?tY{g1cXMM)Jp@d0E!USi)kvC109(LY03jJA# z!A4H^(}Y^OT6h{?-zByoGjYqp^K<|n_13TQ zqZO|PUpQ2C?R1r77LK=r=pN_d0e?Im7ArjzHJW(4A4 zfHX7l@XR%+QdVm~+YIPgy)`~J&I2*p_F=biC!E#~K)Hg|NZw$A%X{)e&cED;|4s*=_s99&CSe5}rG~W8 z=%9=2y&t8pb3}RgaX_ho^KE;r12um&L$=G$xrPq-lyhYzA ztT2GR^h3eeH9n=HMKN#rh4}LC0~HpRH^pzm|Lsu3lPmgvlAfMn*4jZ)e*SQ)(RQ^R zeb61Z3Mb0|0C!$vsWpuwe~%nqYC184^NU>EoUp~z6Xyb(=zkGjdL+Fp8Cz=kuLXnL zc?>Q4JOuWODvR277KU8Y#1=0q*Wv0TQxF`%aT=Q&9nOR^D*ko<>*GX+sru@Jx1R9` zbmF*8um(^BzbLZ|I)NO*oQCaev7aqNIw@(dwZ2`YD3X0%;7#Rob$N(7VFhPVpq7`e zJw=JcbGv}Tu+wLxc~v&BGKO>4sE7Sx%6i{5v+M<}0WTTVopHU^%soR}LWibl4dyl{nX5M&?^MUr= z;A`@S5gcPe>;UGLXKiNpV(Zg&17oUQc}FYO(0K)*xiIKOzQSXa++Amfot5yH9GK0R z=I?@}f=NIX?0-{HI^~V4*oMW@e={cLPo5E$RBl5Ga=oyAVPq*4WZkLUH^HQMTjeDc zXK`mzg&Qk26mi0jcskn>0;_>U05&W#w~eIi>SF%0!2oGd>{{IwC3O{bcJvG=-V*4# zct1SxUN|O&vsb6rm6IGHO9ecq-Z3|o$YT)IC*#4U%`>)ApJ%!eX8(g1yV-n|nyUlo zovZnDqixF&{|r;*RW#CR66fv@$fW{0kDIgU?a-c3grf>+5sg;(j@c%oi|f^16#CB5 z2gP*z4ZtIN|HE^(Skfw)<_v>@E&}UbG`DF!w>?sLD#FH3HY%?Dh>JEf!n}{y5>|=1 z!TP{Cvp{7Cb<^e540gYkjE|CrqyAuQ$&N@-nFw+e1gCziG-C*^Ah467_qe!(_%2mnUgDp4Z`kzSJ5RB|rpm`neX+c0P-M!#R&FV=vqxxA5 zjY+b$0;La)F!ku+BEH(UmVnlwNk)W5eq+M3^e{fn6FqKouN4I+$?Ki(5OS2+IGb1^ za`nM}3}U{Zmw2dim5E3)l><@=RD~$e-F^zYZcL|T1uBg~SB)89egbK}E6nnM-R5hP z<`83(3ov748vq**ks(*Bv*z!NwQ04Qw;2bAZGmiFhN~>s5v0YT)?-W6K*&Ar_rKjC zmzKd1Nz}I|TrvFU45b>p9;3519q+n<$7mSJ=~LeT=X|9K-MxDQ(J$Xhz0BY`gNty7L7a(U%TNhV?9QtPs-R0z^9_j>#T}U; z>rL>UcF&`meLP;Ij+~$1O~-XoSlO79g>8{Y6|dS*Hz2V6Nk(`g(^5OD`XF+B=1lIr z^seb?q1NglyDJCsdjiN+mKIa9R>w`w9B%Qc@$g?9!W_D26bc1RBuWLG(3U$e9&+L( zm45tL;T5;OiEcic2$_f|=vsVoOiL%>7vu9d-}tH3-k-Iib|!^8Tymd#>fFo3kZN2N zca!8K=6wDc-q{@K#r z$8ptqJe)7B$-UrDYXi7n7#SW?2QT;b|$F=e$%xKaX0_m7Q4dT0fVnLh&*w} zHmJ7iRp*woL(aUw&GIXL2>&jhW5p|exR-ae*pay(Ma^5U*XWokCPvW{9YsVN9eO#PCFc|g-{|!mltIreSi&7yvkzr7HF`^~sD5{T?DuIEqR^t7KxtR&vJ7p>UW_g;YPkG>2 zmt_Bk(K-RvDj>?c8cvBWys3$Uh4J?uCym_r1#;P_|qzsO$XZL2_N3# zVJQI7FE`wvhc{jBP^v}AH!c`!8-8nAP;bq;c^(y?ZlL4Z@AK4pk}49`50|-`vCcw8 z|F$exGCldzWa`+Cy%te0BFy=lyO* z0%jJ>7*RiNL8_;Y=GTb$9pO0Rb{b1Ox1^P?$o|k2R>i#6OH3zj6!XoLVJQ%qz<*sN z%~&JM&#|#IV^x<&*=6=a9Uj-6oSaFR8i)^LXQY+6;Fxq{ba;WlKYYqa^3}HYnP)%U znCTINM_A+~5b6zWp`M+nn-4Fze~|B6BZ7HvP2 zBq+d9fY!nGzpx>uqmg!|bG%-zm?JBjx|k_jRNut`g7zpY)GHM@K?Ju85&9^9V~R@>&jh@B0+65H+q9& z-|zX&6@JIvH+e9o(lj)_UQmTVQz5SIjTFC@QqT~9ejUbDmY<|-@78>yhmPB{iv~zv zDK8qkkc~aegfR=dWH(x86m@7hSzc8@cA6Wjts4ch>g!5326eL<$M43 zK0)alV#YUkj&rsdlwwsis{qmvU3q>EeVV=rl#QO4wd`(>aXao%5XFWfcaDRc((c9T zSbP=s=sDyYtK#qfjH{)0o~${PgcU@vK=iIK0aV^7KcknG!yyh8`uUPAm!K34Q@j_s zbrb-$TB+{Xy16khDmJ6E-|KmDZ3{XI04~H#ax^HM-{$Xuk=7DhY*$iJoMvGmQ!xlG-#0~i?skztXX)XKyrM}l~`?T(k9rxNZds^-htb~d0oGG>&yT>)cAf{^nhZ~E{x5-_SggyZ!-@H+nQbASL8{3#W+iv-wWaMm?v#@H77=P}?CwVPJsIBz;Pr%G{>y2}UH1uUW|*goH}%6IQ?hb$N>jyJPW$7sd0 zf#c6Xh0zQvMM;9DW)vihkbiMS#P%xQ@3e``x6a}J_1!Wksh!Id@AP+v>WT;DZH~h` zQY)1^m_W46FY23#=7z=z4my2QZ<46AM`n4CiKDaCn|HrfLL1UZsPDng3<2tm=+IZA z!mNuVi7eNjW3Fz0<%1zt9BndWe)dfq2fIv@+w(q}TTnzQ9cegFP(6UdC|&%=m7;P# z@02Ueh+!v~XnrCOUANvkP1DUJ1W>zfdELF66YzX%-Ue=e+~Cbpx73HW%E<-kp9Fr8nFqlT|nXCd` z2jCm3&_r9Hp?2Bcs#)xC)u}sKhbM(`7z0}>zFmyE0#|LR%2WGK9u&>^G)u6FwfduT zH8?15n zVo>_9{*?ixMF*8#7Fii{9#oYqAh+eiP}soOe>o)*@+by2%)CjjCwz!pUmd79(}xB= z_W1Eh4W>+_sBH*JzgS2C(YaK&>Iih*^wIY0wPfWi6!!5?YnG1Mk8en`1JC3eUc0db zoSAjneSX}OI516*{=FMIazYcMUi{;`DaWR0o-X@TwEJN{zh{zXVDAHMAeJ}cc)5~S zgmUW%`=*$>fB>SvxyM{fw7?$)-?FXl*VPpBm9YP&8O5%MOY3XJxO7+AUJF(=K=ENX zg5-V^Jrk`XZ1q6~#nmH<^!Z+I#SM)!xU?7qnZa0_xbp>ep0Pw4xp3mS`Z1I(Ey$ut zOxC+M9>7e6Pf`0mo1+ZY7Aesb5!R;43%rpApFraH?$|)d#9@EVi1=Ke6l$F%p_d22 ziV&5o&LhFKXx6O0<%yr0k!WRj@L+W8 z_D0t_)Q8~+Lthmm=XG{njsOdd^T3Fw^YW0i-3A<1>OvH;?Wy)rplxpmJd${IxO4zA z6J10p(JMv&BxMwJ(`1^XLpf4_>xP7`?BiUlpDaWyi~7L(Lq)fCkyJxvW*~g(XsQ%m zphOTn_WSa(SVOSjPKft*5?Bu>t)1t@bX#uh^!+Eumkz{V>PIya?Xo|jCh<7P0S0)) zx(aP)B2+y6Dz?q*^=a!=W$e!WAy`8;Igo0i41O4}JmJ9tPsAe+VaEX#Lj|(ZpvG9@ zb3-cH1QZ2p3R(qj7e_9LX#2I}O-1`wA8b-?pl0U}py#Y5j;|&<#yz3&DI$StEPMG2 zdk)=SXSC{V*a?0^EMJkEX5Oz9t3L5v7~TNUh~njgXgLq%3%;=)UzVs?m1Neodb;A| zKaV=SY8WxkBIYCx^2C+=0}U_t7n!HUzV1x^-5m87h7!B>fM_h7uhRMS+`Xr647j!` z1~j~geP!fV(P^<%NpLCLB#a`78=tF2*qQ)7y%spfIg@ne2kCwq^gu^Z-$VNs9UAGT zTT*gl7WNih>f1WfsT&MywvupDc4V#@IilQATm4_ALaPQYbg^Ljt(o-~1V96L73R2e z&Iv21S_|FT0qP=9GFnE||jimC&EbxD~MwYF!2J90Wf8#?bJ;z&W6NyJ;*|Md=IX#)!Q>J6tgV@M7r<9`t`V5OqK6Y2**3XGv{ORLm^U%?{Q2 z>EAN&BGAJdp~uv`GOXQP-x_^0b7TBY%Y-BR4Ro&rJ7@Pb)U_mrlKCA`-dC{y?)CdSka4xgC`iJP?b%#5zsg!@h6r zoNx~i-(3!}3|Ldx8}Tm(Cd8Cjees^;2uMttC=%^uoI>ZxsvMR%3z!t2FmsBiBNc>i z3tgwY`0bY?8(Ls;$o&Q;&A#uDbDR~UCHCC^3fN^2Xb{IO;V&qb^qA$ZPGw5eU$nlN z#EQw`C^Od$utp`MZ>)U$BK6$`7DqZ63MG$&?`)6PoY3|=-cw)q=F`y5+o2emk&wLt zBBhxBHI}s1_(vbWg~V6Ct*q!HU#+g7?|Nd{ea_%-Jr}OcuGl@`*Ga0d)jwy zL_4?PAtQ~B*CHh8$0|UGe$!ls!ltPCUoxP{6C5)@db^=noo$Om5azFnE;iTz1&a0L|%Ns@?Qa%xJ@; zOGa}UU~I!aLXCL{rsVd#44Q&*+FLqHHw=*RBI7x*BZLLKYG=-a1np!30^mU*ok;W$ zNHR>UQ_pl2p@dWZAz#OAu`&kSSu_^|OAN~2LdIda#+df2oh`)hOnv|`Vb+btdT^uw zudiE6QGu=n_{C}YPt*Z4wEKOFRX^(@gaJW!ll7R;(@Kiz1m2gj69QV1eYt!MoS^0Y zb<{Mx#0FOAZ{7bXlk4#(ELn(dYaxVqX+^DSdbdyp_6WkV%*>)-&K^eYjM)UiOD%AC zq&0)Ydy{c8TW)WifkJ>LT$V)Q z3*g&TZUQFX#xM&66a{2B)QmG++Y)BTyvWYM93SpCck;U0ND)ARslI54?x1$FgO z`VgEmZ=zwq<)o{haM};yMR_Nk9>3{u$eR715k>Lae1LZH{%aKlDCM4nuVd> zj6mz^#_VEC>!GnZ6T|B_qCC5Zv;p%2T0}FGye7Gt>#sGe7)$=h?n)q7YTe{V9Vmx-!?ULs(rR-@Pyek}@?};KNo(2jH-7Wm6c6W8u+>_`{^e)gb9x3L zYmhNw+NVW1Qe0Xv0Wm{?ubQO*5wIYphboDOj>dMu_0a98E%l&S5n zwm1Rm)oT$;)I}}Os_Q}$;8F4fdB}o*7aH80HIPX3WLrCT2ApBzYY49+$+#Fu;dSO#53`?n%49X zv&PoyeGA-x30Q@xQCmZ{wgQ+Vzq%lm)eT%S^Yu;CdkKNK5N#QGz@2g~!r9lFz03Jq z`$`{|$Bn=|`@P>DN_#6}HY#}sMLR2G(Zpm+-6W9BdBQN2VTmhcB3^6$`w zftd4lgwf#d@-C$Fwl^ZLDdCzv?D9tXM2puYrQ1w35nju-|@GmXUnBGbB<$U zJgkEKLN6kjIx{>`x!nphdx*`N_(TQ0eCT0c zK9@Xe+(`nIhJ4X6fIp8@vSy=#WD zP>VkvZmW{(-FX3fC`w>tbTO;wBXO!&4d1f@*jp)IH8fyK6mNEf_&#b+GnK5{c&(J2 z70V?ycv>HAB{PdI!>PDFJ-MVJ9bF%K%kdl@p;e`avJwB)FxW>!e$V9gx zCFh;oEx4}3Gk~qA$8)w^uzPk_E{Cm9W(jNz;5=0~MNiOoPm3m0!JYQkqgc7>ZV#~3V3Vj?if36+pKbw7Oef(WGE&zP0>|c6H85EpGGe5-`GZ#_q{SLezrai` z|AxNWO2jlo65J$4^zm~PsEmZF(}oJ#&pF`WG=IifkSkTJcRL?j9_O7G%QKo-m;{es zA#H5w%R30X-HQdZf^Vtjh$^(T)0vf_nW2G$^J*mK7^X_MTBy=moKn2y8PllSAA^Z^ zgmH#d7!fZH?oJ6L;oUtgV8`4y;cG6!BgF(HYx9v0wr#9B8wXyT)0m7OVq{0$+SDg4 zzWKzXBYKZ_Sasm<67yW}D~Ds5es<6Km_4zXr5$~X?DW}!OQwi1sNkE#CvX!!pp-Nk zCoJ+VoC2+pn{>d`-hm;IK_2tj2AXO{O1+NUn;Uq-9;&HK)W|k|rjxSJDhO&7d-^O9 zCDjQdpJr@5&GU|GI9cCYoLt4~*@kmyZrQ1Yn&&5B4N7=p@PQV2aZzj-RX&%|>| z!A7ENh7}W>=%xKfa>~#KSwAu%DqVl3>7rPR1 zovnxT^n*0`w0&A0Php;m)>Y_0i-)vKwE0OuOLIo*Y_p_)RlCnEfzXU^c+4(pLOcZ? zF9$p?*mNmxj}G1`86umphwaMFpVyx#6y9313Rx87uvHVL2dzH;%*YGM4Rh@D(f z;|telP`{=pX8-tvBe)kTL?9aX~cfkYL)ubGwQg?ovbfF z*_H7$4QRfwHz02kBIH+g7Fr$}7$bSUJYw}>@q$04i$Db1jobTWE?t=+4aO_^rD5n6 zYip6jCAzh{v?_WF402W2!_nYbh!b&GhBH-Nhh!$Rb%nN3tHoh4Ceb!rt zAdQZWidQN2v#D#Y{cU7yI!+w>XB3^dwFWsCIvC2o)R;7p7a8)I z$sOT@hP3jh(QB(-T}yDjeH%z5Wu?!!i}RJ;UobyOz#-xgs;W6eru9u5nWLQWln}I~ z!!(3s@grW|wJnblri=D5%4bLUo^qUf*0f}_If*|&ny-4S*3(g$LPzA2;`|Q096bW` zLd%y_d+W&pA*+KFze*&kX&IP;S$E{no3&r-4+Z3o$o=L8S@%_;s=4im4Tw3Qn4z5? z|K$=JhCB4`MkW*K^s(b8l(VJ>nSqF7Okt4Y$fWeUo4uku)<@p1v7jZH-Nb_%$dpYi z95d{@|3w_3+5bCOpd)fTSD@+%pDsUHTlmiB>C|WawAoPfJ2r4S;SJ2QwF6xCZ4#eP z={u=bneTr6-HX`yGLp)1=HsGgJgtTLE)$|`DrXRpK_T?~@c6DaBHK;LENBu=8}hYH zUhVXTm9&-RY&<#yuwXMFpWvwb6CuIAeMHzhAluH>%*4I z7|)&SmF4oSk`2D8y>NBykjhoO5;qiaJ%c~#uKGQK*(053qota0U!3l3_s`zF*iQyy zn6I_4#{rF)3%%QSr2!DR7CVB02ODDSLnaaNptV`p2$z?><(U5RWs3F)k=gtv_%LRY z+XptQ&#yNbr4Yv!)F;Q9uS2DPuw)*#tdt+mfIysYL;nj?1`uxWhm zJOa-4$AoBA5OqeQbB2I$+W5LFt;u+A|o2mF%;ZHPK_hdRD*e}H`$vN!fgBEm@b%BgCNG> zb%vGJulvU$f7Q{g9~6rC$ZU3K2Iz4Rlxuv{1Nla{hv`g}ar=$Ak@P35vv&u`8Dpxhv>`)1@XXQNjY4tv|W28<{FTUyR z0-g7qm;}8VB~WR);>Hu!)wQO85|DEozQ|)`{%5VR;S?Fsst-9_H&V~XkopEe6;Pva zy~&38yI}Pg zOI)9x1o!s0-nLK7pg2-^fRFas&6uH%;Q7X>gT=xYDhyQOf(T3g!Zl0C@^ep%)3c2` zOLvPaSdz??uY;6=3Z5c_ww3 z+rk=V^C;J_Dm}W&bR!&9RLp zKCNY>Ou0z-MN5piDX9Nxq(gr=eaX~ms(bFct5%XdLo>_R?}V77S~0ZEf-G!a-oD-& zuzPp*4mj-HfVtePVkIFmGj<#81)dix3(E2b3v#9))Fl`lO0^%tPkz5@Pe)KbVHnkQ zyPDFt%+-sV`MK;vOQj5YCEYBz$UO}&!7c=RxlnRPiKqtG-KEW5i69G%NX~1#N{mIl zf;EAaSZ|%<6V?=(v7m^h0>Z!+ZD@U9lA+lPzHh0&rb*J4!?Ts{C(q^r7<~ zac_N%+(D?VVO_h_`Fyj80U+gIi^@vz!sJzdOm|{Z?D}L|R1~%Dumo>-?vh{6Zotb{ zH@RWi2?k>IoMxAqu(`07M)0gxTe+Q2mI?T~q;XWa+{luZ72j*T7mv zU_*kUdaCT%b_J}xa5jCZz7?A={Zz0%9}6$QflzM*%r3g>V1mMwwOS(_>|WDO4MLFD z*;~ujAa~h}ZqKj6xSb=iTR)F?rS8xHtH4&%lhoc~w$rm3R5pO%wMNUO z@otDVum}lYU69kEa!n2%w0);Dz*|BTMk{Kqzi<8^dc&Uci#zE@$0ob%53W6M=79oeb4Gni!u+b4uWqMzcyxyg(S+<@_q^C8d`<73_zKP6_cm@ zL~nK4FDhX1YRQ9~4mIiaui9-h+PFZ&dMAP@j-VP*l+zIC+7|^$*!OUfY5QTfPjeR< z4YZUpXnLeDVPvC8YKz3m;Icp10XxYRUJ=>Mt4~q?P$e1q$yut4kfDY4?CtwxO)(WAUy*Dk%v%H=bCYzuNIm`SgHOX2&4f%!ve6_I}^ zNvSVD2<}itGg4ZjNp2XGINK9tPpNJ0s0y`2Ws8%f@Lk~n@0m`!BDcK`pyd>9N)M{M z0UCLYhxqCCOMr}DO;b)aPJBea9Q(O_Q1TF%X$ z?*v2C9H%@cSgYmT=K%qs_{?OzHD13&Wlkd&2L~2vvFlE~8IpeFauP;G@i^hpXyGOv}TbTo$WqA2A2NB(4dCKgm!pdpZD*) z_Pv5wXmUJ6ULi6#)ro^E6gu*&sKDWBa|dBp&#xOPPSN}qYPAmx)d)SUu@DZ`SfJCN z<>jf)VJt`}O32U0ttG#v+lweASTwIVBOM|Lrv&LYn?Jua+37HeK#%r)l`|@4JBA~6N+3Pm_Z&|7fj7WXc-&Q#F)>4>&hsjEkB`jV z*}2Ve;n08yHJ^Fl6OKU9e1+dzN!OZ@MIA z^(-8D^`fy*>Q*8hlw=$|nRUOV^QY91F^9`7V~6id~0|A5;o)P_JvdW@Pw}s^S)s@%E!NQ^oUg z%X{idg8rU%O^;=#73<$10Aykh7SNTM^|GXjJMyfDkuyZ~BT&iGOe{(?uJZnaJxlHc zz`*Y4K)=R4a*Z-oPmsP3V&{JDYU8B-Xz*{AkhilZ5|w|)7U=O&0(m!NxTB+}!?Z*- zg$?%sPaF1_pI9PGwuZ86o26NRFQD)yLo$)+$~V?LQ-?J!=+#`NpBNLnRdAM}?Fkp< zEvCLrNMr}tUo6I~d%N)}CuN~j{{9&)LX<#I+j8nNSd=7RA4I=Fc%b=0@-yO50*3TR zmUAZ@T5p(<6|YfDYhO4(9znHpyuA0Tsv!<>H+x<(-#akY=b7Tgw!43pPOj=Ji?`Em z!s0Ouo(01hg*qq=eG#6zvJ{eWtdU)NDAHXk>8E{qLVtuz0%59h$8gisP<}mToVvO-6;u8(t5)BFpu>bIjPl(>Uzix@I*LloB>m3y`f2seHJ>!2&?Br@2a5pUbH+3 z@E;DspnKDOCvPnR=Vn@}=vTJ24Yp`a))na+&U32t-@LfB$g@@lL8*Z;Lp2IpW;gH_8U% zQ|*-#M$6_JLjhXwNqP1wA_q-A{NR%ixRM2YFKCbS3 z3hgACa<3UYU{-hw1|qdh=&C^Z0K$7D113Nf0d5pLN1h<{B#&M!-jk=FI5Rv0CFm$) zp_yduf?(if(UQ90`Oge(IDUjci5g@Q#W-t%sc4P$4zawwl(lo0AU>&|=wJ+tDl-Q^ zlL1`@mrM!gal+#p`RVoaS47 z`hn$_olZ%TWHE@0=$l1JJ_&K{1wtql#l9Rgz!RI?Uo@#>8j&V!(_Xe@*>KPXF#ANo zFJI|2Ly+OKe$G8J5+KF@yp|BPy6C#rnxBu$l2<_ObU`HOf^D8N@+kPCPVGerqq@*YA(Lhov|tIPHWKW)2~*UFKWK&}V(#5?79=)|NKSyQ zTB5zL?dG9JTUf=u+PR2#0vB!pA%iqEs+vMs9olJa;P-oX?yR{XC76NKsH~) zH2OmeSTJzvi z*cNVBP5y=QhR_84ztS;wJa;*hu`XpVT!- zs+wO^89Ao4`vW~p$xbQ(Z>NKkat+WlmZ*j#Hq^U2u5XfA;+j3h)un-g?QOM@iL}@9 z^6G$gGV<=25Qns-#@*wQ6|w5Rn_>K|Actj`iS}eSCNh-}aUDtx^?z$gk|OcMn$9GX z7T``*Xt9j?(&WiB_|LF$Xh@C(3>SoyXIW=WCH}c@RwE$yIxl1JhL=>3E4o%dZo)-w zdL?~Pq5>$KC-InF@@XzWB4_^WP4=huLU^!_4f17NlWU`bKbOkM8)0wPjR)Tt;6*wt zGN-jyF5=AIkdv*4|Hh+#k+)+z0l1c)`2JwH*cZ7TNG#}}oFFE7aQSL?q~~2)c%OokixFFH?MHcJR9krL=vuqqPh>-Z zz*_7M<;e0shKjdr=UetuRWb!y!^7ZYWZR_C`J@k|y;SJ-t^wtMNZl93Zb@$w$r zZFSL>`!T0EI?0_9cLKuDXEa(lum_iss?E{mGI7p~`Y7xam47)xRU@3X*YLH95wV!< z+T^g8SW1G@K0vsgvgmHMe)za5&v(XEepu#`M((`11XmuWun}UuGUsXVEZD)zs(L;b zldk__HId0s!k`3dJCXtKd)qQ``s;E!_G?bMh!JzD)hMLDDwZ~(1(;&Hnaqy<1T+DE z$#C;rEnIL_loc zHTe@hHId}}1&#lz-Zx^XB0Y2Mq(^Ge^-#5Ocm0CiG$MdUcqvwEFZd5)G~%%g#*y|N zns@KqBgsN&arR^XPZ#~%{MuZl0r()-Jy3jd86{Yna-+J8`NS}?_gf=$U#>71BCk^B z^%HOx@gy>?8l> z>ncSrgY94t(WTF#VJ+>JKXuBt9Q)kHCVyazeM^kYrM2n1wcu?Gfi2q^zq`Oi*>O^f zXu|iS(sk(YXh(S#?vkRTkH$3_rPqHpN1j-W65;ac3N%|TiA^e3?X3u4ozq%KHEOlI zU4~!vG&D2U#!QOh@JZ^=&u{``jY2v-kR;Xq3odEKbB!Ph=Uf-Kqc~9j63j_C!C^U2 zhR@S;0n+Nrd4`lR++HkJ1E_92Cb8)*L)vpLx?91-L(|e)*vYzKcgy9%=myN z$@#=#yll)zWotDJ6q@qiOh^Q8B%>GCe#1??#E+&#MX$DZUS`XflSL$#8OFn39>JqU zY`UQjdvrAK=SImQEiw3$NBP$siwn%iB{qthTvvb44ns!KWG6oD@ME!5P57*SpPD1c zt+JgjPaNK>OVkG7Nuu*j{I^5tvrA!YyD4we@EV0^z{inyRK)&~Hd(XO(6_r2TGoBGTbvdZZT^Kp|;ga6mkaB4|@-lGSOMG=flcHq0`d z)bMiec&M|!t*MjHxprk*(@{4@>t+G+YK=bFp8{8`6SevSA&6!j=F}?W_{K$A$CrrE zUYf2ej?etslVhkC|1rohwJEH`8Igo=?hlQADQ}g-oQGnp3M2qHpLsh@4KH)Ts*$sQ z_o%~Mu8hmk)oc=Y0E|{_+XEwof{8=m_%y>0D!@aei$E>|T}LcORq}-EuR+4(*nLkl z8Q-x7tDdobzy9Le{x#(Dg?12h|AG*Fk-^^rW^_S|k4b%7{(ChHYw>EVvf)My34aE( zkjqWdo_eM8c^Bh`5cP)>_R5N^AM@#V%wnb>tkAk6Jt;$q8^SCVQ$Rgz=oKTcmdr8z~FiG&w!5#?YL zMr%kjlR~0Yy6cm_QEM()_QYHzkz}!~5*KQcg98}o(m$B&yLr?dLHVCIX{|6Fz?YxP z(7Mj3gbOnB0hYI;w8)kiRR1dv#Xm^1;-3==0@IgMXb>&pJ`o%Xnokf4_IHB=zk1suw5kkr?LmOs><}F|rxavV0i}(?{ zCt=QH;)rbeSgIP!DNm9|?v>_?i3ld@JR!Tkj8W!w0shDOiT$to+@BLS06+A+N!LAq z42f7OwZPQ?LNt>V60FIc!2>q!Jb8Z_IbR&Zm4Xfd*1vpVH)gFkNB!FKeX)>(Mh1|h zU7O+8;{c60M$)+y(C#gfvpe?RlpCZl`h15;l3qHvtTBYZ!@ZWS}gNcd;EA+_jjl=0O?0giABDNeX%sF@Xf zU^J5!?ha~^7&k5yxryDP*x*&8IY4VRCu|@T13PC03pPDE{I9D4 z`M@acTLx?$^Igql{B&6P3(SK=_iYUywGr-TIVN!h6M-9}*$=MTB#vrdGSg|h;yd@@ z28BQvuO5$A1@fO4TGf%nP&J)$@e5#^?&?piQgYe&JPGdscT;5L5 zP+*DMAW=F`t$DUm87?_fN__eur%C(%C!)T!Yt?%P#lbxWk7JF9gH7QUFNVUChfm&8 zi6enzONG3oxfifNF;nXY>nJ_Ub1j?As@wlvPRq{xKVa-m#7Ym00v%K8Y;-T^ZL2#O zI>ZM;#ipvDds3G^qkb3h6zbxBr_1mZ*f($CbN+XV&v-#Za0FOL5llT0X5HC%Y_iHI zjcGWo^Nwg(;rAZ*lO^;T=w3H1fMfJSvVj6bsre6r#MhMWsjK6Z?=Q%OfO0+n^IPfq zk|~76Z!}3U1j#zweD*vdP2s@HfgHFJhurTd(jFKmY0t zD@#>K-v6)Q_vY3kju6{^*6SX%*&|HOA4a6nx%f`Gbjq=uIus4j$AU}>&BK?o#?qSS zVn%0hU$CD8VDXkpzlw9kOYPhWZ-S3D*ap-mciK4)S4r#Xwo0R|A)R0Lj1^2Q`GMv( zOdQ5`B;xQT{ocij;8fx?q*OGOAp?3=XO*((F9uy*IVSc7a$E|EOL$H+72*;0IYQ-> z#~qN=fT=fQarG5H&Q5X+N8q_HZ3rQFERpu}gE)TXArEuu-QAPlVGHc^05R~nsFe-o z);^=jgBA;leQ&VmHseg)`OO9(je-lWC>Ad(kzq70cnU+hCh-3i+9%r1Jv)`onMQot zBHbxc$!7=Sc4GUtXYc#}shvC5qi`x#ghbj*-}|u}($UlP)&7@uO9x8Lz=&ZsTn@7N znwe9rbO+w_?+>{ctjbE##=~Az|_(@wLZ2m*=tyzqc>dQ0ThsxoW zah1T9UT3(Q$ka9+*!WPh8J=zU2~!E^c9zuE_Teu?QBHM)<63r;A-nFoAH z9Ys-arInoV0ibM-0@G1s?1iCS$>7M9ypS^M7B!UXAX|N7?#1I%zWbE2Udr~4Btu$_ zG3zKiIn5;{Ch_*IqBr72k-jY3wUFE((HJ8CiSyK@Xxjnu9VTkA^h27GLXGPW_(FgG zJQwy5n2jxcBK4=HR+Wh0Tb6@Qa^Zgx$ zn^SoY3R}$7Vsn3an)K%x&gbH!%ELDX0O5KgeS)OnA+}N8hYT{51K=0B7EFmiI~+U& zjTo+m^0?})1&<&C4x4JWj6^S(o4A`jgY6DEEMwMf@;tN9D}iudLHSig@cJF=LH12t zWA4e^g4$B&&@+3@za@f|zUXMIh0O@Yu5J!h`W{Q>J%)Td#V1D%L)ke-Q_<;g*1)uy8hjmbED zL-K~DU0VMJtltm-%k2=jBm-os6E%n{F*OY41l7D^R0iOB?4&EM^3ula8xD>p!K*3> zj4|~|p7Lhulk3vNkH<#CjcNmxP?V=ugZCTdsQ&f@27#z;L9)ycB~op1lWd$w3y`Rv zavuu6nE`34aojM!$t+j|b-hkTjHuLf!n1vpL`T8rEGmg4N=hxWvm93xN2Ho%%Eg~v z+ZTe@j48arEQ*u6Rudbq;6`NgyvbKk+`}0$qk{ewxvJv!K3fw|m`N_yB{2lw7Fl8$uVa#uvq0-QvEPk^a6*Sd zGIALqMQgj2M2^;w0&>5$S@@|=0hX>Bgc4B!c2L*G!w6|x{SwW@bvqqQ4FnWMp1?xf$JdtMUh1;JWpivlAPbF2Aw%me=Be|Z{!Huh2+l0RtUik zL`diDS%NyG!)qhkxQn%Fus@iv#`Fu+)ZV?-I+trg`NNV*$h-jdMkc`x`I2ANM?}4t z6OX?YNpvPb>WIw%9z0ye-Zl$6`Uz%L=`kZ}8_&HJ|HjC$PFl~)!L&R^@rrD#J$eHHkapnyHQuH;^_*8^u zHLj>eP>WYEdtI^klJv1Qzcpab{{mM?wLqt}E7~G)-{U3VG%Jcy98bJS1n&_}|&ovW4??+Qs+qx(12>9IxTO-lZp8j5M%Ixa7lI)lBaIv_% z*u)KCN*dWu@DszS1fbxeQ*VowFg|OsrUED5@@goenr}=LZ%SqykXxaBCkpawQgw3h zsTCP0QSeEERk(B-zZbNIpg@08W0@v2BVS#KrE5bT`2 zB4xy?8zmFLzDW-D6XWl~V-2+FSLyJ;HK+AKJ6~Z<0xwelc>T=_rCX0Vvx07U!0Jp>a#ION=%)nk=Sl*Wh zHIwVn8-;yTA-ltLE!_Q11Gj$ph1L5g1?;QHwq2vPugxG(T)f>6>{%W=v`)vTlN>eA zBNsA;YSc8VTfigQ?fMEC>VSy7q&ASz^kk%-OS^^r2>C)#lTw>=$n*`;(;yqs&{T%$Zo zZdjduw{gpK$TYulU&owZ&UvgP+!X(up#f!=KD2ZVU*_T}c3kKeK<8G1%HF%h!Qjyv2U8BA3v56GPmULdx5JY!Rvi-W&RQMGtj zKimVQ?S{k)ViwN6+8|CCHn-hx2}g{7!lt1|rTqwQWC9tJ+#EyD>KN*qOE)5lax~QN zw05wNdZRdhyU_ zi*e?T#CuAJE*00Wf%8xCd-N!R{!A18SjaK#Q_=48pkCh85OKqL_CAtSnhg3PD)V|b z19Fvk=|dweM4qUgeAom{k@-E<&Cf{ju)+BnwK34gB)k4&w9mRQjQG=dAr2Fyh@i>} zRK&|nBx|Y|XR7!JoCt1)ROGHapG?i;MeC!e_s3mrsY`0RF}3SCgGr9sGP@?UU`gj- zWq&^C%-gBow1VhAgi@^M@*3efrA~vpHom9!?QFug?L#*aEn=}GEaoz^)4KtZfuA`A z``V131-Zl!b_BF8)q(2cQt22-(jjnBO*p5(tfZJYyj$@J8wF#PzzQBlu*uk5Zw$>u zGF||z5@Jw{yVa9>!%_nbVTosq;}zZiYGs=S%pTjjWIwoq3ba@TJC=K@M3K(LmuuFV zPbxHEi4yYcJz1c5BJyHT)3wagaUD(3Y+E%)>YQ~J9QR?L?zp(>Z=1?}!? zk9Dki_ijnVVMJ1B?WSnxao7W9_K+2R-Xq|T;Ptz9beVhO?|QD}L-&tkQUa{ZL{QFM z4ZIAwF8XE=44GgKpWo_9kJNPM00#veI5*LcdCU`{FcQz@o-gbV_A05u)A~)4AHXi3 zN$FsVaT!42F(BL>FJ1R-D5^50vHB-BDzk|>7Pf`^KeHPajTt;*K#N+x>sbusA0&5B zk}oRKF&db*{Wc!#v6&tVjZEO%i3KoC^@IPT(r7cX#;8*s^bi4{F)sFWK~e_fq=^-T z>v2e(_tEFhYKWr^4Kwgodl^;jA^7u8wfX@8m}WW0wg!!u;MErCQ*nZ-T_`^%A5cR5 z`3d}K?jsd$f$r^(!xdYfv;g|M90foK8`A99FDZ7#Clw02rQ@iNRdU~ADine4*2!b0 z@+Rj+iyO>(bOhpbj&AjAa?au3}2j-BDQfR65GYY_}DYzR^s;}25ceDivqn*nAC!^3gyT{h#$^oK1@bXfWj&x%u03ZF{T(4BI)@~ zx3hIh8b7%nlt0yb0t0cocQzPPYs@3rPM#u1T;O+-Fs!PbQ6sry@DxGl8L6?;ZfG&FYx2mZ`a|B_yKqdYQc>;R+?k5ZQO*Ee_j)iiH^gt<5jDW$bM zjhf=UPPMx?nMe_o;hUAjRRotrnZ$k@0?gedAzE`fJGkyN8oailM+r_WjVE*@YR0xU z0P(%|#Wa!2ZzX!@ah#%Gt;%7gp&IMx%pRPdeH!Rsh2!Syp*!oTaZUZ@{*-Dw;yFe_ zlM8prxZ2R(L_DHq#h8!i6`27O69ec>&Zs0^c9MU<2{UeZOaX_ZJi0W?3!Ru>x|)MswC>XIrvDH!}74?R>gqi=id_s1`$n{%vqv&m2A8nld(>VH(&cB zrjcSkao5UGh~_GZHUG_FaQci*6rLL{cu|cD&yaau`FEb{qcfs=4BB35Q1t-#2hofE zu^FJ1$#O~IzO?oBaWQB|ScgNl-44JD+z9)lnD-qvH1E_dPy1;NKynM|?rwDLlT7`W z-ed%`%?I%1Oal$x!-R>1urUfubP6bJjd3#XpNIYmP)|O^4xZXAC&RI*!__%JusfWI|q~9>!qoP@6_Yzr;ejx9?-n;F+QBRnIM-Pf!@XlN9!P^JK9E#g;t%0EllwQ#2xG@u@uypR2Y;$YwAmp- zev5U073+b;2w#EuX2mLi990>ABYoXRs{f6aNV^`xrjvKeNru|f_{+=W0{T9s-pl{- zuW@-@$bJ9BQlyNSxZz*S0{&#AfNafIUNy zqvTG6)uw@R$@{8)ruzzPkk%ExT8_oB#yMW=i@Yf9WFx|F2=?emp|O!^yRPF`_A(w< z&3tZ@Jh!y&L*JCHe>>6}OsS*nXsza(S(9F$-TNPw{%^83D=;VFf6Y~;7F4bx@15U9 zNVRCu+e9vOUE2-(aTZP&(7uBc^gpaZ>CjDEM~oxAK4i{S`+wPi6*4daIdRa##_O2v z)lHYHP}CQf)UbuHEgn1dj-ghIyT0&Hx4gcc|suiW?Z2v0Rsf#!b zVR_*DDD=_8;e|MK_26malz4rXaD*VJO-t9~SD*Iyl~}WB1QP`j@x+>Z&>Qdi5o374 zH+D<8`wVSmi5Fd zRw}6r-p*1+aNOM=px|JtLxC?j7|)1aHO-dkHn6GY=cA@5kBUN(zE5S7sI+#RXXG8_2zzzL`cv7$U-l6 zHqg!pI~^8{e#!x^^>=hM-sTv)W+-Ctj1CX_Nl6{7%L4Dy#+nqXWC$a|RBX{Bnng}1 zHqGZ$NQYuPr*(QYDM5}PkvM^-GJejXoX0;3M*LMTM?@2Feu4f}jllSmWReQHDpRy& z&bjY=DHIv{Llnq8V4)_$I#jd={t6Ye^>WB~#fu7kv5#iMeagp&W{l`-(7!*(i7P&2 zKi1h3%D{vTmg1IbJvxW%vN+RJeVjy5KAw<0G8`;b9oX5>IoDp&UuVT|i&IDRrXG5}J4t6;gN0(t_hJ@wo zQ)w#Zv42Y~4ps-@M>#=&PkYg5;(I(+7ppg~8$N&qWNc}T70hB5`t9Yxg0Sg99(!!w zEp+1NXU8b-zCTCUFQ?IhR-l_O>#k(*Dcl7jFanWStiS3Ag+TsRpwA0#$ zXp&vJE;LaSxpyGtHagx?hZ>-@|1qzC5uLGhl&ro1x^q4wx~IR3eiM&qbX!p716^<7Ablu*2^C`?{dd`!9V*qR z{W^gal&0OV<8Rc91CeyG&d6f^q1?c%6BLcRc$qjZJyr#P^IKKc>V9+l)XR|jsW4tdloJP6A#i(ANHqB z&?B4)R>mf`}BhM zz2#t$0;~aJPc^`G48cq-Y;k1~Fy^uWO8M?ymXXmK@^Dtb!fk9EaW(>UICAftTOvt~ z*pRP7+WL7D<29GACkW)!wRdl*!-gc*wW@aGhS-?%suVU>&Uahy}3;;+_%%gWjjd8sE)3`h|%Y{=!ERJn(8SkZV&(J=)XW z1(z|IkbjO7O)siMUB|Ny1lL1u`xK_gEE%2@>Eaf>K1!Gpp9TA%zEXmKK8L711?pGE76vK=_`w_EfTdlwd@CL3*$i~UhbWbZQsJblu~b*T zsA0e4hT#+T86Uo00m15JXM?C@;e^p=YD^}c1DuVR8p99*Q4#xnSf+T5zo3V2P*dhZ z0hBLCjzfxG8$=MmXQ%>yTvTubX%KwrsGj1nUfFg@gK&Lfdd1}6Rc^hYUl+}orJLP_ z1pXDEk#KIVS)V;WPEY!xWNnRxLl#X+-_sbF4|4XVAYQpv}Y5r6lQ7;_o zj)85g=l0TLl6ZtHxJsupS#J$Jk30TBI^QX3wi#; zl2>uxm4Ni{OoU8#C>!pkNxIm=!0aHPAISH2-Rz~?`qZR|X$2@H>j`6PFb3;;Oy2zc zpZIN=?b~RsNi08@*Wq$3;+qxg@e5up&WUz=l$9sqrOw7tAOHNEnhMmHN+i78gOwNJi?dNf%J`72dK&xlNT( zSq&XT(xg!e0s+J9Y4jg!*H7D=kXrt=Z5DNSu!=3J*9ME`*st^2W+F$?p?B#~>l^Z! zPdmNHl-7pUqceke4@S&WQZfUqapzJ@d5sFR?(LH2TVFxTj7w{1A)o`i==?h)D6qMf z{;*mDKH_hq=cu+=V_q04<*yTnm7;kM%;}^B&hpO9|7o3J4 zbsu5h7x!UT1Pp^RDAlJb`3^fHR{9Er3NB!{%RD5GE;hW^YM` zzx0gRH4(XqRm>z4lE+e>(eF}mL^1)%ei7tw0YD6NDYl`&5p+JdVzM4WnIx^&ZKt1o zs*5aQHsx+qZgMId+N;phZ?dH-xUG@5js=IP9zwm^-!n9SH?tK7COO=`9q$-n%3rh#>{ zBWJ}&{lez5dOY#qKQYhYgB>)oFts0XE;f7jYKMzN_GNKZyr@=>$6s`>)#BQI2rd&<(C`YMX3z=ORYv-&jg$TlAr1%xywMRASxjph3mjsY$0`PcVn&p{%u(Wbi6^tJ>Ry090fJdN41vEz|yMKd%^&XIIJ$bB7nwt61Ck*wXfCSxY%#q@AHWOFUj$(&(=D zR5r~ruXN446;)|d^x}GIE1|ZT)G3Yw2$EDrNm5#Cqe#&$j{u9$i`^dV5i({<>SIP) zXFA|R=!bf$SGPdSXKfUilQB=*5L%%l%L-@bZU<;XxMjrkct|PVDi>|#bro1C=G!1> z;MB>DLlk*sBZymj=FFMCyiox|Y!6v#ck+adeT_{FW&#y?v_>CrNV0v?MJ(B;`mr%7 zx+umEBMS_;34oe{^raD@i;t1$_iuXV!De$X$x^#~t3o2fEqRKV+BXbTA9`EQ5()xE z$O(2_fh7!IRASMHv$C{xG^*26vG#l{SpCs%`;L>|SK=E3p>RZ0HAMi<%1I&lAAJpF zEdc~s-(&_ecNzTE)s(3-VAL%C7hNr=fs=|4uQ8(u&~vJXpt}C_K^`J*aZ3p-;7-#^ zhlVM}W0z!5%jO_{=MF}KmUGsu{9H_5aK-#6vabKs+T;0U+uE)@)}u412t?zOB3!I# z8(~4I zgqAgl8}4J$$Q>{&2NL=+YTtdlvLga(I2a5lt7C&HVJ%AW1#DCtI@6$t!&F-Fpb=BS|O0oSa^W` zw5)$T03&=s#pcdoVo!@n1e6wnNICxSs|IuS*Wz2PSKI+JyvtE;&9?WO$SMy!I38MG=Yt3!Z0x5K~;aGPeFd~jc{ICXh>cRj|-$6}?2 z4oI|_B9=l-H9tuDY(f$`ezG_8g?S$0?sc~aTf*I1VSwcUPA90Rto$c*?>bNs0 z?j-V*kYAAlf}mOAi2VOWUt~!~P6IYVR+O~J3G`xG=0r_hvzNgxpY<aV6lo|Slo z|K_}cjd)vZWRED#?zFWX-j7p6%F8qC_JXd^6vp=R@ecIaf<+=i%$4IM!c)NhByAA2 zS$>B!tysbT*dRKE7Cy$@@HED40T6vp*H6G4l-3_5;VvT$H(Jk7(8idJUJKN;-+&7k zxMlq5wR}P6qo#uT(#nbr8FPV~@z&`hf2RD%AN`$ii@F29aUzQX^x7D_*iUD?YRJHq z;e$QYiTZa6gE)f-@+q_)f=9SLcR=iuR`<9#bvZN>i-YS=#$Dd4xb+?r*q_SuW1mBK zp{z60WELc$H3VEq>Ia>BPM{#916p!fc{0e5B?NS2s9S9`q8^27kV==(lIibKtG{)i ze|RHp*?B$<{|~yDtZG|OU$u#~E&hgqa~YsY`LHi*&=VG_K|dPDJkcxD&~pHssH2Jm zOpR0S%tQDa1OI&oZ38ipv0ib+gmX-S+ zSzSOm)ZNXBOw+hEasJ`%$2_?L$1~W6-Y-G%)DDT+ev-~N#Y)Hl7~~UVvSa7o19V%N zo~ICvn5<MMSHRsHP=TRGWeIEO=cDhPGia9qp{Pd<9$kMi3eX*ihvW5%!2*I zbe;&1)UE{L7ldvG;>qR}LHrtDKN!}_Z)yJE8z=gRP0)5aq_FFiB7RPyVae zceQ^6G|;B?&IU7Q1F}K(HT-4(snK)6?oV|>{PU8DFpOcf!Th1aUm!G%AN`Yw*cy}% zK8_Bej;V?YKIleo>)~vbPwpgzU3s`B`0e0$ykKHsjNEfO-H?5|Vf-xq*J;D=30mP$ z&nbvjVwp@ln_+UMut4x1<1VPHg>4l-V+$hRvB%NH#x!>P~zNz|$<5+%mbj;*NdoWhGmYTc=om?0+%9lL& z^8Hp3-j;`vRuBX2f%>wQV_hhWxtE?WrC5{-0t-ab(gAq|gQomz340ge;kkwlX(*p1 zW!B?#aOl9LbC03s&bo#@o={RNa=Tg1v%w&}c~tEd<^2pNsBbOOE6c@)W`L25wz4JH zsdBa&s>|@mBLr`GD^1IfatI;f`9ozC*sxU-Q4{#IEpGx@FW%DMJLx1Un0#46_& zFmsMsa7)rILvxJL;2AWUrbu{xT3qMo-&H>WypS%@*mg?{pel52Stq-rEf!yJ_xoB* zYZ>wuLFukMF{*t_f_cm0#y{=W|K)4GO&T6ocf(MNY34`j@XbL#nMl@OWfGy8Ss5Ag zE;2ac);5DU!cesoStwgVKGD&mmW+JR&`r-2qD_wrggnH2Nb#S*R?z(@8VHMDbD&PF zhFIQUIbe3Go3{%5$bxaYqaNcL< z{cI`d{~lMf(@Wf;2+TC+C+#LX4;JPn zpf@WX_nd_RFN?QBCAtM=s1~)*%jt)|jGa zz_{m$?&4>ROF{@pKI)a+Vq|ogXLkHJFQmvG)+1`y?1aua2NE)9ka3y#L8?e7kGq(u z&ba1dgtCV?S({&x#DOlUjSlA5oj$*mm|nx{)1(_DUY#$Jv_u@h2C=3%%u?fluT_s$ zyQ;K;O$8IoIu%Vo&(EfO#s%4pi_ZLm7*+mqY4sU463`KJsO)YMT!3Z= zF9-Th&}U0I;r}A7%>x`JWS^N)vvYab-55CJb?_9JXCqTZy> z^~K(HJplF&J6|%}X6Oe9o@j=reo?hUrvx$CRLZ`ovgHPfPpNZH(&U*}ywj$@SD)X8 z%+T1=(%Gn&C8+MRZ>+$K2}*{)iECyyLu3h3v*-fS@=fVAMomq?RoOE;DC4WEfxlQ&(q%QVt4wyIOaU<2fBVbuYvR|-0a zKqa!SSrZVa+?-~MS>+3BPkOR*sRMZ@9MbtYm6TR>`-|-QUo$Ln z4tsKAvjbXOp{P%!-c5d*xiPG%$2Rfv98qYyB68q(i3jCJQ{O2jLkZQJx4TDs&d%Ta z?T_R=vYbL#miT5F(u*O~Z>25{w+`&oDBjY~;2*WSE*hH)&AF?hX)T&)U#QvZaJR*M zr}YRZ{GnLsmsVKmjGj6nKqqn$Q`^=#Ji0s)?iJL0`|1WJNm_A=y(EaGnOBuKGi1G9 zm0x#Y>!dN7BEn@~OzulbGd!CcT5q{lJcz-klpRCW;W`z=y3R)XICMH)-YgZ+QO8jy zt?9R3YRsxkge3EbH%rs`8pG{zziS!+pAO_?*K4%3rE7;ZzNAceZ4)9R5{~;-pV#zN zu&q%zM)gq$dyF!eo5ulAzwW@dMNfpwb!e^alis*?-kgS9LpiLlhgQ?%g}!`PoYq64 zLmk4;1>a7^BU}DH>SHnK*wV!kC{GhT+_slK0rNyeyu&K+*5`_fFZkm)I|$8R^i6iV zS^I!JY1F6_-D`XcXmdM0!uhr?awq6!Jt#8pk=fAk+9;td$Jo#0AuwjeM{KXaWS@fgje*dB&q=L&c zz}l90u|094h3wyAx{#p@;rlIiraUV~+l_z6Jb$1_+6jlsm_Z!je0@)K__s3OmnD*E z#3~Fz<41<-cBor>>Q{-{G>Mw>ryNH7>$6NYr?DsJB^Q{hod=hb!Fwp^woF$d3KmW3 z^S?G9T`128NjR+d(fomWKhpIHM~VD7O{_!UnmVVDhc`O^kBF^UEp(c$XK3+f6vixX zNQj-$sXz(O7cAzs0qR0Km8a%fC+DhYwnBOAJ0c3VPJQGC_drWPTmP1RSZ-h^2=Lu6 z7~X-eAX}K5Ahl&yqhv+^)#V_3ucp}u-C1Ia{2Bn?nt>#u_hh?oeu6aY<^IM4ZZn07 zr2imr+@bdZB0{)Q%oq*wwO2jj5AP=bevCDpACGSEr6QBi$HkKGxKod zye6$%QDvGbz?Ay=0&E6kHfOm0{@*0Y4n+fzbT0$MoFGT&-o>gqf80{i`e!TEMA*>k zTGrU2a%plI3d2C@*0>zz(Ow`%@exkX4iMW)v?o$pOw9pI2SXw&jUK4ASZ2G+%N|5a zVH5|nGZ5ZJQ6BxlvO-lBeEx${NKSbX0G<5|3GYYrjxQMbfO*N9DN#iQb$+-ReOS5Z z7nja#2+0c_!56H^HT{!-q;g|8a>}{mMx;+&D@fg%br{F1z#lQQBGpxY;{bUOQZ*X= z|DUiP&ZjRyy-<`t#-NxjeCk#&uiwAqE0OBH7ja}IHY|>6qP#0vXLAzAQ$J|tUje?- zvpV`={RU`f@$PiWDBX_nC~bxAxrnK8)Rv*6N@R^nT7Jh_-F5oM<9Yu;_6 zXC3K8hk%*ZnbD*pL6tp4quiOeg z#H7df)DpyLYp7OjKzm@8AGB)|Pcy&llmB78iS2CaFJI2m6IOmzET=>|oAX*K zhIFgI*-qlyM6r)UXy+x9*q|8jfrN>a>wN>}= zD7ImeLPXydU+Bz$#*RMNk+tNN_#_-R)XVu&Et$8YFmC`X2^|rQ=lU-Sd%CMR=nV}r zXJe#E*u;bvP|C6XFDH5n!@@&aqY*stwRu_Ya?&RzzdwBl}_-jy7NKoBm zCwPy0uSpdaPE)J!SL3}LUtbvOTbr(FvHu|ZRmw-dJi>c$%KU`;3%zNXG!dQ)&P5AS z&VDZQ0r3MfVswtRiNxO;%|(9EU*BQA6jt1u3|!!8=g1MP%|?oTUs{*HzCM18x7aNd z{6%!Rm6lf|)GYTprsv>34sg&%`v_Ak|Itl!bqI|@5n$Ja8*Td6=~@jA#MW$%)09uX zAA8ykyA2q_ZT@@e=MRv3mr9)C78?D65Sy&23v^3Jmy5Yoh7C|h3wwPJci=)}#4{XO zSO??GXp{2r!G=B(j^c@FH(P@D191fk?GzUSapEyj}b}#<>xdw zIDQ(Xd_7X{uLXAv$~QW5)vW~sIPM{7KA--z*>&7*isxHD8q|}5DYJhWJqsi;OKz^A zwm)2sKrev!A()C76VULPT5@*;N$L~rM$@6s76E??K)43%Sk{?8Y8?8aI<%Po9cAI) zt-}wtEysUb@UWd+t;--mWI{wizCguLQ!vH!OC*-Ot?nE^M~qNWHtqvjYv?cHxOuamYg}n78Vk<_;fbbb0-0Q<}~0yR~{tV z^)8e>gN{$AfL#V$ohA1D&`DwBbr+r$o>vCusv%!TlJT%6yG%Cov5buT$rCUm4w-_l zJ|r~0>2&W`vgi{&CeRiI( zzmBY&24u*_kxg&oSYaee-1%hK=T*yYL9JXChr5jXcjvgABOZMO2gpin$}mHu#=Q*A+Wa5JR8Ru8M-CQzje z6K7;8I^f6+l+GuPL~F)NzV=8<5Q?x96KHLc?2kM-*cI>8=yOJXll}9xI}ue_^y6R^ zlCXPir^N3Nw1njYOnkD^1YG#`OF%{k*Q>R&WK>YZ{Px-eN|_vhD7j-DLKF@>>_U{{ z>=eoe53QS9?Vn!w8AQgf<~?1@{Oc_1o|8x_t*1`U?ui9_<=Wk$)K`*F&otFva_Hdsf$m))7j#Qn5+*kk1aYQ8p6*5;z&ap!R3wiAfhvwn9{Xl zptR;YHFA6V7ki4Vt#3!Gw)Qb4!xDO{>a~~>3<%cGgAC=B< zsxaP*gEO?^6@dtOC8MNN%|u)fbS~ACbb*;EG0-s<8;+xbUo-#`iNn&&ffo8CCzXR7 zJMzNrKG|bxRJ2E~tQ{8DM;%DJzBQEv=sjC61{SLzB_~Qi#CKJtk6Ktg>XURaSR2tti7aS$zY77sLK?89XjWQJ3_wMt1(MR`XwAVZ^*gh)f@xOg>4Lh?1>~>j^FU zLl8ilg4u_VFpNm|aQqxPgUwCWN9;`bV1UOK(Qkv52&N)r?<_>9J+wF0H<&4f{Xdhc zgfs*}-^3z->`1D{LV-ZZ;y8mbntOE%qB_F*pj)GI`@ftAkFFMxXIbHQsKp*I!hk}m zG#8jV#`Y!xDWhD+WPk)Y^Q&2ftR4G%T_u@IjYz;)sVn32iIDJ*TWO{=AURXvjVkqO z+H{WhQ9P|ZaK_!?+!Qb9ZmIG5uB2U^0$;yuSyg>A$0;kED(_FbjGKB=PcPrmpQhA| zijd#u0=wVb_VEFp36MO3m!~kTLXtq+t|N|_X9qvGMAtf2JeWtp*>eeFg7k`p8KA6X zrzVR-)s;#EK88$+q=S)@7c(X^k_ng>toifWS#^B>s}CSt{rIge&)SJN8a00Mq!p(- zr{^J^-_rMVKoFey)o>bDR#pZ!is#J1zXdeK zgjP!dN5$e;%?2p*@C-H3*fUSFHm^P|Xui)7nqzRog+21ckwokD10Fu{6Z}=NE zCG4)h;E-3jEO;SsXwGpVN`Yxq6F~Xn8d99Kh8_?(Q7Zla(hfrFt<{~s zkI#T*;cS|>Y$rU*85XFCw(J;li85d2JEsaTG@3`xB%v9{kv; zxL}A6tdFI3)b|3QmD!-R(3X}6l0`>^9CU0>TBugeaS`AF-m;Sy^okomAJ7(0w*Ro% zWOUWkqP4%)`w(cd0l8T9%nZR*1Tn7>iu)>zF?wDH+_Ley8bJIq_!3kUShrmmV-%%F zRx@Zm6twlqNC9wbkS%wP-fBjq;N4@dh-ARb(f!K(Kgxe#;Tc6#lBFzg7~|fslO2RP z%{%NxYDciG8zT*C!rd|NS+8XXU@bC2QT+>#ClobDfDfx$YnJmIM<0m3eRSADb#)?d z-P6O33dYLqVacc@L%H+BUup$X{4QulBSzQX3xTVa*hq%d)oPk>Jft;W@N&xPx+*$~ zGmS!h5;*l7o_{Al5O3gTF*07V{12{i8k?7HXWWeECL=rri>xrR;lUI%`zaeB{yWVo z+UWkyiBUgxy-2QNdo|xyD9)N0$T8N@9|I{&_v(|FYGUX&~0tOA5ea)+FHvI zxX}kBJY4`zEJ4+m^Lr=P5xio-F=P+rCj_Xm=BGNu{5YU`UM-q;Iap5$nhEY>SidU#4>Bl6MJ<$ z3;~JmrV8bBfZ6MgW7DYjnAu4@hgBWpZb0Toj&}TY@VuZ#xrlpga?ls9(mCuN?LRX| z=4;=NkSodxF;*4tjVj1*_qo{&sQFr;D2Ysx8=(Em-c7U#JJ31UIl%?vOOL~M7Yvua zMx*@m@^n$A69;eG2POUa26GB)Dc!K+^&Cl*@;SZ2fWKe*Bo zzz2q_Het8Arv5WY@hw2p?|s?)0CEDlN|}#8NXK+pn@0KVJSV6A@5P!0Ak|NW;IecG z5|7YJ9xzx6s;9~|T>UbPe^7w^Q0q^duK;i0r&+a~KQ;dLKUzkE7Rwb#21Y`W3GQt- z!y|sR*O$QfgtJB4Q&mZ^NJ$N^YS~EI17_TbPR1#@O6pi>?AWIC7;FZHUa*3O=PS;; z=4$hgVs~AO8YxfSD@iJ_s{rn!UQoL|GB?(NO^tl3v86O$9!)NHkg%y41mz* zuC^1&Tt*sWEqR(3IvoTM?x zyEGexBu#f?Q7)rOF1fzTt-1?^(v_&h=XBWvaTStr>l>j^k?-v@4A*H|pBRDKlOryY zK(<1_KMkEiyv?ICRMlSepT%{rG}Ul^qqt~f-V{`%xFtXP?@`&Dw8ZH7NfV~qKIelZ)Vt_he_nW&HxfNtJrSaXgCj#~dD`YDR!#Wi5s$4@cg4{fxb1UG=6Bs= z;IIgdz>Evay}W={1JbCPWL-8)LR{$5%!UZMUEa+2L4#|?$y%^4qg=+uoT)hAE+Un2 z9eiI_=kL7ik{fH<+Y;Txk5~(n5zRZt=J8}I?uSrw4X-K7akrro@ziWReN*;@f?9$ZQ=4&qLTy0n2)l#rt+;R* zb}hiXz!9;qw<0$o$f})$x8#(+Bd#7d6>t?;$7sUM4~Du`jY6;~PpiUHi^)Xhs{>wT z9!NKaAOS91V3jxyAffZj*?y0=vj1_NB3&A-IyC2Hny-gK`N*EO;~y>?k{FdTL4Ftn zmM-}3-25#Q`Rg;8<1K2`P1H>xS?%GAY_8JtZ&zff&|26%0$+b$)$dQ^wDaj-@bE0K zq7;VR`i}jXjZ@ydZjAW-iIZ-Jb<;=#z}FI)8@Y1!DY^p?KK4rfmEs9hMcsbWU2OTSPhRCLKgNiACMSBns*p3Oh^hg%^BQC=$U=ZJGmXnU^ z<~W;m8kN}m>~{x6ip%WD9>G)yV`!?u*#fUy*?PTVl6J%a--1rls4stDW57%*P$~s;v~Fv^4MqRm1~RiT7z__T2m3pdm_mJ6DZ`9~ z7op@-z@K=H0B=y*HvG8s&;t##xY*wy5VX|Y3FM&nmZ4TiFzp*}X`Z(Bznbf-7W-i8k%o!O~U-~5@mpu}eg6BbV z^MfiT%I~}+k=sUh4&OBN>R`iGm64e94c=vbC{wW2SsaoGl|UX(>;nYUkhynCo-q_s z&Yv4D3K-x!Oc& zR(qdkhgs~Idg{2^RIbam);>cY?3%%H%Xq)@h-`E;Nvz3Ss^d&72M3)| zb2KFgM(`i@Tp2V>r%#1}b&_puDYv;$yy_SWfbf)_Tb2HW^IDBYeN8P$D~V0m4dk8J z6GzZf7<_`IPy~0uow2UhjcP&UJvgC$hPY*E8{e+vgoM{fTnJ06?tzlIWq^d2Netv5 zP+dKI;n-eGw!2b4YF6+HmzY(7>}ER&n&i~Df1|*rAwXv{XT8+C{UJgH6+(wl}P;D@Vn3VK-G0z!10eQ^Z|I=b$r(Qqw6sL zZ$7M+g0Qx)IrGVWA==N$|H+7JcgCPHY8jjE>vdf~8~>0}v`#1&8>``sDPNAbrc!V* z{93Kb=eL*xL^=JjxeERf_ArY0kL(fLhQfJT=AsInn9vcfPzWI&b1IZRISOB{T}?3GTwtuI4o~Eh z9Nk{i;vSPp0qG(}RZiAO!KpEogOxX0nZ#3~L`YT>@N`!@wDZ+i*n7KVVFlFWc*s)e%Jve zcDh1Wwd-stRJbVg$6{E_ybj9$;9*F^jXMB&?!a(WR?0*0N5fKbpE}qrk8ia3rx?nd zF2Xk7JD56~{PuyNupX5y$XAig;GEV7{On<6+7dBC7uCIylh?%bLMgq^zEN44xke%L zo(nTPR*X52Pz1Yx>q-4euTgjuLWHT{+)j1wbbP>mVQ*)h?y^%+wG_(v6rEe>J6PC2 zrQCnZHhO0Ca$ZR&exm}GWdqkw!k8UgS6|+9fR{imA|~#dxap%h2+{6!fFRcTvPMj6 zcvqT&UZk?kE0mA33PJ80)n-HaqwAyAllQ>jOopk@%n0B`*YaO^I@Aa3yx(FHumBWG z59Ho3B3tL4$Q=Ptck$wfvc6fdl_x#ov|*SraDfJEUJu&*XYJPPI8o=XvAy5-WLjBG zR|1CxrX0^6fSy3aFnf&IvhC>$K^uzpTcFcKy@xyyGs!$}h0ul-#PF~MZUF;aPe&3V z@J3~LNh{G8%ch`?varwFyjX|^NZ}drC5ItLA-Sh6slF{Cb7h!OWlTB&wyf4iq_9Y> zH+AOHsNe=jf3!G`Lnn4^h83sDOA~pfFqI->yYgSuS*VsIAic`d6$nnk`Qz-xp;L^2 z%))r}jL%HWfWYUKej&oS%tw@VR@s3<%Q@Rr_m~uX@B8|D<+*+Skh+!a-Vo=8_If{~ zJGfOD|Mmq<8bNSXXQ-#8{pn{PuQ)QF^gU%MuGfvTw_3A{pWvFoZ(P4S;h2gqVd z7&Co=iiyFKd)^zK6zI^hZ`4i-2$@QdPF+g5-=#i~6hM~a2!R^#1|K!~<(^Enbt+I0 zm%as6T|_u5Pu4N^x605I_*danr9EfJiCPGfFHB{H8i1Q|vj&nS_3z0Qq(VMdbmKi* z&YjYC0K>(c_G5s0I|u_GtjeXa_u7tL6wCliv9eajY26Pg5iY6KYoFQMJ*Uki0{RAN z;m8-d1VK5?>{Oxs9d+9@yIZeXM+A|jfpZD@^ZkbfOK{?AvjF36$&HuhUfm8b7HHTg zf?j*!&9otzoyP!2fXd zA_52}7NxbP<0Mj^LhxslmT8BPS}@x4u>r+^=-q$qBlS#vYX3UhDcT$Tr1VZ0D1rAD z6sAoh?k)%86z)#pYWzVEYWCtnCS@P!p3?;`e(OA?F`4>I64x#V-&y>@QVFtc)nv z@&E~O2+87?MggV<*G~=K-dN5XCf48k zCO0-T@tq$<8ta*j%yp_!u0~kqd#mMtAKvIG=w8YQ`RjtTMYG%^xd<@si`8%46iXvD zA~ZQ?Eo(}(L_AqjLUxFaYtP-J0T@|-ki5jTIdLg_+-xQ?luQl^JovigdA||wGeQ)> z7tOTV8P`R`X3uMyy8~FTD@B9o0tXkHhR!&ua?APKd>g$S%aIrhI})-MX!R}^2WYz~ zMxvWn;ioMQwQD1u?HgU5WQ*GxDkZG_AUnaD1EvJq&G19ij;?z5+4SFXDoP9cpBsvRxb2etFtT^k^^o3)1^Ri=|(_t4a_ZQ=A9?beNBtk zVFp6vNoocCIK!GJp$+$aG!88R60q*GpM(Vw2}g=j-#N%7fwtpWWPAv_vcrxZA=kqK zF)(`*#3?3K~ zQBcWcnVMxk4ENB$3qlFfp*Q(A0B=NHDE6MN&*btD@R3m2)A#=Fk{G-5X--Rov=D;m z8`|ZKBpKv8=#nmS^UBZp4{;2#5CD~9ClCS-eI@R7$Wrbapu>G1XQZ#kTI?Bo8mw;9>&5+ygN5y%Y$#0RmQm%{K>yX%0 zpRxfX;P8?0`+U3ZoFrBvn3`xNgZx9N0Ja@nFawyBEPM;R%(yvhhk0ke=SqUIkcWuvxA{qOK`SdL$L;akH9TAAN< zWk{*Vnz1`-3J7!yGAx7_lH30a9GU+)q2nZm{@9;)p0|d16DpWrGErYVm zFk7Juw(T8Mh`XwVvh4hx;ZpG@M=fWmP;J~;LA*&KdmrUNKh0)U_sUTD3Ta?T}F zTLT!Za+-DSN$47in|HRN z?A&0|mSHKnzaD$Ud3FwIcO)YHrd{EG$m@FiJy_Mr&IOS`C<1A$L+Jm0!t z*{)@p@{mq$!d8Q0Ze+qi1Y_K>DREY^6LmN=<=Z-ZicC^Jp#Y^&Bt7FiSBphzUQ~Cs z(hQ&qccDqoi2 zl8*}GL=YV_uBBL_6}j6ZPwA3{d0F$8m5V>)|7}@pHNCj{FD($0dqv!TJM2Uw4?#dV z>qVzKcIO&;-?U_|tvWNpFLdl3%$q;?^gK`CKVpVo4ykP$Oz@%yo&LR2CmBP=bZm}Y z|Aa_Fdy|B|q~EW;|Nh}CisG815!a2_20Af9c*on*n;!S!wT47Tja2Rkot53#2i92V z!v9M5+X}n#Is^xp>*PJTE@`FjEC98!@JljW4Zx{}LxYp7nl}C7!Yy4c>SvD%Rk7_w z5a8>Zc}cBRX4R!IpsBV>BY&b#qQ)`>k57}y)?)ZF>fAj!Zr(nYj%qm1UY7PQ`?!sA zoOCaGBRV-ro$m{M=;Qhr2R7J)^h??HZW)Bp#AGsq;p-?7Qf3X_D$3lKKqmpNrO4C0O6gKkfDICBtS^z$bpYpsbKZoB5GcvToBsiWs2PVsW0t0s3V?XIOG4Xkjcx z9CQjs!CdQ5z$1V6v3I?l4n61Bns1!hXx{^qAA1&+aKjP8)05Q+6JSktJqTLIG6c%hJgm4V zaBZnii`PcWsP1(o+gREeeC0(7kf-%|oos+QnK`RghX+br_L=hwJ6s_txX<|sm*E;2 z(2O1v>6TIw;hyavx6ePKd!V^sQh1k$F=lBArd@YO5aDh{e_#-RF9E~BM&pV!e=Uu%4Moe^7WLK+T%58V7h{n@KwftXT8b>A z#8B+VfCpR-p}w$$P2*Gc&x?B&%}O&~u; zvqg2pNPq3~c7BW3e^c+kBHzk)8LzioTEaQpicwb^msOGJ3G7GP*W@~p>zd+v(I#fr zL@hM8-H*~wnA{Px)dNA<6#+hCm{%Uk`NYup_@%g$0V5&O+LbPffxCI)`j@|!0ZB33S5LjOB zttN5l4fKXG%6Z?_zlJVbXJ-R!#_wSLHrugm70Z&VH$%MT)vj6SNU`jw#&Hnr;V!G` z(q8_%K@FfHyLE`<3*VqF>lj;XGLwl??;|$+Q}iPSY+?`u>LV}q*2Vp66zMCIL0}XV zE}0@}jn|;81{UnqBQ`ugTq}A*Np3_M0RLOjEvO{DK^lw0GTGAD0@XqEa;Y*;B8y(N zGcNmoj7-%w6a(@-8elb(LYIEc55A@)Jb;c_8^_10m94pv1{$w(t6mzHQ)iQanL zzt>_)K>})bd&z(tW5onVij>_YZh(2#1P%IO*bB*9FPU6?f>?x#lkZ1e-GEsA?@wh5B+BQL$^f3E-fI8y1n)&_Gx#s8w^6_CW5ZZ5Ht&OcSF zXx={PiW5eKENZ$~?PVPE`UKw-lVajdARV&PvVqCs&NgI>o;>MK^*c40K9;O#EkZf_ zD4?R&&^^C*xay8g=FCjqb31m0rTqX|)Zhi>CIvMNm3pRfBTJ(G!)6L(A~Vk0!&ok` zXL<+dBu~r(wVXO^aCACM_j_Ig)9jEYa_0gwcj!XkUhq}I&X$g(weeDCj#I_k$gecC zOVK6Y1%O-g?USG+_*Mnc-%cbRx_*8QzQn7NzjpKn-sRx7A4?X02Nm`6$8)JtfaDCi z@PTJ{VKJZ5Ic8~iuouTs7!_O%)`HuN67l`Zh~Rv$)>V!2ATT2Q|# z=vR|mHZQ~v{XW+Q%%s%9$jcn;fLUX@D_e~W>kOi2v*E2t98uKNb9lk7sTjz0fX_Di zm`5;0ZhzMSivHRRp{=|bxlG}L#Ci{LG00S7j%YBgJ8n?3gB#9Y3Epo<*Z$%m3Rzso zX#VSV>xvH-ipKUlGY`q|9>*W*HV;|49I(P9@=330sWN=R!aFeD2DT4Cr0;`j`359# z0y+botfhpP%*JbYGEq zh|$~Y=SMZ9dAIO%+?vhP^C9`N%HnV7t)UWc0{}RUtbidktXHo6hKd)5Zc}!7IsJ8 zvC^FZ0xt^GV)l^ycnMbU-hemUqloI;Ge z%p~m>`Fhm+4m$8%+D{2+i{-Fk+gL(^fkQ{;E!JUliR)?GGm&b(6bvSQioiPxt&qM( z!8x9WJwn_YIIBuc&mNp2e2>bD^WH2P%oL$Tc8N5uk;f(1lT%!m2oQS`8!Vh! z4eBP$eLRA;tSlI*r(aK|WmYciym)`STPJA(78 zL7N%N8uROm%osn;LYgpp4$V-u>lVPLu{?j`IE7Y&>ek04pbn;xdv#*7cc#Y`&?hyg zb^IH)Pm1YaZ_DR{30;z}63w{L=BweM8kM8aCfnlu;BvWd%KYBs)RR_b?av~o6J$1f zUQsld$v9JB=H-FJMCc<8)(HL$vI$Gy~ zNsH7gMW7mC*Bs{eNdUvn0VM*e@yA7Sv4IMm>DcxJ+eVteUJuyuMAtTG7`iu%bm*2( z1=9n!SfzA9Ba@pt1AU90dCVxAXQt{(pyc;TSm}8XWddw+v3_+@XS-+4flKssxtUx8 zXN4L!aYbfx+qEDAIXH6aI3m=Uio^$p&qGZRp(~-p3ju`_3B&2PG7`L{lVB=0uiP#< zMin$2ggj@WbBjptuSJq4c%v3E92)XoS2PpgUacH26$~AHX<_yVg+5l-lwuifI7Gk8 z)enKw)YpZj8{X^G?^*(UJY2s>t{q+7Q| z%%*AFKV0LL0_z^z`60aQ>vI%PYywmH=pNqoyuZi|fSJ`ml=N-}G3yLIHDoXGX zC-R%<%w7E-BO&^WKy#+7Z#hVer57G2X)x^(?FZn6-(=Hib_U+61fv4+Ow$uXJo_q) zXZ&NRr>s-3>uDc#sexgpzUD;PT=*6#3NWXpW^od;v|D$!yF(Q3&LZ$5hl5p-h4~zg zqd}69+rZ&Q7gtkkmw#-$D3^c3po}ZQv#YcJUN5zKi_o(emo#MjqVitYQY0y-jL??U zS-TLR;D&&BSoG0HuZKa~hmk4jqvb=xD5El-; z#pHIgShybk7pRX9ipnk!#Y21S(pGpQRtPU+1i1I)Eij^}PL~X>G4Gkqx0nXZ(pM9Z zV->X4?1fS7>{+zH*IMNzP6X9V!$9615iR%((p?Tu1;_$XZmISpL&>MPl$~74Mo6B} zvjjOm8J~Eq7IHr?!4un~BY61=-qwf5z)5DCc6HMM&OGu)J)Mk`R1YwBJ**fUcQcoupb=gEY4gau8e8? zp_vD&na6zTEINGZcc-}+j^;-DcvveqC)a7;(Ayj~&mM}gHoEB%%CUtMmTf&+ zp}+w1q+|FT0q*#YP7QJ}!hdJyPQ1$HeGGCE_)h?Co2UcL^~VQyEjR~ zEJSFGsDt1}aub5Yz_m1)!iL3Wq{qY&K|#}YUJyy#0H^F>9@OIm!|=z|yK{p4J5oT}WI0Ws zZ+teY^I;G{hWG_fz`6ntNJRW5o>uT7k7VV44kTz(ON$+d!ON3X{2A4p$+%ZgK{xIt zCousGj0(wtC*yle5{@)Y#lXS>ia_ZzVE-xVMPkiU0R(eE09o!u(H8hnjvcD(P(~Cs zz{WN?2C#YIG*&?y-x8<^FK1+o@kSGviqwLf<8Xkjr~#@yNUU>Oy9N(rEVTWri^b+! z3xvr21*%0Mk%%%5v@e>4<74t==i57ZAd7<7n}t@1OP zCSL7^r&$n{nkx(h115b3|AqFu@(tv>eV`VwFr0W~$)(^K#J59}O3Z;#YgA>MRza%k z`N_=n)ai`s{@M(`5-m?q$&GhEUCP20al zbShtD;%Cg42ubK>r8A?{9FePitYB@ulTM9nZRdHJ_W&gGIxGN%1xhvb{0pxC{ke0f zDB~}srgUaByg;i(fAni|(GVU4L~NP@48XY0Co<0lb9cvkD&$P0=s zGwhkQ6iGITKcD^MAXU@b%=h#x>DPo4 ziLwoCC|?qlE06g4Zqo%?*-Pe)&u&%#y08lPy!m^Qc$awrFL>rc&9y10HC2edlLn&- zAp{R#^!>@lyG^rtLJKQ**y7s zBQkGwu)4QT=`8|TrZx$R^uRL_t_i1`%o99|@i@4ir0@O-`Lj&@AL`J;-Hul*N>Uhu z#`jfE%S|l5h_)l&6PDj^3Fa=)6*dRMp`|2X{FrrdOY0eeLLr!68HC4nU2&dIG*R1r zpVRfvUC#RVwQ@B^R{~npQ%5LjbmRK918m1D#@!^YoylinQRWOnj5xgMP-V8bsnfJU z@a-nP0dshQQm|sP=V`IT3ehxQ^lcAKOqUXXwiRL2{nYzsoKbNS+`Ck4YOBD%?L|M= z7t^mTvW=O?6$vZ9nZefmu3S9gtxudgP?N?Au?iIRjgu2MgoU1H^C>{sTrIo#eZmXP z16wUz{cr0zOqsm`*~dK(FzOvLJ7~lt0+fvVhw-eR+tnRtThyVm`K+z;-q@e|On#8$6plN2yT(nTSnu#kB16&KK*Y4Vq zppYEzpcP4hqhvU0(n8pEA(aPAzm81aU)H!_-t*SE`h zpDeoodIrE;f=9*a*0^*9WQ`PNn}tlL#L@!!sj;{gQL%+q1R4DlWh)ap2KJba7A0QD zVS{fSS93}>pH3ctTjHZ`(H_{csZ&aV=BQuh(SCpM7Pm_Ro`6U0dqa&I z!^=Oa2N{(J>dbp8Zcg@mLrd?XE%VM~dEfai3VOKISZ&m3#ma3<(9||=Bh9~{g7g3N z+Xzy==-Ee}-^17?_tFysMHt%rH{Ls5DnPL(8qky7DtS>_FM7Xe_oaCnkgbJpv-Dqr zyps`Fwu_Ovo(UoO>lZ^jF|*H_-tEa*lzc#Mx)Xz)oj9Aq7wccM0w7GnBVIRI_mG%2 z&ghRSL$>y06P;K(?DN5Qz7#0(9*>Om!(T0p5P^Z!^YL?tKs?dx^|*l|kb%1zQ$!EK zjyAx~alZ19N}%lOK?joqxRIe zSezcM@k7u=#4Q)WY8cRT6{8s~D#UTprb)7nDmJ)ALI(C?0dEDHFF8!QOg9LVvlj$j z&UI}Z)mB=DTd3wT<`;F}2I7nPSGV6GAVt*dJJx(Rj8zWrV+>26Z-cBu*-TdI1#iJ^ zB0k%bH$;k}WF9ZLV39hr6CIe?I0>rPDD8DkfULj(R5n9yU;itCHYSX4w$V8O)YHYH z-BEo6Ty_`kl9Ia>CZ23zvZhN9(KAv9vq5K0Vmrp)w8P@z#-=(6GBM4|yZ>Qq%N1@V z>n^*kY zsV?20DGeVgp_dZj*Ww!WL7yD|{eHz3Y_39b2O%3m>*}^n#J*H1ng1*lH1ceZz(j_Y z9{2YK&f>FyHSyX)`&q-i#>HpTOX7yIbny2b;!R9O&Ai7ohmb@p#&M&E*1lBai#@9j zT9D&iT#i}@t`uXcg&+MT+U&4%w|hPbX4(9^Oao6l{EYb#&~SffjM=M}+%xoEFx>&!^u2Nw}W?)6<} zi0~5Gd5TcY*o|t`jdkVo^Hu{B0qN+mFV@$;6Z6Ea7m>N1Oaw2L4k zc)~G8mqmqe9{5J2(0IKB^`z}j>}U7L>0hQwOx9x!A) z>AMqVG&I=;D!O07$=dxT&&HG)Q7KEInu!ZOgg|~S=4DrgX&rXseb>jS-ekjI3CruD zQNQP@XZ7!Js5rwy**HHmttD%v>9lRSyY$%Hh3iH0XC%Dcn4tJej9S{loa?EzPYF7I z;#Ez}Xe>E7PwoI>!hCN`Igmc~uZs0}O)cx;%_BY;xJb=jdEjk0M#$giRm|0ozG^Dx z2%6DY$5^~D3Huabdn{N(E}<6jBI6C&JJ%~$^x&+TI&B4T+Jp_j?jc}8V2h8ZT(!$* zz2t3W1q3|UQ`i{`=YazU1i5h3W0Mcex#US`23ku^V5^`GH54Y_H__Hv)kyhqxC^Pp zrtR&%m=2YVbD{UQxVwXya8%^LodWQpPp!{)e@{FL}U!Qt{)QaNVKh_r^wo8U&E{No~KpZf7_O<|OF zy&L{=QajKf$(yfo51w}E4lk)hv*OgjScS&bd6i4`*E%}jsnvIe4Iel}FN;?NUCZMp z^P0TRce)s=%GzkA=RvFLH>~e(Z&KNW$DLYeafJ1hu5io= z$nAAP$4?WgEQqtm#QRpI=wC?TGPMjrtr#6)%SQLqq8$*~ zv05!Hd#copM-$~PR~tS4vu11ni3Og40krRJ^3WYfPEsc`7f2k7>JKE&IfLRI}7Pp4|pIn~E#JLKf`@XEV!PujL~FFz!jZu@v|a z=wDErC}L66YjnCL^_i9WU>=~cVU;X1=|@$W7I=Z&yeVr+z!F}|)j1ut{KJ+_?%wkz zwX^C&vVJ4Ofh9ii^iGQ}!lJcL85Rpt`09=5@Ig?`roAw0oYbYhpNrzxNUE?$ zNyur&c*1vATF0)SP6-s6EeH0@=UX?)VO7j*0_!CDNj*G>oBjpl`U~`U3I>}pv!-1u z!^}b;I*mDsW_j!k-k#EkU1Ciz2a z!CBv2JRE3DfA1kKdv(V~JSCH0kR%k&>x_`4^PsYNy`|M*g?n-Muuih#)4bt8wR}C@W)S#HQa?FdAUz zAh{TBaSl_*@xpDbX%tJCVhP}_5UZH$O8~Vp*H<9#Jqg}UvBKV+{mkaC?Qkg>fCM72 zp=7CV97BwRV30z7u>*7A?FN9ZII5eY02J6wn)~O|o^ksLK?Nqyn~_j210^q2F3ZyI zZ7vahNpPe}2FF4p0L@JJ_T58l39xzTVfPWJ)IK9{g>JdKQdv2G%UF(H>v94jvWtgg zK~!OrlyjPz(p&(P_pZ?w=ubC$E%my0@!@GF1EEGtEjN!uFMhcZGQy#AbNaS{R zs{c~*a>f*%D!sqi2;&T}@JpjVSgLa|oTS3OZGTx{!2(P7ep5SP%Wd7 zSspmZY>``>V-i2yR+eB^S-j8DlbtOtr44V~4D63Bzn!_AgN-I2i2Hx?(K~UQgVr1; z)G(s1lfYpkq@TKna0>m{w!mQ2TqaCCMTzX%H6&MR@k|!PL3~HAyb;u|^6atbFq4L$ z{Z8}kPZzlWW14yD#Y@<~$HCS^8_GCB9j9#(Ou}p<$iTiJoJf^?y8R@G+mpY-0#no~ zLsYK*PrjfB4JW*eySI%u`~{^CQBnVQrzLQT;}2Oq0$tC@nt-Vku9zXQ*@yPW(4*;5 z5R#imaUK&rP$wx!uKZ!V-ij;#yrE-0{%orYy?o(kJKgNUwo;9$FdW*U4WvGQ@jvrw z3@lxz-ymMkNHSMt<0FKmxbVf0Xp{J~Se}YOt16wPMb)J+Kk80_%Lq38=unqih>Wky z_qVvr+p=}yo&!GD?c}I^$Uq|GH_lzYHid>IP|37TnQ619Z5vJje5nsUAC?&X{MtrL zIw8;-IAm+?1jWR!Ji+MUE zfgK~`h~6CboOu2ay?=)jSJh*eqt~mhWZsZH9J%;jOgAvHmI_>qmBef}AFq$%_rm>i z#HP6FC0EGEsJm}9av#JNQT-X%{0pyycso@r@Chu|Jq*pmAL6*dOm%Q49M`LY!|@G= z<3~ZqTQCOMRll<+ycU5UrFp3b{6)9#9r+=APRH&H45Z3oKg9EfHw8%j5!_!*-4`Yod!}zZ`;4=;SV? zq0ss8TR;y^a`^iXrUCrw z*K8U#>^$GwvFU;~0U>|cZ0T0gz+lLut2V;4rcFuSD1=l_s3(ssDWR#}41;zifB-LKCA|HvNOoV=8Ni94bjHFl zJ#3C+iJBY-(UO&5MbVq)5E@lOVxll&u}FPbuodk@ue0cFuxp(?E~jtJiSQGkTIkEU zAQUF$BAXl$t1Z+3qPe%J?ZJ(ej4Ci!n|@Ss9QQV3d)>U9*jxeo;I_Ct+SC;XsG8$S z{GhtaTBSUqRBKZUa1Z>>rnWH$(W+Br zbFFZ*NZ23O(4%0$epiK;dUwCZX7J{m)Hz+zn|yVbv<@eScT?$?i+Cg5wkje%vU_h9 zD{F4k2&{q$MtN}DU4Lds{8l}3i^K!!p_(yswm?+QVI8vCQQ7MTSl(AiQ`is7_|W0( zt9v7zu)X}|8Hw`d=7D%FY#{P)SY|$QQlI>@O=)f8_Tuz{<&F^FvMqpK!J*xHO?l!A zo)S}JugXe)40q{#_bd>aHf^?Yh$IP6YwyY zhiAp~W}psbO@_KR@}^?}%mRysO_jA8;&&p>vDknJ2O)!AyWo;0w6OW7RvtB}1bAGe zCrkvloAV5JS}oW>%*X0yp65JIQa!*R?`4rrzrD`}GwaJLUjYvB*Lf8CTCA)=ST{1H zn(JV~N5bCSTx1T7Z%Mc$Gvv*dX>bt+@=-jO*sD#Ep@2Q~Ktsj#PBF~)G6}uK$mj9p zNLe{~41xWq+|g>jVrIAw#}n?6cGkj__6A?A)0BS9d(9<5PGPwVTr`fN+n(~WF0}vIcY*rw z2mdEPz?~V9dIb2&??0xZ&DSG+Vb)GFWY?-;EY5&OA$4o^#>Qmiuz?ro(~Y-4k{KV% zI>R^JU{AxxG`)P#H#Jx?K5O;$6n>R_QlipOgrG37+Rcv>g@~e>2n`q1A=#q*5j*6) z-?O~VuQ)`YJF^Dj)fNL~ZBdLd4y{8I!X6eq=eIcZV>Md@F z%p#a9BrZPI<1~#0`g+iNS#oM)=u$JtDuWO=ka1kA=Hi(BZT&OkDUJj z#8+SVdc-U5$EZi7VMcoHIbDphAeXowsib-HmHe2 z(9a(YW;|tCQ=VpNC}{8w|{d%GKYSGfPA>CJtM6=2R zT^MF2C{xK|G3pkbax?!Lz6ViY@Km;6Hlpiii-%kbFi40n)6_COji79ujpaJ#VPK~i zNt{dH)ZVxrb;6~1gB5OK-2H{V0pGlSVOO_t<+0w6!7I#y_9Nw%C3;>ZE@G0LuyjNf7 zoJralVv&o`8R&tE%SyXB`&6FAGEhP~yFnh_OrFIGhY{mrPp#Mg^jn$0)ZDvjbMvD| zl;U9Mg;nxTKD05G8A{@r7l`!EwGnc7r1S;)O!@4AmkkPPI%uA>m(?-EH>_U@jrr0M zjOMO_-J%RX9!cat*V*yynlp#kH&le)(3UAi`rdU(85LAqA#nY!^S3E|a7$q8jSC&R z#FJl>?I2?bU^q5x=19g7!c)inW4g>yj+;c>SsPkd>|>A4WtabHqk;TE?D0DojR4d< zF8tAYbja$~f0dV!avF8u56WRi>y#_a>lHhnzTK&cW%e+-Rxxd5yXpCH&SCEpj&0OL zcB^rdd`>)(`98Igfa9ElnBg5P&J=xNa-E5!ffV+NQ>Q=^m5)-;g~sA4Ru|T%bmkYi zy|dIgKK&5^rF4#7gf-3|v{Tl>c~&7pjNs(U>Gsa)WczbemWI%onuLt28xDgAqWV|& z6>cBONxTf)L|oYD@#B*7Z^*1?qtgxs!P4u7v((wT>u)VT#f3iH{rM*F+br=2!mcCY zu_*iYvs&Q|lNNf-LLtk5*q`Dr_etiK_Gb*{=kSmBF~4(z^r&g0!{!SE530L6tTZdf z(B3%zte((n>FhO21f2-lrP%BgFdY#m5pfgFLCg{1TkDbR2)XR67Bm;PO<5l-4~r%U zLrDJCB;XJza#?oGOB!0wR9Mr4Wva=-Zk@EVj9v6u+VHSgt0}4>NDg>wiYYt#O0h`s zC1RW7fm9vzMR%2do7LQBl@{2l#|nj08~-POH$HamhM+IZ5@Vp*r5FssK-gruOq7yc zSd>bz>v{KEfE?z-*U@fnAmNd1}nO_sf!|MQLl z`TL$p@(4puGm${_Fo@Q5<^*~QLCv+>PS}$NfmAbb+DVb_rArEX$c0y?PN(-UTm>B9 z)7By4=Kj`ehuAf|ysw8Rzg2tTht6vqDXtt0Z*y817ddpC?k>DPwmIRXEQQ<0e>INJ zcJwAndP)5p6t_6dP8sSiH;O7IM3YQiX88?jL0M>}9>!n3ubi1>a_?wot0G9^pc0k< z?CCPVHi8KOnYqp@4g7pIkMzazDW;{oke_=y=BsXXNnTML+V0X zL`S6Dk78$ab^p0(9!h|2z1CIc`UlS{grrAvHAhiq<(orHgA-;0q0JU5p`}=)d7niw z4yAL8X}yB#@Xgetl$$HVvQ<{h%9ng|RqsO2s`s^!5DHu^T*gotI)~2T-%CsqFZiZ! zC@bWX5NezpGs9^6eHO!v;ZH^|8i++NZ-9bWV_$8VRk5<)~# zx!^U>0+6cB+?*WOn)#b&-E5R(E&{p8Yd2hEnsPjui|0|}fD2g=Wvb82=QKgh?^y2n zNH130*unV}R`$ow_SJA~G87Ue@hC@_ba)IA_rECCA)Dl;G!)}g!h4P8o3-~QN9CWZ zTH}>uGb3a$|6!sWPeaEV(Ghbr#i-;1vBTipL@t3Zs_z%d1@ZP2PS*!>@Q-HcK%YGv za{y>2zA?tJPgw_X!v^8$Pjr19#3ZIVwHG=i7((Hy6^(61FbQCSglOb?FGL0PQFzh9 z#Qa5+yAQlEQZYBWJq0S^5N$|T5tZ$qPu~{jD#3VqkmHIK+}M0jajchRo4Ob&2lt7v z{--J52B0m#%VauIc*@#p-c0*Xu7j6Pk*wlN136>0RlPlYIgj0_kjleIzmkdAatz0f8NfMB z#O`e}kp2u`pB_W&EuVvoMO8>!l|hsK(b=XZYasd3YPZqi%bl(xXFxSfU^H*^+A?Sz zS0La}Dz5}1E~qVldO43lUQyqP&@qEv@)KTk&WBzmXO3jQ1bx>_6_AVI+oP<(IIC~H zxual?P5+bJ*tm+RxiBZ>goxfZi2nv!U7-+eVBpv@XqRSe6lN`C6^N@Sz{W=~Lz!VO7;q@8#yZ!(Xw08%IuSXiqIEXlj8b zm|v1B4~qa}tPlRgpkLrZrCG;)R2a!buu}ma{5S?0!R^Z5>JwPY0N%c{Tc?kO_T=k> z;oe=nu~1owfj6pd41+U4X#%v1$I;#=A=y^;Lw@07pDuJ9FX(w!bFD-ZFS~LKV8zx-qb~?x*vi^8ox!LuhbBLR(rN69Z?GC3kQ!ZFTPz$ zL(sB(gzQTVp@O}TpBBar+uKIoXuhFRI7v#9|H(6fSoRGCdomso8`yzDYya_1&93Hl0|2I5$I{7B zgp@Ij%KYngV;}Qhsc&k)w z$Y8aiET38d$L*6{j|ya<7#p;Y@t9f`{M#q#o57OJ7bk`>gJC|lP}!$D=BkFv+Cb2p zUaY8y;qBUG&zP3uE*!H=*1gjMxuky!Pl3#22(Idn9@@}qPD(l|2M=H^VLi1$<{+OC zqNW~E=03&O3{84dkQObbiT~&r)%e?ViTN@`!J+;bGOCv?)(mOxsImxy6|P<1;*SbI+({Ig=s43azIr22`2k2 zSIK4WD{bpVg|cd8_NGb(4WkJFalx8us2bDv$u}oQZsL1dF(_YMdEE9G{k{aqa?G`a z?@SjWxzk{Rb~kzqL+12@^mzVut!~AGJwClA>4Bt`Sw8M=o93v+r9U+f=5=J4guGeV zg7r%Vjrlie?$Hu}DgPW%$g0`9Yv&3Xido5Mf?<);Il|xd;Ph?dnj^JIL!VFAhjTaJ zamrr`2Ozm0JezlW^x)KZW;5g2BoW4ZUzUI7H>m>1CF z9D;fL7KTLZy@C$}l?jX?ofAdfSUH+z1bJK7K4t zaZ_)G4+TfkA)gK`D36M=(62<_G0rBJU+uTbc;rAD&1-FsN{v&vP2Flq}n5naND;bK5@WH@MG(CBz{S)Q*`FI>J={ z2rxT-ZUf4Bt>nQ_qwEEF{+X@{?m5w@Cx-ZBn@5{ir$!3xLT<_^b4;|tx7LpKH03q& z2=cAXA?VT{E93!sPR{;WO49>!v6u&~U z9&~F4$4Gu&js>gDyOQ!akt&7M`hXv|d2sjB9G`Bt00noUd*pSGiGQ71 zLvV+Z=DVM#Na~PMsDqGbs5-o3AAN<#(->qngm0FA#%kvEiRiUm%vYc_XHI;^D}L@R zmKq7`djbO|wv^CHHOVL{5rk`*+P-_a#`gMNJl^`tAnMcz{UAi&tEZB4Ur#&i=Rs?< zEKW>P=Y9SF_Ab+md1Mcvzev8<-w|tv0NKE_?N2j`b1_W@g=i@8z!6*C@nma5zx^PG zba03fe%8rCZD4#|CfVyUt{B zm3N!WHW@BRXZNlC_Wi-vLQjN+APgyJ9%|WE#l$q)Zz-Wq{D`-@Id&`lcDkggf5(3R_ zs5B_FINdfeXN`-7b8arafGPi?XClx?P7$Sg3)2*d?u%PqIGSe7yH=Sl7}mJt0tH(^ z`HYb$qBi?*!b)VyGTjxC9D#)ihdXjP`hqVHOk>6cW)ps7Ct`K^_Jn8eAKmK8=rUwh zQDKWUvxwtfrDk~S?@^CSN{)S2`=@Z%2?a{h zYAuNjH~@-1r9l{x$!S1{!EItsJN*Z9rCVkp5T%0ET0-i#k)ApP{rJ-(&_=TKBYSYn zsy>}qq@#psyC8|PRzO4$!4;CZ6_nsJt&^_o#&R@LdQXg*4*No|ffI7H0=0bdSahW2uNX#>zl04F8EZ#{rrAI&1b4E^V{>BuF_*c2Tx{aXhA zc6j?HAdl$oKZ-aGWI?u|@^qyZI;nMu6=jzQn$l^}s^HZZG%E`hHx6d^eouQz9=^tg z=ppgPp3rdKo*Q84`zi|tNoWCYR^0~_NhCjzmnnR{*hye47ria7Mj=2iRCH5ZtQK)M z6mbAJSMFb1b)4!y zf5otO`}TUwq?w^NLC!fy_|Q;4qh;6bexc^pb5?nD)DKK&@J&wLc$fQ)E_QVTey6Ye;-XdWJK^}qpe{yq=Chd3yAwT3@J+ON& zju^c##P9QrB?a173?1+kl(8X;LuGD=1N3>yA)4Na)a=MdxdMFGYH|t2d36W6h}=e! zap4m|j3$9dn~x9K;@JJ?Epv=WoBBxy*${J3|j zI$p?vBO`QUq`ivChzZUkIj`ir=TG}C2<5EbU#GKS*Apl*++V$@35kddEBsLJB=k*6 zho+X-Y5GMZ2O%tL!PMv!R7gR;Cn>2swM#S{0RD}&fGc=9v$vYyf%P}lu(@DUCd9?H z-ebj(#jH_Tb3|vMcE9fq2r43?xy6(cMNEwi!2ft31oJ~1ie{G4V;Xhnfm)RrU z5AEL)%5c(Sqo+aq?uqwq23d#o97#@d;sP@YaC7#v;6GI-4tP>##S;Enybo;PiBba8 z18Kzv``hk<_b8?{?7#q>Dda@VhqMwqZq-CC`Rqpc-%4ER@`}7lzX>boQ5JNt|Gb?V zmp$qVcWJho`7ES~l$u`B+gx=*hp`mL;9QeLD}e0*&Bdk=Bp>r{Q?J$6{6paDvuaY7 ze-M(`S|qx0cvT~k(%uQJSWTX|YS~ZI0wH6dnGI8E2m-E-5DDf>IQb^a(S_gcUR@-9 zU*R;QlV`sgR4|aS^==ggi4KgwW`3U{zS7?rY&LVvZQe7$xt}$KhV-p31MGVGR!_Mc z22$!r+HM^FIazj#b2lnZrCc^TMH=*LOz0=_I5hQTxfOz}yXhNA|`6cqz( z;6DEd4Ut`;f_OC>Q5EAfJL;2DgjahUS%LAj(TD}1RTM&q zVw($ft=Y3ktFoStpFiFWXtU=0@wpe-IstML`8@-6rQTn}KFZ9u3nh4G>agT|+yv(V+w^%T4ecVl}Fvz}mAV9?7TNY=Yu!f`fDehP*3rqo_&^5GK_gE`fNny z36l`9{lF0IbTg=i8>nM^VPiE}NIsly7a6(55X7eq_5 zVi!ZhokJPh%^%eD@R?bm{(gVGaRl!)q}t0}yV2rB1XrYN9(UXQw(?t460X{bE;G<1 z5xjD1M!}|VK#mQI4S(1NWmM|wz+9y z6@m#2oJDtqa9d%nV z1UOQo3mM8@*EF_ne;J7Y-I556N?`K!PrscXOe{q(RsLYu>)pzU4GXFiD=^QTKkkio z2YG|C7a6Pfqm~Tp(+!mUJ6MPj-Ma4dSmqvvHKxAiP zd@O}Vsm4*65_hfp+ut{uW`dnO=dKkjacaJA?;t%H390i|#-2cEbIf7VbN%5Nv$zpg zaAuk30OR0Jm#rM zonxfEo&^f(33dZ91m!k+M29yk<2lcLO6&|n|H(24uZQzFDUUDU+VsBLB!i+!lQjw* zG0dC_ACvSda9a4;Z*dG^NK%iSM<*A_5L7F8W_eASM_t(Q>?H&X`- zPAlCpkpGrSPD|-f$+Cb52NrEBV+o?fj!zuCm|rJKKikXBF`95#MpD6j7X5C%vct6otCOKVrDy#TJLYUUH4My#IBiCo`;DWfu?<_ZIR#&ij zNM*MasgQJV-v*>M9Hz*EF!MiPIGmVwZKLM51YzO2wtMb1Vm$gL7CmwSsxMt{gDx_J z`T0Y}0^HCzPz>`TMG6P}jjpdGg4Z}}Vu5OJP>xwg`ld_;9_;?!zgy|i@vj~-v2@wG z?b@BerPr8bvhtjl6?PJ+VA!p>BSs_duE(nQu6NyyHy&NuUNfUxT^o1H5gjlH7^}Hf zUSU+iUJjj&p!SL}%%&HY9`1Y$1hA0+#=V)&j zx0_NtwDnsl)KCSS&5^mr;^5PRwW1HWE2a+Cn1~F}1KFzc^szK$?R-@!yt>N*VEBT0Ce zh~M*7eLEt;!cJKrY?aj}Wwuc^0l zJYKahv-TAD_g~A;*Gh~nly=)#$7u{1zgH>i-5Q5fX5uN2j0}4dp~@-hQ*RACFIGn6 z0xEVxAP3&dwM9I&4UyslvsnJ&@_&m4P1jks)Ah*cI8&`Ci?SctQ5YJU`?6@UZ)A2| z)8YYLl-i|~p+@u{6UWXEAjBzDdyCgGxXuT*X(>mO%~NWIoRfol@lEFNjN}KjMHUwbIby?)u|-m?C5)}2^44ThoCS3OC2cr!=GIn# zE_J!2yT?J!KcKsS*w%0I$e;lq#lVFS6adba+%hL{D50K`%Xj`U8R!2#fyMR>NEq4q zPYeov=h#d+J@MkX7NQ4q2^z5U?{pQQ_5H(f7 zh`h~jKQO-}^4vv+^I0e$E^J8Dkyry*gp*ob_k@*QW{NRfUzgzV3#1(i7FNxBb`gdf zq4|PHk<=Rr{h5_l*xqLFv5^utb%7mD+~W=&8&g)vMqs0n$r)tiYk z?%&>|KoRR zLqP~$TZ8r~yi3A_4L=!>T>#^ZI9snAYvUm6lYSF+HWNj;aEK@C4~cOm%GIK%w1Lov zF`DsS&eb!ijOl?pQao+6J4OPnxNn483;rSrZmA$QmcJnA(>{%elE5#hy!EuP_n`#P2uXa7$G1@3LBM8*3SbGKH=HDPaz)W? zw{t2lFU^pG)~c`i=U~6o4#I$C+SN$o^{_H?KMD7?UbuToy<#e^RbDOFD#e~&RhPHi ze;G%}MoVLcY#5#wL4jN2<=evRwYHi2S=*-1oXm}qP&ESToD2^PZKu8UZv>6L#E3c9 zFK-L$9L>8BYOUM|V5YT$uugNrUw36Y||O(rtqFs2phS|LuNy`>Mo(&e1)szgmEa(?QHZr!7T% z*L@UKO6Z~izoIB<$q3Svha#%GMu9vU9?gpWj1d#9sn$>H9QM1K{0ZXFE?${DYH=p{cE1uW#g*!uj zN~%XCyJk+x5__q*(-Iaos5mL5B!e@A0y`|x4`boi80K+)?{@s zG<+p0;1B{(wD3aXHsw8ty&p^1J;90}5*fu@D!7_H3v^FlD_0*&cJ*JoJFDp+;ALWp!;*F4vXkO=Q z2(GVkc=ngflWYlNI%y73Y@dnowg+FJC~5*1Frb}nQ#A_V~tpfPjS8y z?vJ=#h-KzuSG0u0%^+a-Yk_hFO2k!|%v?&kjqP@xz}!P+@g86I%~RiG8b~2n4VDK@ zBJj3Rf7{kjr*HO_Z3+DBIRXgYdQ3rc5gR`Hm{l+z6W@0(Mw_;vLJBZLhbCSbsl*NT zKs06*pQPgp>J5;ItDJ(^f={F`E35~BYxLY&ko~-)TGVq4h+>=$AGHQvw@QC6L{|S! zl~MS7EG$nAGfD=5sy<~VBWO*g!5z7<7V1Ry|K7@ODHn>`u0k)|}u66PYoWwQHHB3A8o!@STyzI!#6wa5(}I zNK1*pZRgp~i52?9dP~l+01`; zD|(P)LMYG0BUy}9_iIdVy08U!aXGdSYA#hVL-}Okzz|w=KU$O{i&47+t25*vh7kT% zOr|c5cmBsj*C$ICNJ9oZo+ika>-Ia{5;7aW51~}ZqGb{>$3uuoq89<6XG0#(J<*zs z+>hn!h|@Dlhk$X2s$tfPe2_HYr}W;{xpPnb8t>GNcp!|yKi)TXxV!R^f8~6!ekmj+ zk&7oiYDGMHmv-nCC-*p*uxbgbadq2>uEDaZ&Z80hwTJV|kTf{&^_T2OAlURdf)W2) z-XFwi0cpU!S(D{FnCDI8t&K>?qoN|De%~Ma!Y{hS)gA}nD~hz{JK^Z;Jb(L%L+3!i zD~E-hmzJDK#Fh!5h<;MJGNTugxTGii4=N=$=kQ$=Gqat2p5Af;v+xCk6$q5h^@`JJ zkZ{)p7^Y1`h&yy<*oFuFriqs17RdiHJBX$4&Fx?jE?AT2bzq|_bAZ+A?j@Io$zO!f z^bv8kxvKg5X+AbJlIyrUvXoVO!`~Z~_Lzh*TyJ`gjGHP)4}0OJrD@r0U*43OtP!57pibZTcNJ_4YEG&AG**Wt_3FY}L~oP~^a zx6%@_k*K<@fVzXMg65})Oig3Whhcf5=#OjLcJ4Q77?>8WoebDA;`}bnDFo<=H><3K zp)N;L&<&*+-UXm))xKjksx$w$s+ZiHmv6ImI*Q%>m98fqe_udF&mOtSP7(2Iw-XQp zTL-35a*FKiki!Ow30Oi;tuGbsWTOodxa|6T=dpL&NFu0?2*@ z!k-=KyQOdv)u@GB#_US_GF!XJ3DKa(!|g>Ay_`;t4+rT0*Gb1pYn2? zB;m@59)!5DxYANcl>#K#)od@R-l9LuHKI_Q^`6m9#&LI7VTxuvHf~Od-03Kq^fSlfx%6cZXeI7V{|cf)c#gIv2s0 zJFU?1Xg~%%G z2WH26+VXZ4)TD;|kls6Be-^6R6{EU*8;A5RhFZ-KEeX5eS78}$v4bC43 z7PKG&T{o?{vP_0l_fdFHs{NL2IMOdgilf>^0)Wr0z%3KoR#BSfLE6parzGwaLtTic zhvEC^0Of`!KrB-qyjhiSP`LZ-QTdIdiVA8QB9?r`%73}z;fb~v?WVNYD^ z%GXqV`KVLeLJKcmDk)#4TeixJe9u0jYW+li>8h+_D5n`MoSc?@!_X#t1PCL^`nghI zt|PMhoQ6?`$FwVbOzb5)thBO|wgs)WW8}~{f-IbOXp>w9t+KveI$>vZO{DPf?Qa4{ zxt>gZyAT9?t%(YB8Ko&FsI$qI3*+>8mLbKJ^m}VW>6TxPwnpTN?T$w;)c&zAS^Zp1 zrZR>qEB0ObJ~hq_2L|RVTe}!rNP4fMl1--dB<5lnl0z*Gl4B+&IE*>m%O0c88OSh; z9WdYgRRCw=FZ!z;F&+~(nGHO2qn_Nv9(vGegdbsk8F{~e#dnUSqPp=j(xV_QHMSV% zu)}&g9(KGabXS5=k-0%BwM22QsGvdj74rfGQraKI6J~4XGN(q*DTODP-vp;h z*bz%ijQhbo&a=4kg8EOC{Q`g0CcQ@wCq5R}hBo7b)e)MArM_g-_Q2;SM=F)V7@^4t z)sJboeVN89VE=X1o(%XU6HAsc<0w^1Vq$bUk7VELRDt>|qI*G5+J9BZ;6W>1PR7PR z53IGA%5#K3Bc%6y9Zf}Ty`$pDN3y}%ctm+elosR3bJarZr%vJyVss3`Ln16#|D*LJ zdF@;Mz3beh>ONm_dOs%fHjtwo?*ZUY!2MoYtEFspa!majrKFzHwBVDEDpQeWjKWH? zz#zyca;wZp`hKm}SkqyyJI3$vq*9DSa*Yjd>9y%oMP(SRS;l>iL0Y}pADO}^QOV-9 zuL>S=I3et`Pcls8m@~HX_z2YQ+5KOEy;{1Tmb&gp%!JWw%0fFVBx%sq3C9{FQ4anx z(4-IcTFaMo0bCOecqnT4^1cG$Zb@1j{8AX!r*yakE9W#*kjDNlnoE&txDO~du-AH~Xg8QK5l z1z~6cUGN&Ch&jIeZ*@X}N$kLjG&o-}wAa9w7KOx|5WI}|6Rw-fAMAJy?bXOcbEjW* zH3l?U=$#^oDvw~?WHZh!C@U+!c3E^UBG=3DnYhyj^=*SZmXb0R!GxKZY ztAEM9RG4;)kgg=_Xm(b6?akNh?Rb~4w#sMO25cH~^U$-Q)Tq3$TfoTP?d0vbf4lUM z3eU)V3v%gWk{@K*u?)0+%YF5`-5I#hydPR?#jqt#6yxqb-$#Eqq{#0#BZ*dK(5&On zSOlg2!NVA!g!vu_J*Tfp3poQrm9IC0}5Y4j*3-6 zPOjPImot&6c9p@bR3b8qpFpXX-dqjMRFlBvrFzUlR0d>+p3fHIXo@2vEWb2O#8xou zQM>z#EOA?GMjLRsL+z$amz{i3#n#nVw!_0hTwz63lSUBH{WU#+bQ(_jM6VNi>mCBUi65pr?5%am)dmUQ3{)1oiBee!lp9pv)$2H z{BoIz;xhsGBRcOb_#q0FkQ=O2!zeHz_$zW@D(`yxd$K4PtC~AV` z0bw)T`a3p&?yM7fg%8LhQ{_fGwUdgRftvqR<(~!6NU({3H>2s#ASzz#tcig!`7ThJRD%| z)V2tN_f_a~y!KJ^Ly%=;F3e;JsZM)rvrj3K^%7|GFp9!cQ?qzDb>*DFW1rkVq^Z9n z0^679Z(hbG#OnePvX73+w;&hQQ3bwfN`SZ@MtUiARUZq3mX-4sp%Pb$%uw&+>v6Kw z)w;!%%i;T>oQ-!LV?iQ|gLcW44QHbz_dgW-71w;Rl$&wYwB6zD;%Ao`#0hpIgUy@-3-M?%E$z;FtHv&&)=JXdgwF$!F(0G^TdGsV+ z`%P!nr%I9y-OfY-r3O84A$MRt<)a{~{)?+;bo3|G%&zL(F z|2qHx7%uJVN6Cq5&AspZ8(K+HS;)aETzqb`czo?N)231LE_q2W6+*?AQ+#l=w5mm7 zBZPwYU5|)KJ>(bq$LRHDSbbzH+AA}X5#f42V2NuFiWBnSrI`EtP$c8GSk7t?7jA36RIIthLIl%3vAiFQf~T!xr42t#FLdZl%!T0^a1)MZ z0hv7}*79xtlnb=tOf5}z0nDX&&i=d_1^3{*PUQBuc6CAyDvnOq^%Z-`=#PH{j5TU6wh z)y4(6{XuFu7|sp~HdCXjwByMS5FWY%zMS)7#d+wYp&;Eiit(`8`OIReui>_V1GoS= zj=!!_lOCE6n}T^^d0X%4eYl#cE_6=MR&adGC>}UF@j`qjTt2*L(uS|hTgwvPdhcZI z)1Wt>@X)IoL-CUZP7_K4G}h=gVIqO%$6*n$P0N36sl%#lWZgmSbQf7zlS3e{Z;S*j zzri0GLoLu7JuW&pveqOmjoWz=qFBeiicOrv%8Jeu@N-35lZ%0)@T3BS)xv;|v*uu^ z{n>4@SFZQX;fdM_db8c^8ie-{IHTg24`2n1Imd=PSI-We02%)F<~ty)w=+&dRk#?? zzIGJmJm}K=1z|2Ji!~`NG{6wi22FUY0@rOpTNR@SUI&D@EI*=(i*%OvGsl4GXEd7w zf|R&cZ723IkGsi%bWcuw2EqlE3DLZ;L@WD(I)xnO`!@>r}Fa!8%*JzFzZVKs|=lS zE(!nk^R$HL)rDPs`BW8dY9jT7{~rAqs4qw0Ul^zY@v^C>u*c7TLbm z;{n0=oR8^@*8==1bcmMKg6eP-z7A>^1}W%2rY~|v4^O#<@S4&nwRp=wWJaj8)?qeiWxujtUAA6vyf3s~wyV=!gmZ z+-%6YZTI(c#6HE~hSraSVR)wafIW~&*EYpn%u8k?CR0JM@JiniVl;L7bjAHh{>eFp zGB-q5DsE9qX`Q`f3 znDt=E^NS8o`_A?Sjyu%OAIkY*8jnGiHQGI27Xet!GBuv3ydLa zvW9ocYAAEs9N{>8W=g%#x`$*yb`s8KT`BQ``S#nz9R~-tK!MYQOCq(D*~F3O%K$Ro z_yzX0QQ4&kq_`}WTJ-wCe6#R_n_MM-A*Mg0;Ir22m1IhQT7A-Y4g+^TwYtwrf=_dx z@i>BVyi;O-p0K1gC@Gs0N!}99pj2&Hy;X)ZJn&BpaV8!jLAhfq(YzDT=Y zl^sfwFwIeVa(UidB|$Lxl+#}ZgF%Z{qnbY`pzpM7>4-cwDBdb_Y2om?rWJ?wx{O+w zt?Vss=UZA{d6w4K!AxLuF`IYM1Nf~6^T-sqhXrJ5j+Ywe1Dc-c;|uBVVr~*8vxra~ z!WMg1*DWUmM({4gB+*C|u{Xg6UU*vj6!d&{=n3CV`BF5Nx7BzV)u2`L_az*bB1F3!EmPwjJCMqt`#5dqP3dm)aWtb;iZ(&kU{Z@x; z+lAK&sm;e)WSgt$ozE+$B%c|b5+F{+&4=XNZ*JG<_P-XGDS94UFi*5J*n=XP?-i+r zJ}*G8BBs9sALSCK%mJB%OnqXCUQq3|vaGYGa=31B9ZV4k26zF+DW`>5?fs&}@6$`n z;AJsG^Z*5L6s2>`>3UnUI$1XW3oB%BC5)xn4r;YyIb z8=8Q39siVcv@M?ePlJnF-pRTp(Uq-xt)V!sZjI*`+69WhFh-L>M2+G(2l_zi>PX;0 zke;=Kx+koI3gHO~pQ=s((qlm9{mpaej-9A9d zy#{oxy)*=L7*szxe{#m>HM&86n))18o^|GnE< zX@-aTa*GX%^QeyUC*Bkiq8>!vgljxvSBQrg#Z~vOptPqNeuiMG%eHR+B7t&GSZX!-jG;XUVGO)KA9_0t^Ec^zBBZ~{0_4v|g8i>J zI>tN`aM65{_UH9BUA3Eq#t<*U>LOSUY`L($qOAJs_ zEbzUK_BI3R$toi!zvU+d_I~qhW&(*8Y;`at=*WEvh}znL@qRTw)5os*vkmG5P7fa2UNj<=zcB;b>m1~obNV`f~{(}-X* z2>A5&YHLbz`sg0>XU3RQIvF?U$e*7f$iJ zW|Vl9ee(HBg5G^|0(VIE^=Z)HvmZ9v2CXdoR60^bpp_DQQZD z3$vzy<1zloHIiBQK<&adNnlgmZjrR`igU{%cXEuBZ2bC^#L8B<+vkJ2PuV{ih|psg zO<^s~^?k*m{_Q+gnyXEv{@SRO`j(UcDSQdca#p75o8Gk<%}5zbgx4N5FcEl~L|E5? z&zqckekgd0AS_~71Ye2ML%6bS88BGBmCi*+ju8mzOF13tQaTHn?HBep;&(dqMFRs0 zzjn#}7c4By^8dCw9%9oGW~?1zzUYZJ8Hn){iN5J53k14 z42cC86+Bh1sMoud!6-``Jf9s8h8oz;FEs8I6u0J%N_=N875CgcF?F@$^LE8Tbm><( zC!Di?0*5K3i=aXejC5->>?iMWRr%7x{NXDkUG>0Fni*oiCLTIFX;d5KD(v?$=okXu zujf^7LD{5X9DT4|LUPE)8q1FG)@2nZ)JNC*7hsy z3Cgz=J`0z&lz#X3d98hF-|=63dH@6d61?VAxi~kSqH>#eOHZ^p|3J=xzrk)h{PW^8 zQZ1S?c4?W*AZp26@mR!;xD<6C_>F0U!Gx!81x!E;JVpuLUHy25*=hD}iVT+gDUBWb z2ct+3qvz@Sq2k!W%tT!yh*o+sVM>bTnM)dwN@f!suVQF5>v?}lo3h=2R8p5h(yInHf!Q0ZFujN}4@1P>^ zVA4w-aN5O5uKqbqWU&0zZbk-8y?8-GYWVtAg9C5*kVqPm$D=oa+>noG%h z8U^Oxibf^spsh54#CRjui!BrGF($9<5`RYJ#(mkFTC~AHs9}Rt@J0K8ri^%)xky7ly_2r%w8MZIQO4Zy5t#lMvcjO)2(ub{6 zFU(@U#|+9hc2(x5Vq3g|`u{iz{ZTag3^#YHT8DzW&)#R8R>cropc_};Ei02Lv2ayE5NGR#nx*FM4x$ zWUslHE}P}PjuA=I{!Ev5Bwcegt4mn=dPFn4Rl{+)G!u2AdMiv72GhCGSTT|!-bw4} z8{eWPi$uf)?m9 z)swK(YCJVha2(JTkp33A0k%FD#xVZ@pqZv(5kuZV==y|;e8HP#7HpqIc*!jV9mz(v z&f73_q z88ywhl*U_Roc0O0G}@yO%(_v7vE?o*j%TZ;9cg~JtZMyk%Le}y1xJSTsvkb>L_esg zJp>f>feko_v?f?>5HOv+gs@GQ3T#;=vp>fkMBKXi7CX~A|A!KvWxFu5E7h{%ihnC###Plkv)>m~+xCaJxKe6QCKzRPn5RL|T;wPJ~Fh zGWrZ2Uec*iefWZz0zzbIM-IHY-H`9%+g7`&Qs_A9xOb`RtAo~vL7$_y=Kv^IQ~bH%s?Ild}1TfZGBFH-;Q2uhifZ*)lILeMk8z^7y~UHMnHctF0j}W3Ms*NxI6-Ll``W4kQz@FD zhLF5^imXV-`jQD5IOj52IVBfr26MQe+dR_YN_;nR^u?a?+-}}e$4a13IsVB!a6&$f zcsAHsxv1XbnQknn#hrZqivqaKQB4(xQq~Er*^33l}~fe$6z+>228 zFd;~W@%_M@-+kC9j1HtgxL4nb8ro-XA?3Q)7(l^T`Z1G$$FV`eSUHcb3Cl__%((1m$ z2ADF=&s8a~SJJGtO=zGkO#TG@iY%)q6(53Ss?CEVT8u z(9$HHyQFkc531gz!5&_XsITFn}07$h05{U!R_j|19} zOw?6)B<1=AL;hXR3^A$fV_7~D9LSUO`7N_?75i-UgYs!?pZ)Qs7_xj!zkkUv(i1Yq zWjxhNLSPJ8ngFgAKsY&?!!@EB7^MD*3Xidfr0TO|PnL341E{5!QpiD0-y)&gNgp~Zp^ETAi#n?{jT|B`!etH3*~+f`uhw=yeAi3T!-N0Zg2xE*by;?v1YxC~&$_5Ia(a2AL0O@|TS)wrTw8BB;* z=DEI^e{q+?R5y~Dk7i_s5Cp)R0^NJ}-obRuPXoF`B>K&f_{Op@YOsbxz0ZR({g9&U zPG0~`4XH#-G661(pdPd8G3dOS$Qx|ah=op-;liFYgg8rox*4m~sNZy& z`HrnmOJdCw76Z%_;3~;bZ8@VtAZ=PzLP-Fzj|2 z-*@PnEesVR;=Egwn0yt2SY2s^H9LHxO&&t6jiNgR%u?7`B}{-;iaaXlP1aFY8ic%_8tNp-iiGPyz^P`C zDqDu@bo-y>%;7DlkFu#%59*AUj>{Av^2Jj721og4s|Q2iqKaEyL_2d3l5}7dXZ;Fn0Wd{*$JBg`?bma;f=RZ* zbKhBU@l>|pIQIaXGKuM!ZUZO+sRJJKQD>BRD>7?E11egG;mVY<8yCUrDKY8!+n}Tl zY?Np=SJOopA#SXC=VcM8a*>tnV*O^O6IRjd#_{Yqxl@Q`)+=PFD8C-kk zW@BE$q}JMCj|?SO>(Bt(o(MEH86m{8gw?CDK35{R*2%)K(42~7cyB)ghUV@`SKg&F zdd3F+9F6@jukH{U`e{A1t`kh z+V4D9iJJkv>X>(pe*WjW4QERHjy zj0I#onYIksnf_yF>;2A>BIY5=e~dNSvvo))@`$dMZATEi&~fiWTvJ$d&a4J?02uyH zIc$?|L8!(E3i~|^*@`b&PIa%maT(F3DUi+qB95d6E2w%m)G=1g#TMO%!qqdAsA3(6 zK>*4lVxF>tba`SE-FR{2!LL@&d`!=$ovKq6qNH`(5E=K20BtgQzsBv;UH=f+Z({!{ zfoVSfn2Ftc*|F^wm|IMbf#3q%IaFDZ3^&Prlc2sUCxWc_q-OP5U^`~KdMOs7ZVB`d zyu6^A#eyOsP{EE)j)%8-D_4i&WFlQF4z=*JP=0D!aKe;JP$nGF!P9L?5Dp(-?8FUl zzt%3goMuV*1G|y1y09hyg$bKb@i2{F+@cyap*$griuKy8IMTN3YWi56SzG^;mTy{{ z%@Z;VRF0@bb_?;8g=m}cTnt1>ei(qX2^tl34nK$p>J3Nlww4Ok3h#(}BaSHIQYy|@ zCqTBVjkv&r5quaC>VdooP{2&8ZBL;`}LbmzqJ0B@{Ub~H};

LMXf|Zf{^JLf?7nG3ZZN9>SFzl70>M!BQ}=(bOJwEU=*5}j9M5^I- z71EI)nn&q}w{@z{YMU?*bPFKr{^JlyYFlPRQ3FJDPd-;Tcr_~kMd-lIv3lu8^u-m0V!;aYMsVH+T6^^5N zhw=cTT=V;g52_alq%^T%!i@MNje-~El(FDf=P+(-VHydQ16?^myygInyFIA5Y!PZ^ ze2;(egq#4kvV921A!~#;Ko$*h^Y9ELx7G~z(zaf^%AxC2n5PbB)*7=#rXosxS`AvR zs&xLlqZ6q@TrVk^PSq}c3fV?@>!tu@#@ep(h&lO;VXHC17@t6CR(wtZ>@VZSA}xQ= z3s6fVYd!dh()TUbkZVmpZ@yPyYr!jpAA=ck1t>P>_{423+=^Wmsip)ri1-&LL%$j6 zgFINfD;6WeAS_w`5Iu1;OwUkb(%akZxXao1nr;`lOX|AG3-gYjOMfBLWvtf?##|&6 zpeDdP)Z!paQ>fJ0Ll^5(RF@9ZU7W+yNdMUHAAbUA)|>w;Uom$uV`9;cY`#g9{rv3h03$UHRpQB?g&0T(B#4u?N;GY)sY>KnS9nN zowlKGh)-*4L*a@Yb`AL3`XG#X9_NnBJMBwxdEMi!CIxzUM_GiU@H^uaA$Z-SKh5_f zRFInTof;oZVN#X%XrOQ}eHsHIqx2_=Jy-v}Wcr(izG!Q$HRJbdeC5i=<0<=c&H8*f z$6d#A6`aNPf>hhso?|5f8{&`Wip7Fk=Y9D|zr>jsCAU$OW$FBMLuZ)U&rCHNu}?O? z0}ZV!a*g%W5JSfR=Np-%V$8+YFq__H%Y>SpmNE)s`_pP?H|+s%6inCBDkt)o?11L- zA1#O&7QXOYBxZjW-$J6_>R$Kr<$by(YYqgc>u9roY$*p#2ZH0<1sLJ{E-L~Il0(S} z-_S#k?8|DG#yTfhnSQj?Vk`W?LcHpiy&)5fov$WvF_w!5X$pBA_D>kw-R#(9K5GCx-`^-~<#^2|u?4>|?O4&vZEmxPs zq*}!Mlbu;TBIw28#dQeKKok#b;E9qT&7u`kJ8e(IjK13a5y0B(Id-Xm&l29+Z|uN6 zp->OW_bIt(7w^kpOA@WKLOkK!%w?kce6=hqVYgwHD?%>$oU{^1nuX^>8(a#+)+RdG z!g;+3F0oe#p7o1LTM&8;5W-pmlmpjkfEl(qL63;>GoPU<=h#7DX8}DQTAvoKg82H> zcCJ9{aM^H+-AED2Ym+k~R@s@<;|PP9u|AU_F@5x*UdDmN^;j>!$^nu?N`Q z_gJAoL6ExOL(UPjrHPkKFs>dGE}pZ(nOq7}_0`0;P+qA*1`-mhvSjkk!KjFbo6ymB zW`xyk(Ka*$T)v>!L{=F>(~8~( z8`|uh_|=U<1!g)ktl1A%#mgU*2KS#%VpTOTt-H-EY>k9Bce=li>w8<6OJiRxz zrTy@nt08cUUQD26!EBmoc{*@4#M6&1cNZlI$miq{JzE>K>*LH$Y(@uU%V;%~-n~5z zfK|{IIwm-9q2S51Yn^)N45EO}-$c_wk{8kpo-x@C? zlnY>~6@p^ezRaH)!q}H5icE=vZnq7@iq-&fJ|(B01$pOVt8|I}$SJ)Fi-;zo;@Phd zme&_P$iYLbKEaBghG7=}B63$h{Eh65oN4N|p!g=@WGz0Bzkg|4$>|MF`LnKoE0< zo7phxP3!tY!^ECLqCK>$c=2T7(UOcoW+@1c3cPXyaIL+eKNBh-m8ei!T)XI46-m+}Q)~~pA92a8jt@#Z1 z5S+zaPh4T-=)Z|TxHUG_+z9oAP)$a?#fx$uo(sr>9J{hRHwOt={v$8FBN}t7v9?7! z{;%18!+D?yq_^sVs3pXViqW#db+dQlZ2{E`*J<6up5s(2bzVVz=c!lWKoIJcQxUpH zK-b=Atfc?g_B3VI(6A();!oBwI!w(Wo1||)CXc|?2Jz^SRPrQ-+XU~l)HO3McEZJ) zFsC$P*ycBLTtb-FbzwzES+8O>YH0Dmi5iaigqN92wIF9=iC1*Oycz!8Yrnd|2!X+@ zQu|@kV?BAwJ)-L5_awDV*~%H|zttDgB4-o7FNgRhuZ88V=JYPDU%{z!COt}BF4qy@ z!#6(Dz>gnTB(*71X$o88h|KvZcP7pWK18rpjJ8`yp15O|#@;Zb&(=W;w%IzTq#rFe z!2n4sd)4(h+3d3|j~=k&z#AEAFW#eb)`YAhv`V6=D^1zGmA{5xMx4V#DA10BJ0e0Xo@MK{fHb+gTDLWF$w&p$V}Bc611qp>#01%!sMP445gs%8YSZ z?N22wc(mz|N4iY6FkE)*n$ziklNc?IoC$}&i9tvLORA?uLl{CEMGd={cuT2g&JZ%N zz-q~MdBl}@?QCy_@wjW~aZ>o$?79M|Kn%uiK}9mbR(|1NF$cN2zAfWsaXgaiYIc(Z z(SN3;&*wJidz!_m_kU>WP9m+rCG#vK6-hY!o^pgCFMFkGxBpN-aCXX>#0aBSC*cjW zP?81N4m}U~@H=MPp}b|R`RmvZE#d%hY9PA)(YM%)jgNkLvi#* zV%CZ?$_uSbm$1Z?AMOf$r-=tCqCE$TXVs1H2OxXBBve#XV-b1_N_T~68V>yeA-Cn08k(t`#BzPKK4h2pZtY3WV7QwCr zTMGx9hy5=kK#{i*n2)o@G zi4LX_6D%Z!7%FPvoAWqj@C>=z;%zWqe{z5}75cJN zdb#~QiZUjpHq|3)K%JiNlY(7$p;Xi^hR=DSU6T_QH~E}mQ`sy{F~CqZ7THVXfSxJB zhKUwZANa~S_>4^UqmQe77Dm+^d(H4m#BSQFHEL6Z9LTz5FPfW{3SlLL=qLf)U-57k zYQFh=UAzG&30rP z$`Q)I-Gfz+DYa=r26lr={k?6dG9KNc?L`z%Wiv*^lD)z24PmNAi5ado6>sEfaOrx@ zTSrnwxX!a+3;D?O-2=wOgj)*KY6JI|jrr87ocYTFUcDJqU6a>&LV8fowFK0;vEix# zxr+ZySJ(K*jvvE26Hb?~=c=^A(UbDsMd6;S(+^9fEC%U5yL7FT!33mFd`WUZd$yD> zcE4j&;6sl@)c-PRL%Xk}GTQ+qVjST)b^Da>4ZPZdA=fx9?$I+6=1`AlYMvFW^LFTg6BlM?fc!}%M|X`c zy}ID))&O#iFYcIqYBXZDXC)(u+gIu&UL%7_0&ydf-(Dz4zpFv_(+7Np7gF3_a;PnJ zZgI5sbEOTG%dkw=-pY6o%z4jCgFAMy=zvm_9$qR0unZrm?Di$ro~c37r8v-1Bbe4p zbk!i$n!Hv(WgJSG7~!%Qg<*`%U4Ru^!&-*6L_hTv;fQY?Rtj*#^7obV=5wqa!O)38 zwr4^!3pJWk*ZVo4oZlLC&}JF;V|Ps?4{#EtZCqK(uJ z0Ssa%3}}xt#U~G$(RLJ3G`*fsQBH)3V#+}G&O}#TV=oCx0xG)^hllE6>A%Xz3tY#6 zy?GNv=My|uELzh|t0ugS@t)IY%XE35ToP|rXiAhz z5|@|>m(FHJ*tmGyO|5|zV-S%8vZavMLv{|v9dG5= zoV1#mHRV!Hx8U=$H~2Iix7AOR><4J}nc~|zL{;39LnzhgS7xWXOEXoeOjHU5uyg6>$U% znvr=bDW|7MWBM>_tSfI43r~)d_N*H<0Ou{Ls#yBE8~OZzyFq21=Ua6!W|U4PW;WQn zzpn__T)D|SnYgr#&$H%|qM%3yfrEb!-n`iqH0Yj)%89Pn2pB$cJac>J#$v~Zc#fAq zoQe2AimiBabNJg6cLB78G)`>68^f+0%EzOMTzYzg72U33Qb)X+6m+P7c5+Xh zU3$xp>l4VZcrUZlP-so4SovS+OeR;E5Ork9=_8;angnjC5|A*)<7L)KVlq;Ru!B*a zh`AzunHZojn2eq+T_T1rMv~ylUs69yQCyf2GPV1GQ+FUYdyGl5TiYz{3l)=ZugXD~ zPLFtiHYeyn;aG|i%7AblLJ^caHtwRV36ZzYa?9g4Mp@VEL&d(##91qUSqqVn1sF_v zuxg)I-!g%U0Vk~(jyH*ZK+|gc2v*ZtR5dReFTRGc|LNLH4g)y=U=om&Z{-i*iC-H( zJS}h-K~akM3tX8#qEuD$9E39}yEDOM1cAzjN?ngmrz^CF``SzOK3Op>e-zam&UNZ{ ztPUFH4c+@SL*D6g>OrE##H}&07Wkf1!76wh-1e#Cgsi6v9w81F-jS{l5&>iX`K^di zL1jp&1~hn$Zk9C+FCdNLE+X-_twhx2N5t+j%&E2V8yHd*&|Tb7o<6D4HjfQgr0S!b zt!mAI@LN$v3stwFzKIFE4F&C~MVlX`^QtdQ8a-*x10oi*l6ARH*SDdCtq z9h-h27dg~OM*EQJ2ToA*qQBHI_)N=>B?E zUldnhBN6TNdoKqMrfa#j(+jZ;KIVH9jww3}ai_Y{rwB=_z<*fkZtKyN=BMVE&*Hw!rsZ_K0v*Tm z9G;Zfpi&!~VaT^~`S|nFAF({D*41Iku4v<;vZS&jR?6;D^=KZAV%rOZv z?Uz?egL0%UXW4Owd+Qoqb%6hN?a3ZvKV55f+a_405)!X>Qkp$0VE5y_63uQ5jXq3R zsDd{>pN{ip@2F$le4OTS!GqnlAULvN(0ZYf;B>SKpGVHgZX_aFDMIwBKSgt% z5e8p+LA`lj!NRF`{EqwFd~z^B%Nc|Gl^mrg{0fbWUTk?2Pz!X$3Cwce{vwB0s^9$= z%v#6!LE|H)v1bc#;yQ^Rb(y%$zZJE>Gx!zUbUg~7Tov{rj!STd_aho~X|68o?){9+ zDsbjI5Wk`*l<;X_o(Pw{(<8c3bx@}`oVHj9b?~pfIeN^cWspjVv(TEYVc!m#Th6*y zU+40?xlPtMEjlLD@{Y>=9W4yLtAfv|csE@O zQZ}|8qivLPR{X~xAH!74Y-lr64lg{tqIE{S(;f!kfErK>n5 z9M!HWp+$ta76sx@mfoeJCtdJGCSe;13Bt13i0JxBHG}I%wM2Aea`dHrilh^oR;$%~ zy_VTx>vXG$CMU$xy)fnWybxEg<;qNUfv0rV!4??Z&e{) zP$w(mju`X)lj62};?nP2ReKQA^i#A%H^wPWD8$wAA(Zs{KU^8s5Jff}pn4mqgZTBhzfS0nQy9IJS z37Yh{^9yXgKG3)(iN^A2s*jaDdfAJ11Z-Im02i}re}^pB*dk=lrDWcbWZN>Hsz5t` zg>SE0H8l?7kvRtis6bcf)IF?w53{Q>$ep@HcA)Rn@+Frg`_bJsR4UeD087VCNeQPs zL|M`iB`|!Tj~KD&=~av3fX`r-)>3T=B77QLWb^e}0oLj4PVo&p^;V~vc;AH)%PbeT zB2`rK#IbXgLr@8P(*M=hZ4y{`OAm;g0B1UHGSdfRc&C1dN>S~Vb}BE`gl`m$tMfdf zdc>v$=v-^(oG2{Z<1WQ_sT7;6b1K#}EW0FuQ2H{Hesz|le zBz=YCtgSIHpYx1#zuuzL5-e?89RH)6E5meJT2HpAdDxse%@9h-8%9M}ZVR3% zsmIov?5urPS2*`2>L2zGfS!n}QR$P>3&Vw4#fGxs_fm=7_Sk$5eW+r(DcS@0Xk+?X zEhf3*F-z_Q69!!6@FvmWWcEXpROAmY7~KG&fWj25*kLTe0|@?R&znIp$O#<@?B!-> z(_Xm4V1Vr}2S2y7;*&=@xV*1$CT2t~JyO|&%<*fpKvo*C1FOo|uHV)-!B7=v0-0PexL0NtcSl{|=FiPkTx)v{QAW(GNhflwLHECB}BuUV7;Z$$`tr7Mf0Uq06mkQ?UO!!ikEZkFyL1N6L&->bcolXpd!^vqr`TC?eyA6j2|2!1gA7W>X=F&6 zS0e-V78~m=b|6V5DpBWn=szqWb^6W}90(!FcDLB~V87ik(0M6uPUSo4b109+HWc9& z1MUQ*|lOJM@>y}cCE zy?sWTXQi)7G2?ODJCm9jgjO-Af!2ry6TAEw!_;12wN@;z_s&ZfjfD5G=$-GP(`MUY4_Ik2>?Y*7-k^oh-nnbCK*PFw$JIY?U+G{%}B_^56J2e!H zikXb->4&lSG=LphV36>nVL$XgQBnC=An)c2-rbW#ce|GK6zy}TA%IxYaeC&Hw2{xI z6s8L=wc@O_Ktf6pv(~}=-&YI9tfe$tRF}Q8G}>)o(!ocYiUmA%KM#_k`}~|;2^HaG z<|TmW)pdz@TB;`Y=ZT?yicrc+oYE715}m`Vz6N7A==%tUB5oVreX)eA44 z6wPf0I)klM_nBv@GV`|`r*b9KTECZ9>XHC0Nxf z5{zUg3$Uq8h~g-cC_=Nx|BCb{^ukAp&^@AKgNll{G}o6y71jf_}W4llAmvj9vE z0wRX2U;vJ$N~aWRYxn-sZ0Z)3$e*o~g$yPCY@-pPo#e(1LueJ9m}Og46c-RsYk)N> z7$Bicf2gmNfpFwO*Lc`E8 zh!Mtk7GH9|sM=kmpaBMZ2G9qKr+q$Pp0RK5k(Fi3F|w&=r5W~d<1dFubKNiEi^vvE zZ>oD>@F%XkE)~iwY3=X1-2o4r$WYr~vA3nbRh{lCS4>lFt8>1_Q0v)3VWjVBGP6j} zL1I^S!8OgpqjM3e*C|`EIz!F3n4}zmHwYt@c{BDRbT?;+=+^w$iQ^w&2&O;xa$3ID z>59{bg!s8-$!iwe1e71&LV2sv2Dq%b)bm9IlOQp}ZG_IGNLSlD!uZFm7QBc83e$m9 zf6nr>k$gw;w8oadp1mU~h);SA)2o^-woG~x1htWp<}?-Xn*7^0s_7vgqE~Wdf7ylK z{nHY(kTiKvQ>Ab}uMmB;k3Y+qvg5`Oj{TkEYo%7Uk-DB-Mnv~Ko=g>a0RzL=Jw><1Z;%+pkpU3+7q_+G%B;s+E79dJS z@bu%;X+wT0?)w_}d|kZ|0aaiz!MgG0j%J&z#mTZc3nB=z5h(@dR^@+ z0`oTc=qjwSUV`(oo9iauG$BhKZ{IiFjPw(7bSVl=dY77Lj!`ql`B5r~X zd+8;3x(v%QEdJ`QZom4*8V(gJ8|o4I+$$-vIyfLTryGm_$Azx=_8zN_#HrPxKZ}fg zf{6$3(6tyk-kmDIcD_3ruNGwdZ)@0r7-F`sb#89MPZpb1dK4gtC}G5%O;Ose!@{6G z_x7-8Sl9!^Tq?1-%etNca1)?+gd4q8L`CPuaRpMjT(+<|Ev}{nqejU-WxsK{E(5i# zF8yqIaZ0nSZ=3WVT}skvm48S;dsOVHW|VEsJ_+)te2Df;v^|{kqY#?3scSG&y2f2n zX}!;ep;>=V#kX>;@t%m9DKEvPFt_K=P?Vx4)e|ZgMhe9|{x3~YT;0>OeTN2?lk56e zJ7F&bLYPa7u5Uc~CzkBURio6ewJDLc)GV_DSsFuJ^CmhT_20ixr%aZF4PUPF(=_j~ znP`6SJCo2P5&iyK2Y=KRI$+bnDPZWdp@PxE1aX!|JXbCEAYY(9DU2Fru|+3k4lZs<0FcX|k4qQ{Q|+ zt9}Pck2x-vXGp%08s`GBHHNethHteTSK%yxc;oG=2$VXxD(#1Rzq{)< z{|kbzeXM7$~b2Z_Hz%rNC!^<+*Rz;cPs5>BB|0Tv3mfLF)ahH2FoJ#W4 zxe#ji&kE9B&82|LqNX!yZPKdiwEm$3#SvUv)%j83Ti2O7)Zi=K433J4;{wL;iesOqXtCuyuQguI8(V-Zv7IhAMH;(Va9P>>nU9Iwr zHGuvhOWyo%fRH#lz1N2dIhim+_X!-3PMn-3?8p2iNhaH?6$(LR3foY#kj=yrJ0IiX zq{#E;!6t+SjW{;gRM&N*Cc%*J1kg9V-g#+<3KW|h;wT6^#1&Fx?l7ww^GQI`!am^> zqz|h43ZbH44X~vN!Q124M4XXuC%dn-9aD_}bGyz#$^^w#DxCJo@Xi>>B=d|L|3a*h74Te3A_<^O!!o5}!oQ z58j=ObYVLg3NQ46i^Et(N`?QlF=c;Z#M&UZ`lwgExm>PG z{x4n0=snFwsB~kTzgE!+N>?=*H-2gjn_=EeO>}|F6{TQN#!l8lp!Vth763YSlx_`_ z1vQ+IQ8|5QFZZCQ>Kba(&Yk8W&hT+zsj2|iqrbC z3?W}06$bYoS{H8{`aghxc2lzx>LnsB6OXuMKlp;7wXI1{=T_9 z6_hzk5^8F&H+tqG>2P5nTuIsGG3!Y75xAPHfQps}5DA9;U-kdHpj|m=AZqA0lKn0&jmW`QmrdqEa zc)?_%>UHt!v8>zYt{hxQ7-43hbfyo#qqfvlq}2zo8IRTQNt#r}1={=aJAbG9Tis`( zTUZ8#5O~IUHIaEU@2DpK1Ri=C0yeQghzj-r3i;BExitm|Y{qDV&Y){xYDRX@%M7HF zaiXtUMOpPSJRAC*uwTeJ;2hEfs9Z6ps|e5$SY}`(H+*e&_b)wd@r4+e);vPDk01Tj zWM`+oRnUGUMBk48fYNk$cwIj3?;m~T>Fk1y%Hy*T4o(~v%u_OQBey~xt2sx$W~Ckh z^o2v4h!NeUYGU+* zg+*29TxPTd3B25y0L+W%5?53&ch;_*YS1puXueU)R^Nt!Fj+}`c7pn)BEv0@g%qvZ z^4juRW{Jj=T}sZ$e|$RQIuQ1yA-a+KN>_fZvxh&sfW8Y1h#*2RTtpWhAbp6^fZDeS zHt1lXQew=`HzD}n@e)!ys3UndjYA80N#h5s3BBEde{myB733lL(fgo1^G#Coud=l3 z?n+FVJc#SxxQ%a!4Ytaw{jUZx$=b=M^yTe7Y2>}bzGpY^#Tr%Ap#dKObZ4e8oyK3? z4BqoEE&F*z#mA6J0E=^}SxrH$Kz3P9!o_L5#z+^R>H{a1nYCf6P0i93(|W=3JCp`o-$#>1o?r6Q zJ)ZoTnoFne|Bs~Q7naf>Qm#|${=yHmhTJZqq31k1y2ZA)i-jnACP_72u5Owrq6`wp zWV4*(s7kiWnGISeRm6T%M9$Gc(d)C|7%)U~g%VmJ5)>0~b{dH_<|EwVhbOCS4`83; zF+$dvM70Vy_OFqHqDUei{!)v|fs4**43Ss98T7+b#jHvF6$eo7O>TvB>T9Eer1+`p z6D0p0BRiZ#ru{%uDA0p9$jK8o@~QAmU8XavmIOkYU)y`>3qw2gs$@$B<-7zx>zt1Bk4@nJ&mRP_6JVP=OYnMl-1&PbGWOUjZ4-q^vF&`Tj$$5-oS#Q z{dMJ7B;5M%@q>7cZ;H--BsG@*Xj04<1z7K}V1?eIhV4&N!XF3|AS;${L_$K%ZPNch zErUTd{Qj8Ur_5LVe5Zx_KZf{oiGWW%mtnVFgksEn&oZ>NG(=y>i=_G&1Ryv`Cnj4@ z$DP{du2dilxpdKb%1!|h^qFG)n2e^*ssJefou3d;&;`GO!JAlTOZlr%Ko)GCc*ASe zk3Xt7mTQ@d>4W|tMZfqvco6ZM;6te`wjj1QyxF61kw4QWv`qj(j|?bqw2@UKck44H zn=5GtYGh;Er>l&o98&cw-cF-F6CP(i$~#}Q*AEb^?VQ#8R>fg zlry+7Y~VgcmF=2!n73&1x^GGeru6v^yKAt+PuC{QY(wQ$jEJknSPv4)+TGtj8q&w^ z#k`l2dmmx3E15}U7dhWfO6Wsx=i_@FjZ=|7TxC7fYh;Ua9CQ)AWRtntC^I`lV3D+c<;u6<$RYN64%R5?|_04$@+V(iAGMRp|!}DlZi1Elt94K z_(QVpGUS+v%cnim`2VSVs`%QA+6jS!+ghF4`0lC~Wh;4x!*-w9lIZ@!_g)jD&P=u3 zzkHNNzt8vJc)s%FAHQ25NI@|e(7p*Si&MV02VH3Tg6>Cj&gVaD+N z{W%a_&CICpCLCNdOIoiO;V>Ha9b#FM3w~U~Y$~TAwTK)Nf&PT_=5Pmut0^eeDDfk{ z$re#QN5qFy98Z>ZMKyTFo-V3&HRw`GcAP79GUe;C#eRISCC4GSS*4deE_gBH>kSRB(xp zO`>X;<%CqS`!PJea+qdsqnEmnxeg-I@dHTFEmp0`UAu1^O0O8FHV<5)S8eAO4i+7z z<~(HTKW1@qc-V#Oeh?VA;1jUNeyiAs3(f-4@kkRlQJ97SzAr9#o6V^{#{rpr%|h;) zN%ePK<>dWY)lR$-o>>H1&qnacVt~Kh0Q^VP^{~Y_o56bSo7A$xq@`Bi1iYf z0$1kqNfP=3Ul~5xmPA*$lYyotNB#W7!>oA0YQ=}RUY3m$0imZXN(#{0i{4V60v?&H zfb8d6M_N4e6RRDF2*eR{HODN$xLFL;_|1J?$gq3Oe-zKr^HWm0oU-hEI`#Tg^bEAtku5WLYV7Q&gr&V~h z97a86F@o10?{iB?N-V_sM(#=yYaXyTh%7{H;=-t%M#B5f8Ma{c9)D)k6#Y}>J1DMk?`>*@;NK(CUR}sXtJx(!IYX)uByFzHGpa z^kNS*RI?f@jh!m5YL~D4D7DWSIf(BoyUax8uQY{>S&isuiRmbYR1l#IG*W@lalhiw z@MgPGo$R1`_FC!d!l#KUWn{QYMKoz!tJ_Z=@=Is3iJT}DBC^I#bXfkeB177s;M%ug zw#r8I)|)==JTE*G0~HII$oZog4c6`p#AJ}4w)v8-t{!o^Ix4=VxmH_Z*Q<_@OrqzE z&RmsEqE{zP>S9B}&Z3NAzGv814Q>#JgE=g>DKb|Cxt^8@$oSj+*~DQMyuk>as#!uM zHnXz<4b5~}UO2-D?Z-Bj1oh)0)@P*f{3mt^9ad&CpSb*UygN&#+)P^`1g>X-$q5>s4$!L#v;gFalEGg=e!2XRNW$USH z8%0wkKowYN(Elj+ZH09d^S6{Uv^5pA*c1TpoGj;7#`J-GcwOjwC=Ldb#4@J3lGy_A zJXLvw&F@4DGmHF+(o0@j8KqhCp1y+`A41NQ6%k&DZUJ8Qq|?w>SVA@#9GlU8F2b!~ z{J5ZOkCvkw!5P9{7>w?Tjdg;0(%c$loI38i!tIOi_Me$fa3MpSZYM}2sGcKn;Q#*m ziE}kNx8WEtOvD9@`U|-9!>19??ckWAlF(o>OBJPz!i7xx^(2(0s~Wv28tv9tjM|Pq z%*sbG0LoQ}qUWlBAI$*@ib4ljR#w{cG`X?`o$(#gCzV3&=pMEp1=yT!yQ1R`KnI;hcSXl3 zRz((mk>iQk$yGJ->6fQqN7#bP1;i%uiu!~J{@2Wz91oz1mBO>cD7^Kf;iLtT zva$nz<%dXiE-JYA0|w{T?}070`eaG5sh9uQC(iF#0>pA+TDr3OoBU-_TL-)R5*$K* z-0`_0!{)6ny>fPx0^b`iuuM-dhB_!QWr0-pqsj#9Isv020`xI4C%9o6e%Z!K#KD~# z@u43|um%#LwN>pTt3jpBA47x^|^J)j-_)s0Z?iW!pL8)(ce%p81uGK(m9GF_Qg z3Xo!;L@gvZuWLJHq`A8DUYm@oEdL%-sGBp!_jFn$Z2Da}l*%OAd5>|S#JtuCTt^KA zfo&39#`SawU6M!p$kBgXeu1_nLZBg*o?TO?=L{XR1c9b5@nB?7x5%g?O+!I|$R%;> z&dK(d{?YfVFvwTL>o)eAj8p5hX~S?E8LX!>uPQehTuVvkki3T5JBCZ-GNuR_fypTf zYD5Y&(7kl>@@BNzxr7exRD(ij;_0s`>>?R2c$8y@`T?Tpl%J``mL^}E?=t$C+VY{= zOtTcdeIMQzuedL&@E#QFaIc7PQS5i#Vr`D#ym=aPGv8-x;KQV>?^i^U)`}YTh5l58 zM|!^H;qnvQBrP?!?ai=*zdHW=#QT>5-y{Hmw-^fD&e5q~tk25I<(C#g3@Uwc~V@*xfc>A@N@HQH6fZcPRMT5a)p3>#htL%lJc4xz2E6%lyYET$HP(D)LN*p@fktKc!amY&0@WOk<0^DRUq$_c`{7qVnP| z*o1Cv@c_N9nprLJId3t*rh5e#k~Kvb!7)KqZHf#$2K~~4J*ehI17sgrGYuK)3|k&M z+&3g|C3gP9j+CQjU#JQo#4W9Gi|3b@#<>V72i7vQOW9(pt?4AYYE+msHD~7ecmKpj zXW9rSuVd_R^OmGKmkI?+1N7Lz5xW8fGuI11y?bJ3&pj`i&b9hY}H;KH(GjJGD`~KpQh8*J83k_e`GMDu6joT*wt&by2N@|98FjM zbJ?H_=@xACv&oY1a_6+SZD=a&i#!l-5TtGvWfUGjx{Mfv^x@;Uzq%h2L5h4hRI_vw zq5-0@zMN;c0m$IIzuAN_P%VZzZ+)l15J3qov{8CrMrgA5{n)dX6%5BLckv^;A3pF= z!ZtcsG~M9ha9R)_-@Q$CW@-MM)GlL!axk9ijcj4SL@iniRB)C2ek3OcL)4ouk8>J< z>0-0MZK@<(Y6%YPzB-RxDRM4_nNzDi)givs59g*zQ^4DO78M~?)0xE>YP<>RV3OU5 ztY%SsQ0RHrro;qUZ|=T%_QVw+OGxUB+h4Q&@n|6#yG32b=*1Xt(_3T3n|mKePhQ)6 z=@%aeAv@SqHh(1l^ls^vr@gEdYP`X^#EFZnjrudwK|8>{N+Nr974YH*%Fe#o58$p= z7`1(=Ub1PwX3g(B77)^RmWpj?w)eLhCG$0I3OSuM&n7LAC;YY#^BB?CN0LY%KfbR1 zF|InZu9Rc|E0XK3mEqa#qBEZXXz&nW|Cjz(oT9Ym<0q*drCxK|L*ZT!@FqRLcHaNX zqalzsxkP;}9$HD@s-$;us?a)15%0b!p*`0!oTozz9`tr_rP*|b7|HVQ;n30qmzi4@ z4mi+Il|7->G2||-_(%NMF`DS$s9hl$Yr?ZzT(0jJCJga0m^|yW&$JKrid{99pAW=V zhjp3t>J4QHT~bs0ilzt`C0qJ(mWLq`XB^}p(O*v87RtWHo};aH0s9PqQ7mO(UxA;H`%4;IM5|NhBF+ZgD(L`IKu z@D&kIY+b>>SS;+oVq7ma7jYQ-(s24SV)exOf0|KzFf}s!j??h zBME(Dt#1Vk3Wa!mj+ph_@=&_2RvC*Y0lnnNKZi>sr|W5`kTnN!lTS$IH3=(a@l7cIvwZ} z^NmrAlx|JZXlVqSH8$^q;?0SjC8wV-?MSgUd_0MhPP|73-pmwq2@jw%)UTwSB&01S z^VAsjxOWfl3aY{erHPj&^IrCGeklM7`IN|uXfVN+$`m}O*KCzykphuXIFln{xI_cE?7L3iBbiq=r zIH)<}eoCUBgd%s;ba%Q7DgP?xSeu>{#@=Q3^ze985q(LyRR}Hx1LPoGp0=ap{dJvJ z0+4U@=~4NzhTMsu?UgGF%_?B~<)mvLp3s6)-ZNJflBfqi+Fo$0N%OsUM8)ebV&jvg zp9twnFILT0v%p5w7&aINwV(Md%aZ0+gE?-2MTdTY&-3C(s3aAE;_xivN3*!Rwi)PFQ6)@JlXT$Rq^|BJ2h9yyNnt zobw;XZW=ZWd{v|KhjEN2Y5)(v9i!pHmz?vIf##2fthYnm&49)O?p=4rqT*i|Nv_o& zWRPazvNJu}SUK{Lw^He(qCX6TPz5FS6wlJnpnr(-Eu^2CLaE>IkICA4D$!I6;qVoUCE=QVGY*q7+dxQr!!kn)A45eEcoDiAswy<_$rrsHGx!njA97W^`Qr& z2Ji2kcfa=T=PKMzNakY{E5!+EX)RCiaW)nu z)MyWwh3zKNv7Z=c0@TxN5$z>Sk%}!Df#*0+$JNMg!|&#QG->r$vaXg0@f$w*8)Dcl zzOTxbRFq*vdk3_wKt~}MNWztKlbbL*TuiQpA<^dMsTo&8>?VNb{|c#ID;>U797XG~ zKGX2(k5Rmh$R|0pXEKTrFU4+|uNokwl|#wssNbksDwH;>g;=*Kk!qTR&_TVo$U9`d z6$AyN2L@9aAWY}Yro}98TQ}i!^7}9}SCDI1fMIGEVp01-rsSr_VO$VZyv=ihLuG7! zSwuMvoQQW@&RJE+0UgA~`m1@E*HT%-&;D%P-TZ5+srAXifoqeA*fl&gokLSkO_w=v)?E=Zj84g$Kk@@UX)Z}@Gzoc46?2`Vr zGX-f59R7xUmnuthCf~$NH1-F(pOMh`PwVn^)rns4*|hKs1y1@29K2=NacW57O=X0! zx%8bj{nir;bqX8@$-4@8IMwN4K`|)CE2Fbn;mQ;8!D&*#31#{+>XQh%P3=*E4&D8FjNA8^lsFl7uyOwnkg1ZuCoo&O9>N2P}!P8e2C|&SdLhr^~ zBh>p<;?<-09?j>+E>wZ{Oe@#hbOv6eVV0p~hzB5%{vt@cMrI>yK1ZdRIQ?6lr*BLb z_c75bL@eFvsqyO8NhN4&cB=v?4gAp5N{#KbJOfet_zWhJqB^JfyANtJ_EKA$^g3gfJ8S9S2jZ(C0xGF$5W+ zTcU1aEDgQvX7o*`3{BZ(@l=}GWxqLM14_{jw*66^rbDcI1nA7Fj&QEmPZcf8)JOqp zoT%0M{3b`FL#8DZUQ##ik=-DhuZw5yj5Ejz0Tsy*3;v2CU&u_-4LlH5_06S?neE0_ z>=DD>)duXWkhxQ_!ujiTZ^PlGMkS7_J#8Y?wHbl$)oOj~k8*J|E{C~f)lPUz=>$;4 z2^^ZE5VUtWcttdH;@ONS^@z7?1A$UXt|FaTPI%z3aPOvv0MKrqJLcjTb`ih z#;Ilx@B07RrnXlMh!*|}wYU_n9Yw*)zb%l~Y4&r=7baRz8J>hoifbo|sDk9gQN}iH z?nil?l(+E*1ju%0patzGpF`e1{mxY6X>%X8XF~NT(-^aPJ-+Cg1RGqhPRE%z$cH}C zY$?Zs8YMD2zltaZqW@-?T!=&ytKI(p^9Y(vE<2e{=(X1Rb2?qlO$}k5o$vA9v&fx1 zZ#R^~H%nws=^m@MT%A9gcQ4x`u7E`78MBwt#Qpf%Wt){i8@HY--wr;05+yoo0-cTZ zzOps>LD9t;8r$Ts1(SVAgv)mNt}KS+?slH6b+Z*2k&S;hC=3p^SNb#i+S|}!WB(|A zrEu!zj%zJ+@<9jrW|7x*Z<+r#{M`wR!=>uwZ>S53#xjg|AP&F z4xzT!8b^HB!#=GI@6RSXi`O4)ZSN-05O~4xjV31+dkms2l+wcH^>x19DQ8@qFX5-7Hs!0S(MGMs^_TL{E13-alsCpPi zN;J`4KDW6~Rk-llVZDImvSf`b^@fhG>c$+=e7>n0Om9?CAyD&Xf7b*#Y z_=k+a5C|=bj7;^4le?!cTOumg3BB>TVwTp=;z)ejezz;!DGhb{G2Zz7%Ta} zWwBe^gP!A^!IU39#{FSiYI77n?TX}w7ukKx%&b_C*RSFnd@IZ5*-)&u(tdWqv1t<9 zU$My`)YUIoNX&iHrnS3%34@TP&_AbDog7wG%UMApS|Rxph}4(2c{Zfm1|n}9tp$_Zb~duO4_sod%D zWS6+YZzX0?I=atviZ`woBL$-y1Q&_9T<~3eU&r44IGdk%g1fEMZO{-ML zvL!7<=m>VCHPHb#no#G&$W##f)-w@(PRBc%NX6{AtJ;P`l+P&aHu#)Pd?|T?m2~KDj4k{3VL8)rWS~}$1GSRD=aVCxp#GR!tGFl?HQ7en($AWn$@S0~?D?m8fIsW9 z#tgB8)w0d)h9^g=5x2rZam}FR)xCekP_~05KLgHz-cGWN%`jEw7|m!RB$zBC)p{Np z-6$L{@(u2W0~>yu9HjflwNMws4&Xaef5O7uj6dmB<1*wwdt#=pGprVBj4qNs{z9KZ zey3k=1c3$Cj8ga>2j#z+hX1LV20AykY zCKC9>_MA>L1>7tI3_TBwzGuVk@M>Lqjpvi~Fd40uvr9GDVu(7He>9hv7k~}#b-(qW z^?VFcMYq)s*)+td5exk4xPbwGtnq<|w7C(nr`#L=2eO6M&!Pm4k=&ad;W@w%RTAOn zC{5r~m!Pthv=LQ}Z4LjP`xVD;8_ybp7~!=J^yF@EF$PU7FGztSU{Mr6gf)|DOxc)P zqOI~D8NnMN@WDeW0P61I8)-^DUjixkaoul?m5H%^P$wtEvj${@dBi?vUP{HJV6hGn z={it!?D-`6^tD1P1Y+!kgZqq79Rbkpa<$) zQr^y&GK1jZ#=|QXIgOZrCtN%9y*#3&u*U40{ASeU| zN5}Ss+?f?)`|hUyFSn4a+slqRaS*b(Qs$08V(7`KUhavIWR^99<$#RWi7#?2*0)L2 zO~}kEo}olSRD7tCcfBJS3ZF6OjpZ&f=Td)rmU=fYoB;l>b{Q~Uh}HA6U3Jv+FMr(9 zw~VLlNYWJB1mT@3duD+#HHAu;dD3iow5NW5N1ccOy56K_yl}o|sMDMPx#*t7{T)G| zoQkCm#I=EpAE&4%W1!BGhJdS*yyBfv#L@y5+VtIE8S0kY!Pf%$qD7y~5{Q_ObxSL* z`ReAJwzJ;pqL+BZ*l*mTBqVxp88J=Pgst{^yO>^nhq49it)}8W+;Bbe&O`37(28G& zJ~>|6n6%Zm3 zRsxy@hip1UvY4?BUt|Ou1u%kZZ6O{_yH%De1jU^YtviVh@Q5w6bl|RJki`DFwYf;--pXU`NT|9V2%J)T)h zSc}S>&{Hg==yELlP50CpyG&`(HsVL-6l_Z#`RwXm+4 z%}1=zBm!z`?GTZa!mEAhjVpEMYRp2;I06sCQXgv{b%IWt2p=*h8j*xAJ1kxQLFKsj z-cV{7o&BC|bt6sc!O1123nA%bqgvD$LfN!=G%FZ_;i>BvkO2>`AcrlgM}ke=mNWf(DiO8%+V0X_O{m9Wb$)|F0_ z<-jVaQv@FaDgfQ^JFUWl%94^z(gj0=9H1TfdLt$4z$jQVOkq_frC*W2`4J=IIys!;Y>@XK8^*DDNEc4@CAu<4kt zV$qYi$+xpgWd`eJtP=b_(!N!cHXdsdj?xW##e0(BvQoyC=Hr!dn0nlbp!A}hN5f#> zPLmvkhlZf+0=h#H$?mT|PaA4Q6!F>@6Zmr;bO#=iB5@EadfGBOTg#ojV>fmI69tI} z4gru4d%B5JO%2EU_L#7&lUcJ^1R@`TT`(1-cnRc$`eIx0c@pv4O);7iHm`)~j{)M^ zr7`h>LQ(;&Wh$2c9iSPMoC&RiGn;fCIROyyeo(yu)j-NK(wB~j zVBw`0g5#+e{WV~ptIAI(+~gYW!Tng#);@LVvsoZ5uhpB&>sxT{l>^s6B;TxyoQ{2T z`bLjxmVv$STxG~cDTt+07YFyT4_xE_`GJ0Qndv(YfU#a8GJ(Yio9S@2y)>K5tYs=L zSExR%JHhNG8T;mY-pGyXGTQh~+xI4u82N>eg%P4$oSp|#wO5Aw7cxV4xFdg8m}v}T zKk(kTM@JlXbJDr3NIMNY9Azm^$ST&kcBE5%f(*fx!5dy9Sqg>@VeQ#waD+6#zkm?D zV1N?Czl7CC!W0mx9nX}=d#FabC4g5#c6Rgli*15RAjb74`;xXV^B@`#2|{VSp|34A z+)}*fi`m>rNaU#Hvef+)*Euu{<$zv}>^ zxRSZ1eA_Yk86;i%p;WPxN5u6BdX?}Omr6)7Oe!p342ICE`ylo;s2+>h1G@eEtc2J&2`|my8e6}$f-KI-0W!XD>1{FuDj!|&cAUqb5tHUSSz-6t5;r!Np*RBv~ z6FQrq=2{*Ju3HW1BjKVx;lebox$U{rR-pt&30jmxv8iNG_tH9$&c-x{MiIiE40PVA zwn`0shJ>Qjq?CFED^-C}2t+2ZAH|~g_AE@B$Xgs{Lv*w%cs)-|XzGsXF%O`H>~)YG zyMl`lzpMDxNdLJGip&^OlR2`M2?KTL3j^mi^ED1zJo;URja5Ntn3TJF!J?>k0b(j? zfTecFc)>2IwJ*25_8}LSTQ6<^pqpA$`(Oa+FHH*X;1!09E_`)XyjB*2j!8SQ7F>+F zVj2eM=)u{X#9l4P!J$zo_$Y#>mZs8_{WH=WnR2i_+z}%)Pg*{RCmSeyshttbviGY+ zf7xf5BFAZXgUt^qww13Q1d_TaA{G${R&Q+g(Ck`3(({xAJ&U!~9$~UxM3gK*{3u}* z3S`H>r#zv&)pbW`kecI`(%SN!=DS9hstmPWZI;gc3lS-PTl)U%(T(F$oGb>uy?y_6 zKTR%Ny9ZWP{38Q9{eTL+DB?M{z>Pz^YNHuE?!PANz?o)JvoRUA7;~$ZJ2%I^Zmrx) z?=LbyZzQh1L)VXjvyKm4*JpMSd&57Ply`MvQu4zLBDGT!Iv70x?v?Z}<7b&>ip1MG z3YXy&#D$b_;8PQP7XFvz4vX-hd-VH%judDrx3>`PgX+l*{x?XF%*Lg%kZc55U!bGa z$=Z~?MZCMX$HWV}6?5xN|6Wkf1jYn7RYE#jna0RnfyiUo9b@H}EyleKdR*tbOI@c` zAXlrn9Ge%gc0umPtNC7tOCi7}qf&*$x#74UhFQ!&e-em3Ay#h18V((S>6c(YwcHlx z3q3p`g}Kh~?KOamOlwK_5K(g;8{_r&@5mD*S)upp+joF?{5Fiye@5sF=;rinwwW$L z55xXzve8WPZ?GIo4yj}a-NxD(50OY#RqF`QY7R&&)ral8N8mD%{UoqI)Qu*W9OZDA zsX#c9AZsbW6SUE%f(Xk#W&~7)gBnvnlnJ({9z!`=pJI%2FPFPV3%GjbpePC2!qc`j z-#K<$H9N7?DLRa&FAoaWypRxNi6Ptd6kS&PpK?ZY;^f3mje)yC#!M|-$D!^eB zsb6z3i=)_<8NX?MeD<@(p#{_^b#D(<@QLhX-0@*|WgVW>yI?GI8c(02W*<0c269INd6J+xJZJKl+tM>h z@7ED@YLF!NB`n_ociFLCQl+ks@&voMm~*b_ziMEU#CH6y2m%Q(=SNw<+aElF>~Zd# zJ6qD)XSgoi&ZT9Z<-G}oN=gtY<6yUkNl6C{>a`F(R1#aKG&_&0}OnrL~y@uEH3TdseZSbd)%)H zas{kY5#YOuV(D|pXzU<4vE=|2%Sc<*Yib^;^D|f;0n!=6^3l-&6TM|wam#^a25uQ^ zo*3v^F>JA6Nu)2~67pvjAW8D9OmNCrbAbF(y4v0O!RAU6>rBLm046V2 zuHbkKvdO!TIwWE ztuzGfYy8z2oe#RxPbK8`1ZWOYg&u63@LWK2=Gt16t9UhPvZFUafyl=K=jQL9zUA++ ztTzg$42y9S9oK`hVr{qP;!m0!AJ-!;(cdUSHK(Q_#(9-#Gmu4Be@=u45d7nF%Z9DD zh?bfK#2`6Yz!g`Mm;$`ISj3b{T9N`^qo^~ChLNEVk@+)8erc2m+IeAb<*U-;n&JF` z@l458wOoR2hWI`Q-A<0zvk215?Ml#}P|;DD>7uzB+(U6e8yS23id`6sLs+-mJzx-f zavg3&91edKoOJvUfI_+ylc>kzxQe(P%7^MTiCuAlnR=2uy(8gwU;}WUvJ)5?G$c{A z2WmdFjnxehmemhvtF6s|m1|oa>)9K^x8M(LE8kQQg3jnLV-+p%@tO`XVWr-7;X+R# z@uHdwhy|A)o?oMi-7N&(U*c*pk%WeI1Lx{)*=N=a&2v+zp@4si=ib_I=bA3E0Yij} zpw8mX0?1AOH1ShMAQ~+2K98qJuF`7jsVZJ=q1SR*!bfe+e5PQrwj;OvQd=53EA5n= z9wLOA3TSA|zCOJwa%SvjmshUGk!F_mtbnPNpfdf- z2hnDKG*vZA*z+^RV9b?MX(z$T8BiQT_GooDD}x8Eg=B09(1z*7&3yVuYF%~^f(b8e ztkEkQ-&0v@9Q8mB&pW&!&s~kGL#88709i;T6UuSxvIbn`e`H(|Bt0xb16N&_WtX42 zU|`ub<~od{PG&%wIT;GF*Qsl#nm95VEwq{)K%f%iZ(@<#hv~VUK(FiqV?X99vMlf~ ze%I0$Vr2X5?eRO$pP&*Z+(RcL*+sGn~!%W7uLbRf&B(HB})kBo_S-x>H1O@#X)4wU!@Asl8KCALX?1RX`SM_Ud3?RqEgH%^*-XE2@jb94UQ5INN-c z)`i;I_i_)2;8Qanfip}Yp6vxzi;U+C!-#x^UBj`=&z>yX5jG&N(YU;;Q?EQg~W?Q2bxFKVw8iPUM~DJ*a)%g{yB_tp$`>-M3!y`Zi9$jVK||? zmABeAlX!D7nQLZZ+gYWVjL^{6iLc*yd$|ARtG)-B5)zri9iT9gcPzA%SPN-$YY%?g zz5hCp`KQ*m2@6IfH!hdLWBUhBkJUlY&(i16&nkC!WC94De;0=(#nA=OB@iL%mz_Qq zV=s`DJv~-=V>id;2u-~!+C6yoF2fg|VaB-cS3F&#u`(xF)wEb&uv~m;wu)vxPRoyl zhhIAtRo@mVztL+qDOO67I9~{;K2@5K#I)|kEgG^p!zUnM8Jn_G`sB{F8@lqq5h3&3 zRQ3srm*DSr=P>Ag`yU14U?wIa71sxsFUcKFG&w71W-@mAM@c;<$~w%uL`7*40OK=F^4E5b%FI_x;`1a-DUN&)Y1)*R2vA0dS*Z%@>wD1jeFHXQMUGW43|(R< zF;=dyHclBwiVQd)r02257kx9A9l@=e#?JY2CuhyveCv>lhNMzEtR+#~^mkVDupXmZ zs#1g{Bnvw4gA*=Iiwd4IyHYvF6Yr-yvA=sba;B-(*!L%02zBhvEM@0_O!;oqQ!HTCc z0rm2|Im?(#V!r==?q1VA4f10#l$KM^bcsdC^Yzi|4K-eFt=DU=)Sa$Brh_fx|LC0} z8T8$)V#gPk!+e-`lVa zh$RhO1Q@GYl?@M+tsfXmgnZLzb5jpK=cP_haR_uC;=R<}AO&|0=>KGBnlA^&09DAm zd*x|T5!I@d<#wCwN^a3G!`pq_msmbK6zmR&2KW}l8B5M(US?5d4tP%?wF_^#Yw9GKM7NA8KV>Twu$BrMxX(x$?InNcW5Gj> z>G5`&!zrt$2@AAiiybId6RY1VAdiz{=38JHGIr~`-_9tM#AN>Qyp>FH1qAYh(N)KX zsTRJwJRtI=G^m_-x-CAOO_6}fE&SuVO9()Mu1zzL9UpWgGBH9N9TJgJCj2w=ID~Z| zd|isOUCdjv6{#G=2nKgoLAn>+LXxJHvd6C=tii};5(CTH7>}(buMWo_@`w;+a z1HQv$%rP+-p9fQeBa=Zo%Div){d*k?rc43ZXhy~`ZL8Z8v z`Zlnx*w3-tR7>#yVceETBI1i%=Q(1Uq&G^DG3fYRKZ_=>GEPi)F^7Nrr9R z2N|H5V#8_RyBGRR&f>;hNZcY}vo(Bl^F$=PSKnn&FMA)BrG@oml@iUC<-zk}NEg}w z+c|W~eIdZ_VN!9gads~<7(yfVX`Zy_b9jnEjQZS0r2dT)2ATgk+mwpXq(&_-?wqzX zJ`O-tBBJM*RBI(C47YUHi%hXwnD}^x5#Z}w13=s4^HTvlNRF3FYje2ar&Un=&ov&mo}VaFpL)O$bC*p;`OkfEQ{I(IbRNyxKl6!4}7Mmt>CT+)0|$UqT^a$@ww* z6~it(hDj~vqSgRjy3kFHZiN$8F6rWt$ho*%BZ=UoFVjS5knMT=?r?mX-fbk@3Nfev zTbnvaMCp+3F-q!UMVZ&u7n0goM&T>JBs^v%KWp9Zi#JsWF1tPn&@tMAQyG*VC=eUK zw{h*+A#hkyPo)QB1I{VEnsx&Xb@&^>7Y*bt%IwXx^Tv>qojy?9NuL6EmD_K{ z1t!oMi8k}0)Mt^V3Nc>p+Vm@`FX%EH^-~JZ0tbXfAWfHD>ov1+Hj8(3>BuF8bNgFI zMpI0Pk3Ts8m=BM|Z}&K6r4G76{Pcu>^{up+VL)t04n-qo8|}+DPx!dbB8y?&!GcTo zr6Fu<3&XWMxDUuUO5Zm_xpE}x`W6k`1Hs(SU)?#T_t@Na*7?!b0=iN#Te$$1Mj*Lg zHhoz74eMQ^^lwT3VvcgH`{3)kZ-R@bJ9w-8A$L_o$r2v#!<;@V?em!^BOoW04FTZB zZ@EQD@>Qi?e}iB5bR_D8FKO{UQV*q>!=4htq{W1ax{PF6u6tGOCYz zQs7Bja;%EMFvFGypJ%_3b~`V7`X=xzxJR&sB+_e+TJ_(d2ibNJ)iUxWiJcZM^q~=QzSs1_%ROsmR}fL`@qXH1`j3dlL`!t?@ke9Sy<# zHuRn;m|R*-=>l5~GK$k+PuGwM(IPx9_x_3+N<)BkC{<<775IT&(BRy>M%WmDEfx9@ zNXW&?q4C6jM0JNLl(H5e&OKqD>P^W=Z}!euG#cURHnC0Or3ft7R;Vpkh=LnZ2!myH z*o)l>Q8AEy0{ke~h=qKfLlT0+mwY(my<=@UmXR2ZqzzKK6}*Nk`_4BGSS@tEz9_ zOwab_e&so#aVSK#n4w+IHyAkC=7%x8*vs??IXQt{vVG}mrsY2wMh41H&bm?a$6qw1gK7Kgr_;{K*+vog@K;D@0{>7W4~F&@ORt#iBDVrB`pU~9om zG;b=b$nIhl6Y%r&^j)?kK!nmE)xx}S6UFSmyc#^deEG-1<7#p@6$*1FWDnm1jB~8#B(3 z4ma7{DCU1~-}ILjaI9cskB10rIn8{!HkTFNu%#zro;Yyh!dLA%<(W0irO&%=NQuSR zhp(F@56H^2N-CqV2Omi5)6g6CvoEJ;iXeKC;m(AODbgq~heP^UMWfytVLg3nfr+&i z{W)Tj@sey$me^C~SIqW_l@R^B>n&@i|6es+p4G_=aS%ZRg#pm)A4D39^F3omD_SH7)gpsrxkQ< zHt)E&mM41dlG+@O2)Mt@=zpiz75Lq^<%D{!B3xVZz}0}#KzzmaVsw&$l%GG&HB{e9 zOlg`1#T%yg>c(D$U{zcb{xVcDf;;Rlg{Ctbc;Su-6|_jz(ugc9MlVQ^&IKUZsBIMS zFxt#Uv;PeOX+9p0=6yxar*K+#`fe#dTML`RPobCJ0VR1SY^`qUM${lviDHgYP!}O9 zJ-)dKMCxZrViX3g^07=Z^!%<(b^I3(`n+z!f&1d+*_KK?y^wW1591vof!wq95C)va zYEncg)r9$;8Tm)nb7OQlfm^D7KB@$+C6NeF8o)qjU5EFl?Vl; zx*yv@F2|=DPxJMTqibc4)1Ij6n~1T(+>Qfe8fMQczpxRyo5m5e166W+56T{A*8jMH zdlo!G^=q1WpB@8>oAmpN@4coO3C@H+B^z%upTU$v610DbQlU3}0`P<`mUF6<_pkuL zkIu)g7Am9MPhcOiSA%`>W}?Zgt2}Z{aWFx>U#m+FOj=39^Y+IAT=pwxCOG*G|4`7T z;P$O`V_S2zoCLCGfJH)haPj;4L{=JWr-CQ4Gb)_-HA2`=*GiW7{RR&H|IrSUY+1~g zSG_ZA2k#=eY)q0~#P@}5p^lC0rPcxn$wI(YMcstz~ zI^P$GHi*SAany8BSK!(M`>Wa3fnc#~8eoo6eboO%2C%eURoEiA;en$t=4IPBQD|5W z>KNjdN%Cf>Qr_37cShj~CGw-VQXIhCMAT?*8|Zth<2Dl6mQF|;@2=et`e(9X(G|eC zI_;*a=fHW#H~s+&b%_Q`gmJ%%PZOu|!Q+ZWl|$ z-Ej}Rbv~HxAVWGti#%=ZiqjEPGOB+}26^&V)7soA`e-PVbXbtaO#3 zi$=3Q5NkY1o~#87_W3~)ZNCnM7Pz*pITWk3Z_+Vd)5VQOwu+LlxS`*WYynna8|n#@ zJ0d~8JytKJ^XXtduop202J4UQ%0wpxz_v}gsi2;NzAoOmdLpmPCKMJMtLgDLT>;LR zw&}qmTHa_g!vM)_KE95k6BWqRg2mPN4Fv^S*`AWj?yNtYw05M{1csApDX_UiT$`V2 zZ_j)nUB!n^dppxv7l3!LytFW78c#P#$34}qM`kA>X+ukeu}gWAv3gs>1)?F!uC3$~ zMbNEc;|bV6U)$%h0G}NEO*d3Hob48CJN#9t=l5HLjZhD3`ksIG<=bpNAOB=>Y(Qh@C-Xs zZaDdobZJKSwyRnTJ`Gu1K^WNTuQUvo7sY_3ozN}P{St(SBe(#}Mak0|jqpVGaoI+mNL?4(4zVl>s2FzZd^4a{ z6ewW~--#E*iH@fxyu10N^*ah%*}SKR{F*8BH1TrG;kvba+t$%H=M7glOJrKSPJXjM z%4pOFe>P@nteXAGGEMJ2{TSY1y{Od^x&Z+3BCsM_Whgurj1L(>?uU*u56%T;YcR2V z5fFc{h86c1hvh?x%t+f-|o0p^d52*q0 zamS}7E+oazzaj}p1T*>ti6L>?FLw!g!dT_Cz=$(uRx>B771$c4JF_~i{7aIcd z=vtFkV>u$VEI4yYAr?ivu#MNU;#%omGKSy95LuAEACi=M)MEQzzC!iXx5U;v>GO?` z0dqLq?4rOKR>T#tSdkg;&RLDesh)Fil*c1q2wl?)bnRux2m973*Ifg~q@B2%Loq>X zSMJJ!n2!?t14q>&X^*zbkL#^sN4oxy&@<-5VBZaCbR*Y0_7%LsQf;VNk#=Anb1 z;D3Jwpqq-nUuyj7ol>#Lyy#-<;-Wd(b!1`Pp<%O$H$Q#*bEnL^2|mtUC7x8@>f3bi zjs3RKSCFSTI{$51ihTa&!XS*9W!0u3N4M>5w!uKRP!y0WU?cjqypN=w+rG{f^=QB$uqg`92OHD@Qs zpe+g%MX&b67=3DzPx|2vcSs~TwzokOa1M?zSP`)!ZP9_UE2eX);#}^ z@d^P_R+Wu`=!Cu#!L6N)0dPFK!&<1MtX?V#CgYpZLErUeI9gB10(ME63@mH+r|DEI z=_NRSlt~n>7xx%EwR(Oo;iBP6n#p0?ha5{37B@{SmjZ#t-EjiL_hm9|^hd|ZPq`O^ zBuBTdsL3k?{lB~kYY*;TuQ zXgfW$lS|Tin9&LK$A?)U07a`@lVJJ})C-jI4A2||aJseg`FSRW zXAwrbgr7KOMH*a}Toq=$e+PyQZm2m}m^9|`Sr`Ffzl1Em0|pYwW@guJf}e)X(&OKS z+&a5)~7QrV7y*dz5CT9Uwq$O{cE)#ArY^eJ5 zMVE|W)l6a%fz#(9p?o}4qO&=GKcPJoGw;gE(8UT^)&(X$Q9sg#nmcd7hXHoTtXvNX z0FX?zQX|t6s!-kN*2af4;SgU@2jel3FR3Qv@5#cfFG@kDaaU5V!wnPu>Vo%Ky`HU3 zS6Q&>I5NS=Ne5YjbyUYHuS;iyulT(T%g`G_cMbVmNg$dNT-L&-fi0E2F>a`@^6e3@ zu$yK`H8~mq94fxOhEJ92SE)h4IJ;reT$!rCmio&ju0(FTbko#H-dmkD>&~Dqk63hZ zxW_O(aQb7yAHn7I^`mWY*nD!0aT_wRqY`n;_8x{fBg_aCzi1i{sjvkCKeFk4xoWX+ zHHn`2xv}YZokTiQs`L1wM$-VKla%PSjXmyEHZ(4_2~6nK(`{5Gq7>o5=nrwSH@>P!2T44$)ra~aj6Ks zLcTqUvU)vbvw(X0@jL?iXAH#)Y6;gEAyO|=LNcx{BMIb|m8vXasRI*8xG;i{iZRF} zUXgDtN=9UN^(%$j59=2rUBn4=ki0Lva=lvf6F95?HG>l3&2w#I(?}avv()J&mKYBM zs7oHBe3mB_xtq4t>p(}{61W0aW63`n(1U*gnz@(Nfp!)N;O*yfTWtLfhXzn>QzIu` z#=CT~_U2V`)SFW<#4je_d|L!r<%qsKTx67|n=>8_IqtA^S=YbX=gEw!_WF32__K;( z!JF{2-h+u7>OMZcXyD-P@WfYP0R}ZcVrcyyFf{B;DVoU(xb|8!nL7Daw$E68qr80c zCDlbCg(HP&DD4et-DPR>wme|Ifp1?@yt%TG9j)J&A_q(Bei17z$pqpBl5%Rpb#aMf z@AFt#QJ1(zaI@;eI$E{S%m;vzU zYl#=5x!pKn1y^c_lo7H2+ZGR7g&{S}LlXfCP7vx{<)w7CMDD)gNXEqC8nLO#id^(c zR_p=*jLh7J-m)pp^(gh$DmgV5g-OLClwKfBW@+)RthfqdCRd9DIMl65UW0sgt|r}K zO)h^Iw~D6KFV1X+{KM$VfYz8zY%Sn{0<(rkd*l~0LU+De`m{D}MbvHZWc}gNc)IiS z#R6w{V{sw!^eV&S<^Gm89UgwIM-w~N_I@4!^)ZonketINojm| z_Zss})ZsV97u5fY4(8GB|FHz|oZR>^<`_ik&9F#vRcDzuPuNJpFKdvPaTHgiEWJ0My;c1V}_;br+|#W09rRcNx+iui3K; z80Qxo>h8h(`4(Eu1-fGlr{`anbdSePG34J0DqV47fC}_h&R!X0Pw)!<;UYKTwV^3e z?gmtmx5LLu$?JI~jo2^|4dI5@pdoln#Qs!mwotx^B!#2vV#pFB<#_9G1{}ATDAT@p zL|*gR&BU@FBlRfJ1P?$WBKIhC6s%mlikk*iN&K#z1B=58*TxstlLU11YFNv~^pz0W z(Nu2cX14X^t0&OE9cuhquhcDuDJZ-LUt%fVfn z8+;?!9MfI>UfIr)hDTRVpkv9onZRa-okr?Dr?^4@2Tr@4{cogFLcN8v_6up?W4Bb< z zsUHwYVgi^kverZ#ia-HS0W=c)%MHRzjEjZOx%Hq3Y@l}nN1jKI)OT?@Scv}jah&d? zIXvZy;$TQ-nrs-6#O!xKMz3Q5UGu@e!`07nCLx)(T48M5>FUB>tt4)m7?wN;YikH? zLCR@1wy+gL*qP`9fAuPFc#dx&PfJ>BMuD=+PefzrPf0gAYyg{+DE?f7edHVt4aJtr z@61MoNv*c)Qtw0hW+sYpREl25N|ugW^_bw^^>Gk6!nSqfb1>*IESB6Ei>+-BAqN!Q zLu{q%rQnLhn&(G_+l|4j>nzzIz6?PHS54{3Y?1ApianxcBjy^yx&HNfTHkpDI*Q~b z8l-&jQY`r+9~(rmpMBN09ce%}b8LL`mBogv#Mev*Lr;qJ64#icW)aj*+78@0__%Y;Ip7sONZizCY5W+J>dh92 z?$=)Z;`fU=GXIVeY8BRMrd94ZqJcfCcOy*uKtouW+1OU#|C`s`Tx0|OhbI9cO+`9U zP+n?>H$nP~?RBAv)Lw_BWQ35L_*YFC>I=&9x2His z9a5L;)%`Gx|6E+sx7CmZBP521@=L7q=&E{EtP+fyir!k(kd>T=TPp z`WHJ(su%Dl;o4Y%#bJ=Xf}rGUCjt>tDqe%lcz{s~J^pAdh5z=YRZa6^Q(8R>P{_u6 zyYzMY>+?};#bxX13AY!)3hY;+xre>bFCCYlIXSd^%JU4-;9R_I?xh_ibP+H#dn~=N zug}{4+ls~j=x*JlLb@&(NFCR&&pAqZGf!e;_&XvwyZ|RC#lBysN-pCzsd;|>Of}oj z5RKQcH&96r#D0T;Oywndaf2mLTH3vsw^iqB|iU%as{^YcpN-Db)g2nqj?!lN>>I>!eb}EG)rlQZVJf!JX-o_;mdJt zHaB-CD%7Ej{#^{i_%qs4SyS=IvUl_%cD?zx`m=UQBg*&Pmm%^p{o6VVs&GU3f8eCB zk|@0qPn#J{e)n6Z^sdC<#1I?}F*+;<#zLYD ztl6Ai0nOoZ^nFu%h>Ub__E75nl-CQ3o&`!$d z8@A0R&X__^(8LVNH(ixLLxV&?jfS;Oe}ctMWP(KyCIXxHe;veJ%6M;X(b0!(+T6Wi ztSv;No$qya?k$7>vn)IVC!LrB1ys=?B1JO;9eWA$1ghqo-?<8j4WQoe=Z4GZ%7^)a z=rWx%$wH&hQN6^i;*BZy5zhNBHgCR;e8WUQC1tnBVe$>6<{+KIxF~65($qvFCT3IT zrWW&Zxru@$zwQhDnSpZP*R%=vLj_1UUF=|`=qIxnGNFFM-^zkK({Jl2YwfwbZNVa9 z#G&|Q_Cp8PoahIw`6jmM0|l`Hj?lni9?@`Vs;&(_J<4;rFt@a=Vev?sB}Qi@i{S%vkbKhrYyqsRjO`lI7iL6D%T1OUqivVYC`a_9*riV_m+j^Q`@VXt zLi&{*YZ!ZX7oG5`Cz?KFxHPftAC@rY9|&=2*p{SH`_atsB z_OC9SF3^&}AEwawLArUJ^=u7?LkqT`Ef|llJ@qQwhp%` zFz5kKRuVthbV9V2Nb_c_jddIv!yFFF*|d@Rhd#Q?Lh<+?fnFWLcZ)mWk{DGX{?M#s zUfhb`I>jF8(>d6Sr&mn15D8HZKSlArfkpYB*0F1pK-mW@We@~b31)obUkRGg9p z#c%xvt1bMnN{wz12fK`LSk-D{$(aY!zKOG_n$Hekk9od=f>dFb-H3jVeUZT4{@+Ye z&_*&(Ccofwc4@s^l%6?&>~zhO#VMim6*F>=(T}0gzDt z0DB;*;SDe6x6*@Iz?eF}-xU5A%;3ASVcUFNi67)fYiblkX(Plapy&wxE3u%8D_lkT zusJ@Mu`Fcg+@`z)+G$~!8;L6s05*Jsp!FyTwpLdkS6E6ZRHLs3K_DQDo&}-+pKOQx z4o|}6r}LYwb2RJF3G2v$1eiN7OlX=S(T0%|FIs3@*6X}*IO;vos4%pMF{u3WHGXJr zx{WblmwsPqm4fO5OIjA=M2@C*70GNh{y%hA83@=dS3Yccm4hKR)0Ma zM+oo3NP|1^88y=tXWTg{mt5LQ0ceOII5GSR)$)OQIOI zY!;t-6i)|~)QSKDG^wQk*Nfo`Nw629Llt2jg4(Ep)7r7cXgZ10zObwx#X>c{`4m-l zLyoF|d{?T!^f+?x>mY@0^>t+3A;Kf__DOriVJ9y!KW&R(Mc#$M^w#!j4x8)7%1!`D zF&PbnL2KeR7aS*0Y+=$%jyJ9Nt?w`IIkgoUOD`4Veiiey&x7=t9L}HeZerQ;MCBd4 zz~>U!%Hj?=P&4k`t;7$irM*cVvQ*CWlSop4aI@f^IT%NpZ;#-=4OqX@K7%$>L`o$e z)i-IXAyoBABV@cShKgIF8Qov0@ntsk*y0sn$m2WVy;NF2QmoAJph^7mUj{E=6|>LW zhKExImA!fvfRlL#cDqpB*uro??hv8Rim=99%SY$$4)_g>*@9s5 z$09FNa}Z%}&*$r#td5%&7~5V!vzc3*qi`wlDcA81J|gxylx7e7emV$KgW(_b<+7f0 zGF>n+DR z9*+9z!+P&l^5@@@i`7G>TzzyF%pj@f9$`_66JL-jGGO1mygs7_>5F!J7^<|{%TA3f zQNTGcHd4IVFtGH1VU9QQADefEdM5g$oa+Uj=qW1H^`W)ivpz3y-NWnja;O45WJ+vp&PuP{j56bQ-lj!w7DGX?*L!N}~{vf5W zOxLS5q^qJ^ejnE_Hf}k{TpNM8(JVOziFCXH!4@350G%JJK@#?7tZN|XOARvKbDMn1 zW|0cZ47>hr-s)+Ec3ERrH~(_&Ibt~h{z+IT1{IXYQq^?O0qwGyZVjp;3wRI{j$v@Z z4qVpyi7YaMO?U;sw{ur-q))P=cxT(v4Y9yw`Wt=xXgl5H5q$ySc4aI0b3cV~Ef7=o zk|1e39zBE1ceAc}?K<_Va@!0enfnVK11nkQEa>|X;VZi@)6Al> zph`1xeVyzNW6^#ZE*x7+O;D7dlBK4b1W@&{s0I2;@T1F!1l(w&}xjs&Y2_30-!?Nm?ZV6i(Ga~21sjs?qj#3Cp0<7-(GXnr_Mr@32g zTZ@kht);p+eN}S(|FxIp|C`?Hy!!Aw)EVUN#qX-&a-BI%W2Wnx3)$hH-tjR_eE|Ca zNDQ>Wp~{Os6s=lt_Nr#p5_5lv#k?%Ip?C6uwjgVlD*!mn3OS`QgT zkdb5!wpG^)LL_5Lq7cbrkK;)g(`ntE6^0*6>iBzr`&mgTIG{7p`q%!WK{d`bEZrVw z3Hq8>nx$b4SK`u7!iPLkKU{3qcja^obc^S5 zvx>0(29GuVuam%BYx~XSeKM0|Ny?d=QcTE$8fK?|*46Q~Kt^B42PAd{etcDtN{urn z+QTLUI%$U6TAU|Ua>lAo*^g9sDc-f)zI*McP zHoJEbAQDuHey5rbd^MA}7Pr00WdD%Q1CaGfScK+0BYY+$z?1(~`Q+YiuR!pkOyY1p zr-M;5huch%O?NQ>s1F?1GT~JA%)}c~RkU1P=`&jYFs;f$`%MUK@ImXVxI)mNVKk`8 zp<}>GaeuHvCp{*Mb}_&aOma*un0Ruy0S4JCQ+hdEOH@2Z(QX`o50aE0wck?<>G#vW zF(+W{l;h~6h#Oj8;2bMuvB7q|bZ_MoT$KE{I)3$s$M7?xxJ~NPXE@A*?nu4ETX>^% z6}u~hyu*EC)(*4i{IQ{Et>O48Tms}ZJ|ueMRh@!k`%l-3>+y#(TMZy*`zq4LT zqVkitoVh5Y`TG8wu2*gwrk4HK`Qf~n!Ffj^BKTuz2Gy|CA`1*|b02HW0%?Y!-vb@x z-`MO8P}k2{*N@2T6!A!*F-!^o^47+EfU@Ykz(9Tj_a7~Tj?ewz#cy()UCrc3L}TJ+ zCMrrl)vOh%lBIMF=A-6I81Ma-^d)xC>M>Jng1#S<3u9~pGDOTW&^_A=VOwk7W!vP*9 zxo!YzcNO4_Og<0tRew|kz2qatVLt-Ji|Dp^ig&$KODH6=byevWCmd|+lBffpAX?S50=e%yflN zbK}8x8kKV?iv#J$`^TeQM~*wkge*|MDqDw3S7L zodC>CTtaeZi1_oussWPv;k|pWG%RP-Yy835`H_g;J7NM4f28CbRtLB0r~5BuMtz;x zYFCRg$9d(qEA4})U%Iq51XMNV6ohg7_AMUo2Tw?3s)#GGkt?jWJ@RL_$+pvc9Od;p zZSuO?F>S})X%A_(Fc;?JNyv`DN~jsKpe-@mYTus^#>*&=?h-*0iJfbDCz6JQ#l5A- zq%tw(wZ)V-cq4SbJic5haM-q2%-AVSdZwZxc(7aq=^#Q6MzA>Gq}Wrh^cX32(^j0y z)QZLKJfc()2S(_jyJc+<|GyuF-O!mnCsd46tZy9PyEh5e~}LSku?#rY?WnU|Fnfm*Q6^seVPf^2-uDawDx{VavCj+vdJVwhoU^z#w&z@JY@X3`Kn&tx$Y=Yxw&Zt?xo$1;d@7#!YE3QV4LLN@5tr0Q-46+D+Kw#sVMsIzv2)N5`su}m zZyF*nY`GYFOuEZ&?v@)10K)pqF93wNc^OhYPV= zP2Jnkr#`Wm!N(V<7qi_hq^a~f*uO?H#{<)i)SzQFZQ<5bEc8&fj}7L$hi&eu0lRa8 zs|_Ux9hxEF2Df-ADLnZ-0Q;A0TuPclbODGe+kfoM{yF;v>6(hXzS4}i!8E@(aUU(8 z^Cu8aY?}uuivCln$`^m&!5BdW zvpeUc0a}ymZSVN}pl?Yp$j=rZAM7XTSycX15DndyJUaO+#6Z-35DRLnhV%z_3Fw!1 zn34G!cw6a2>ZeOKr*QxXhSb7cUB6bPf|>~o%{Ro|x|YIfR&@WvGpzb;!*oSLeS;i~ z?J%y)l32Na#G$RpP|`cGqR08FQ91a>NF%FrE{#PP=Z)|dM7{UJ2s0*sjS7y4Wuko5 zgdK)TtWypcxD?`{lCVhf`VFRX1srju=dI%vc@?Swtbk{qyZx4uFBtb~W!b5uFDDI$nD!rI|W9*y;>qDoZN7mP4{zPG>!gU~3ppS_*e zR#C-fMF%M4{5y|7s!IGw0oD0!jsc#%zJxkFWTWkeCJbG+%FjEUZl9{-N9BFFfZ0n-lFEmJ^zgx@x`$;UyT3 z?9;s}dsaN^k7E;P3Ib6TNX4WW#oYK4`6YA>mWl$y(4@Z-t!D|bHgslG7vffv+<^vG z{)~e}mTr{)qZ;%k#1BH7wDla3SUyO!J*0OTtq00OBSwITVm|*jNKqU&_`r`^8R2jQ z!L}F`jHs4Cwp=$9VfSdr*FOE?8o3G*YOiya$~smh4gMZxD9vmfzfWLBi&W zM+`l7TyYN=&%2|q4(3>=@#Y6Z0>1Nw5kxy5rH03G1m#Ml&m=ph`YzTj=Fq0sQT@4|-0gU;$RCRGqghGbzk`KSDg?NApbopvXGc zROK3&*y839neIlaoPAs+|KfUqVGT?+zIMRcH=?c|k0lqgJQdIXh2X6gn=Ev%n~Tg- zoYN+jp!wi8t>zCjwI`BUnLDx$VgHZjHjZuG5#P%n-~YJqwrB-Lg{GzmC@+a%_Q{z= zXI{Iwi6}3wzQxtfe-^Yf zu1mHjO^9hGHnxNPHHg)4FTSvGOsT%_LIdE>)@R6sB4j5co|B0#plqQTdQkiy98s%e#s2cp8$@b zkKK7i$VBtqqnLXZRu&kN_+-x5e2Da~KPAJ158Q3oWf#FRmT>|?k}MMzTCIv3Zs|+* z<9oy9M-9DM85(H!H5GW&Y5P;g z&b)-d@L(Lvfn)|@_FRCX-_yEwN>J<8 z{DIcGfD>4-RT=E7`|%mxMcR!Q8N#L3$yP9P0{@{G`7XmH09R$NYx6nzV2{n7ZXj#X zxqW2(0R==qINFzzLKT_S#pN66g4w)wP#)MN?#M^$@u`;n1-c`h=dZ{tib$V-VCpD* zb<&cVRZB>jc1+M*T#}F=g+jYzlZKm98K%Tlfa{nEKa5urm%S0za*FqRG;6l3sx5|V z0S?OqWoMOHnVP!@oTkDiVk3A#l~={I$XG<%dfPh|Z!I+bC0HQ^=o(jJ`MmyRrEvYZ zxU-<{s)1!C-K=y$oKHUHyHjZL*WN$>oE-j_BC_Z={H@_?)sq|7F(dT7rT5x8OF-9u|d zuh`H_xT%z3!tjhhK-cXj8X3Y=~! zC1rH@JYXmg#@M;N2Qbjy#+vO!R4~hFoOybgvU1lMm`V|I(lAK&EG2z^(aD+WTFvY+ zQx^Je?-r(;#XLV_38ZaFq`T&-$PdfW_3XoXs~FtHBscY`aNYQ=*+0$*5d|PDZ9~f%ZA&@BiVRo# zUV(K^9;^9om(ct6F84LX-MJh?r27EO@m2=+Ow91*#)qX5)2*aJ9uDD#wjZmDaJ(an zxEIP-`R2$}$Czo2W3flo$jK*AEiO|Fwt#oSh7G%vhE{hD#Av||eG)sM_B}StzRO

VlyrV3*y#Pz!? zKCzWt!e?)ux=x{u>5lqX+tKewlCk~EDD||nVWdtV;ba9O9g_$-NAN@{LOho@7FM-@rbYi>-BBRZkxtY>dod7xoKhWIs7QswM2Y?zj6HhENg>k8q^v|4N%$+$y2Skwmq7$Hc03l92mKxU* z0k$8i$oRW_x&N26#%c!_&)QFhkTtLM+nHwCaCILu?EWtdOXn!SsxOa!kgv5L9obF( z4G&tK_Hl7`C>+Q@)LWaen9J4t{^R6&bU ziYwrkr9ZgwS1N%&&j+$tp6xysaI46Uj=$IKG|Bn~q-(C1FhQ0JZsVn!#bJ+F#Ryup zCWEK1J{{}Ews3E*M`!p9?9nH5%M<6)7#SDuLu)CC<+Q<7@SqsQu>|+ltcSTa|9##8 zYe#f-K~B@)*GQ0U3}l}H)A;$x^z8kfN-?-a@#d*u$1dmkL2yFd$u^~uAqpzqk*4}B zOm<1P7qUBITaJDUR;Td?74`vwJ9us>Fa5tFRD>V1kgC^g`Zve5GOH98{jSAFaw^eG z&@nLr4L-3T2gi1hgY5wzJUlV9D?${D1QUV99rW3NA-y1tYt2ISQ5SM6T&Kx?Q~_OT ztuVA=seo^7S1GZ?U|aZ?0gRW23mxsiwO6<{*Y1?p{$9KWKUMI7*&;6w2QEc=rVpl6rZ@0LbuwyX`}tuu8cxS>8p%z>AvQ+Z%pBp)vdq-z{aC87b`J zdqIlH@Ij@BM&^AYTu~^*4C%vQ3k}kcWx%})b-OEs^w zPoRtg{>(wAM=a&{rdW2vZlbh5z_HK_KFV__6X2kZnIyuRCLS!16MmIwBCR(9qe&9* zBS@8-cx$xR5R(%;4=R_q_fy>OpWh_2ow(xaDxS38I&Z-S;W)6F|^4+>J;EI*b8B%#epEci+0R=WeI|( zYIhG^#A7%_U~y-B!SP_F#A4;cHv_X1u_N(2hW21Bh&ZB&MoJL3O57O(NK};I+|@m4 zkK8DJS+J3*hAc?TCQS3!W2%bvjIcr)^)PT`Te4l^ix&xe?69|T?B?(EhW%?Llsj3?N>94LU(eEks#pp^Wg+e9EV1=J#rpoz3 zV69czQMIxZ%dML}=`2m?EhlklO~bpjnDFs2a-E+^*f1JOxMb%Pmw7vGG(ZZV!&hUx z3C8!zPh44ZLhTHaHki~a?dQqdjpd&yG8EwcAypJQ>T859R^`D=nt|C4H=M?*h@6`e zAZ)qW_Qr5;rBKqn;W2g@-m4izUr%#VIMX%VqS(@!E5y(d3OtBnlU8_jmXY!seg_{X zG-ajhsSQz2${}-F4}EI6zdq~7^r2|^Kv()0hGYGqtfy;)CBHx6qb>rmBSPo31-_hz z215#q>wmzOf{vMb_LD>2WtHgXt#S_5z*UEIci3G*Ff9p*A~kO#1-Q&3Bu(hfFMfIL zA55{?h=}HmR;hW(=%E2~(L1sMY1zkCyNRArK~6p8u+F?JSn!wu#r4)5#p}#> zTQv~bt+-sCoc+fktj}>H`FjK`S-e!vUw@i(m^)nl+FRMTpu)t6((i0rou+l>%NghZ zyu|R|t6gJ-%*`4&MZQN}xt@0RY1!>_Q`gqGvOH^iIz+lDxQe6P4h-xkUO?h+7C!D; z+Bql`;bN8rpca^MMj<>-#-%--{6XJkQhmM{Nedn+SjbuCOy>+6A2wa*U8e49W?RMI zlR#m_{^35WN;seaIomxjigb`;-Ns-KE<<>=;(Q$)c~Co4k7D&tQ}bX?f+45(I*=Sy z{awz%PQ~=Zc%vuk#E&}`X`YKMwONz4xpHAZss-#I#2<_x)mWt1>Z`uAs>=+^^!(id z*CfIXR5d^l@q*{dyl8a!{f#{PAjW&2lGS3Ls-iF;m`e69atLxKt+{iHw}(S}ybT18 ze3G|0@B5XOEuJG*asv1>w@WR939owo4=l*G1VBzu9r^5{epS0ANVbupLCNJfv}Wx) zBt_d}Y5poBOiAsyDzo)kZEM*Sbi!}zYmgLo*EuUV^BWtaEDU8Df&nTc(uwYSU^cUx z$zJNx;0Ko%DRu@tPD=mJ|CqV?c)S3>)ySse*Ktby8W`&hNVh?@NPrPAyMQ0Dlf!U4 znY>N0?sHw^Ek4tIIN+Q7>l>c5&1U7OP>oI z*To{0ZZko9;L*RLF{#DZv@DGMnLad&)S;c2wSaRr%D0%yWAV5is?ot#po$(P5cWBB zP>WRDN6bK{3c-WcN80^f1zh+?Rx^Z78Xw4TF!X)jxtCvJ(Ps9>yao<$W+bq-3K`V2 zwP?OAf-M)%Kj8|^H~Y|B-lKv53DV8}1yEQ`|i|Hg3Ds zy^+3%&kZeJvAxYOW$#~vEA(|`xeO{D4U>uW$rf&ew#YXV7LkPKI8>7Wd#kfOmPm8B ziLe8uVE&1bM9E{R^6Lq{7j{x z6qR5nivV(egStNCQQBuZc-Zu2Y0rQEoY8mSc#+%uU+9C-jo8YtPLG~Z|9g6=K?vP$ zOpLARdMB((F%z^_eL5}YCE_+bSBku$FS)mk^TKireCR`(*NB`tbTX4q5K>q>P>~c} z*eAqgKW0!K%_{7*+ztt-YS=AFjtpk+Silgln3j%m91P{ZZqE3n(&0 z3}99mq!ORPPqUQbZmTRA zjXs)(V}XJ-LA{5G{)1gA26@Pfq(t<|hipIFPi&;y-y;bAk(Kv3EFEhW6EJ!YC@5p((Y?VXd!Z|f?_Y$2 zmw-8X`w!vTNDmJ%(>5xd#h>`{>1C>?#;X?vbTvF#MNpvrODmjThkp%3rEsB zlVe-EM{FY5G?G5z9RtBqS}o1e`bL2$0jIa~P?PyKWwF}4v;vbu%kT!W>HJKh`{r3c zvwuE>Aln#be@Kk#SW0zz8SP1mk%4Z}dV*5d5&D&v;~cH8m^xKTjpe8~3dyA8wvbJO z4X!7QJ~?C~2rNTnHgnLGkypN`SBd+Qu*i+A38q;}F%OX(vw~gvn5FxOp+WJ#0{j>l zcr_WMt)gOPx&?j@LvYg~Y6^)G;V0)+V>TK1vq`wXf&3+19<_QH!1^Vl%DkYkfVQOX zVuUrpDa^25frU8q7h|3t0m0BN6+^t-wz$I|!2*t8uc~LmkrVc7HL=TtrP%#oC$m$q z&@@eTyT{30U5QSA9#CHWds@9MRDDI#nY{tK@2)P5d}9p%7-dt)vV52w94xUjX zF4n!r*jqGFXDIX`*m`_QZazib3ng9{P_+ac1z1}wJx6HT6!#OGEVJyC%|FGsa^gHd ztnGB!()BOM7GG$=?>Gfo*$Ct5wo&9Na@dg7x!E*Knau?AWxYvhN23)ZjZGCKQ5=U>f+}ZQ%mKnAz;%$ zSea~?qUK&Bk~cZ?RH-DN-DnHj+*{^%8ZXd6a`T)~O=jI<_iCdH<4bN-yHY6|0^{$a zj74Y$7E<@QHgB%+6~ID%Ul{)vU63(PN&MYAM^8f=Eb16l9I~XrW1~mHjt?!JZi}8! z?0gR;NwC^Girya)O`PNyVP#$Y)pTXjpkk}{DbF=}H}LD(*xJF3E&Z9*!qiPFy9cdY zEn`R&|ITz55K7@Fk?&Mo+e=7|nSb^sY^LD= zg#$X=2O_OtjPZG_8tv#2rNKPtrS zbo{4b^Q6S%1#wkG_>$k@+UFSO32Q|OUBe6VR+%pWc{nF|&hu<2QY$EuG|fZTb_I?MQV{ z4ks27i(xT0AYmr@Am>4wLO^A(o5`iocMzcFOVG}=#WQ$4_5az>zdN`FJ359tpB?Rd zDzp)@syOXUQpHr2w63w9t<>7v@V;2YZzP2_k#Y6+l`R`|D9fr&e;%7MH>32`DZ$kE z&^6G#anG|o3E6pLMGgFN01D=jpC1R0{@s%F<0fgUu~GTVlh*VYz01aEEH|S#_j>3L zf@K8GUd9iTtlo0Tyan)yL}T)yHsjTa&`whS8Ut)^trv902dd7wPWj+_$jcMXm;7CX zuR|nm!}n3gk!!{a$Ahyx&tQ=BrOW%&V5i24tivMz0Yy}P^w3AH6A98bWIo%~a+Q6= znQbV~nI%sgoveY}VenX+9M3P#N$d$tzw@z79PeM}U@I_D0F5j%*RK_8fl?cx;C-+1 zx;V1pOgxFa(cq^7{1#WBM^zAj?W+IV41;k~Tz)_s`XxOK}t;sUDdS*UbBX1`j$9Q$v}Ucfqh420Ml@y&`*Tl1mCKHC&B zlKysrZal93&%+$zK%#3f{Dk=F`q41vU?Vk)D`d82d6VZ6e&%M&gx3^}Jg?Tb}!tw=)<1G2>Q+K+DzXT83<9 zphZqxL(l~*EtFc_v#_-6^_{D}TJ<;7%=)l=lZG^yLFPG0C^uWEvskQ6?oXo<13#~F z_JFAIgBTGrLfk40Ik30MDqBoU4srD6j{Lgxm8DHYV$NyD$Ds8M^QGs1HcQqXcoaNp z;T1Z=!fq#I-4r*q{tSF41OnXr>nSXSH$*z>urC(*I;f_;DV%>K;{tBZyciy8ZWO1P zWn$8l?A66s?-52qPouHVPzCoDa>T^}^U$G9!A6wZ$gCq|P|2^%lG~?z&#dx=YqbtK zv9mJrk?kd)4mT=8C?t@q4XN z)oLP9kIMawp=k$DF#wB+{fLnx(cg@ggk%CHYrI?;x?LYS7F#3t$u@SP&)I&c(=;H_ zRo4;=2!nJtY|E%s=j&s|9T|bXf9i(*KC-LxZ&!4NLKO>P+?&jFWCD$MXK>*-Ha3t! z;ite!3OFeS1e31P3Jq~RMAs2QIrK6NRxMc7CFP~{CMU#b*t49OC9rHPXiV-``lA4D z+4401`(fX4v3hQ)`1MLI7a%H$Y5A?g1q zG)DmohPDDZUFToH-4%b2r1vXab^5t2CW}WUUksfH27fe$QyOi;);=Wdu;q3EXW!V{ zdi2Nj*^-5Rp!4!&&}ok~8~AWZqN)_0&NF7+I`$wMsd_vNO!2JAnbpvwzRM9{B!+up zI>fbBC>Zf@)9ja3NXw7G4(>9(_K!nH1YB|kts7X-h!lF06`X4ErN~gL`L)GL+9rP^ zju2IC2V@7Lc$Hb6hE8B^yy-NxsK+dwld(q;7%JE_`z%a+4u~#G6)_J)C`k}-FJak- zaT^%@EkF$uyvj-S!-|jKG%k&G!r+a2%l#)ut4urVoEk{#%R`GSSp}5_Y_^;Bk{40D zB91*hq+2Z<5R&>g9x3=d$4bgP9T)Ffhb9+T_yWaJe;jmjF;6Q#JDGVjm`autE<&ng zLg=IO$noKGXe9pLv7qJjU1fJq(~VHkKeeOepML`h9IR5oQ2ZGia@ycNCtNhBE{brd z$8TgZWWbIT3<7PGp<9{K+7|AabA$!-yVguH*rw#sFdl7LR!}_kAqW~M)4PhmpDM~7 zCNcky%nRK?E;qP>a0GKebSE*=nrNe0Q(Dcae~3oMy1KFvY<(}p;s1@jceeQ`0qr+q zyC8xb%5rOBUOS}Em+nyEbYzPy4SCTfz4Fi1aw^m?mST(mx5GIqL*u9GY=OP1CA`Kk z2vo7+psw=Cv%^doLE~&;&nL!NUDR^6U=cuYRR%!w%rhafJ(Dc+DbmuNK^;sC)yIki z&bd-s9J9w*6J&nOdfcMWt#qX8gnEiEpU}ei!bAmi(yV4hI%O8d1`0&A`ImSyWl844 zJxD%=q7nA*6y>E05mSay?L*MhS@jQTyZKuP&pfddb{{M$RJK<~ffM_#Fv3=oiZ07! zeZFvYMQ!^?ou*gJ=&Eczh=>kG7&NO&S0_0Kk()EBf2tq7jWacxXye0aV{WWx`7$!P z>aBFG{UcW?93H%X)JXwM9{4bVt=mYXG{?E@wg+5UUN6~h>qkH!ot9qRm^M>wVl>Pn z_I37?IRX6a!PTTLU^7pKce&z#nAsqfw?2Il4T{i3kXb06*BCl zI5EIhq({|U&K#W#+9@oY1ecDu@EzP&rk6GWS4JoZT?ns?s1_b=c8{25zVsY_(bV;<|#f) zRPf_MhPyJ-Z!_J)i`MRQF{;nJ05Q)IMWg|*-LWL>D_>*cm!REMcANT07Szt6w_0r( z7K0dQ0UB9j8d3YgsQlRp9!>hdynu10m(|tYBX(mG#g*B_nO^s1 z@0oWWDQ5=fD%LH>&V+;>;Lln|Xu`?_+C!dgqTN8Cfo!HP{UBCwo|ZCK2}hlWtfsl; z$9e8vGBM1>!Wzzd`4gGT>V~SDlaTOnaOGRWJW?Stz*$Sd3U@~(X|J%Bq}+v)Qctu@ zFQ?puQ}-+Z+Z}oO*;<`|ffZXt;d{;UnYB3W=q{QkQx@g$$!tq?{%`_*`DfRSCa6_myL~xPBNQW{lAJ>(5Yv0 zwtNH04=rFxXV`U9*Tw0OK}86Y?Wk1tbG%x3J!TkFpJMa4qXZ}-A$Nkzk{fHZHD$;#M}B9A#sE< zUV@sPZt+3uepcX{?U}rFo*vSj`H{er!M%*FR@t~!IaV}xkg+YAi78O|w~HYDvB2g3sTGOV^eGny=Mk42SYx}1hh9|*M@B>yYKAuu_W!u}}2DQ)CcO8!(kis;-62hKT@kP?-;g&N_=O26=Sob|UX?En#l zYctd)>V0st`ur>Nn`@Q&XREEt&>onnut%CvyB&5XBt>*dZ!arR8R88wkU|b9nno^p ze7zZmrB3D+?!73?B4?KZU)=?C;MVKK?0=4ZHktTOi12eu5 z0Wjt9hno|>_xto*yu1q$pu0>?2A-*186ORM! zkRqoXI@4k%+;)q0VyEZT0eRo72)WLXE&897$J!TXJx;guf&twchmRFmo_ zD!!N03QCzS@R68u&;eoKfqu-?U@S!%c(>v9dlr5L!&}^`zviC za7&qO&J&fLZ?5xMRj6%SHoxwP(QHolepX3y-$YnY7&~HBGv+!k!jHGsD(&?*X#|~B zVfg-BO%B*=J=_IAz|wAO%Y*--{L0Yq{bE6O1X-2jle#Xis-g^g9jZHI*7eDa&j zv^z`b5f`J&@3O>AV!jdf6hh)?x|r?KMX1r4Mhi48dltvel`SYvV0GA4dm4>?QL28o z@y-Me`7l;7MK-EI|F5MAI{c3W1nTy~Bi=%_X`*m^J5t}Ztss(9aY)k#_zmN@BJR%t z$ZEm83^B<1We6^8d@1C~5czSn(+noT5jG1C+s%TB*UkYbh{di(uz==>EeNeE;^|wM zeT;z(@L4qIp>#h?lFeq7i*~?Cykh&I3m0IuByY+!*62OVs{O>pk8hx z_|N0baZ@mSDC_Ls){d*bxL>pj{jQ8vnH2Sk&%c5SV`i@4gSkSoQqEMZDJ{Vc>)sxE z6_AuucZNBc9*loX);w&m5m|8klzDeQLMVYE@D(h=MqtLKi?b$7heqhZ9?YpCIPha- zYofJ#Qgxuo2(AS0HkM_S*ZE-=>_==9^Wj0Hj{G4Qcdf^W2 zRUWZ0Ql%|%04te=y9OrhNw>2pS$=%xz?4W$Yy^yLPPT@HT>P&Za>dP@?6K9>*!0eQ zomaX4*o+=YK(n}zPd4{EjHZ3G%(t6B#IC{ANP1dj)7_)3r6^?es{r(!5B=D`v_Tzw zLcDB^PLVkWBcnv;{`}n(-(S$y(|2iIRyWK7dHX9iPwe9}c8#cYX^`ve*Z@WMDvn^G zb!xb=>4D)_{DhT`HFTy2viZ8pBzn;9?2|5LP7rm)VJTC#F#62&{kZ4j`S(p-nL@x$ z-0_yA+_#T=*W7B*_83ZrV-#$1|0i7zr9!Y>*?3=+%7s3zf{rY{9^?)%;E~9d9;>Dk zFC5i!bNpn*78Ih-;mqu)Kvo!phHEPCq@2@#$%sK1SN>^K+hKN!n~6p2`#KTQ-Zelj z-WLZ#pCPT!;QSK@_sQ=6^f=O$_}iDqxO!rb9DB|rYmy|Eq8EdMCTIkj zI>jr6(|ceiz(|RRJ5QGbO_VYPV5L8pW?kvh@pi%xkUmMljlRkNo;>2Pm`VloF-*}K zl>ewn@)AC+-v4#oL;-v8P{3A-mrh!wSzXK1C(%G;`X#NRk?NzyeAsS!>um0>;~de{ z$~_z;k@v1Vbk17rwmP%2}WYamq&p9moE6tE43|3nmBJ ziOvHgW?Id)i=Vw!9K1u^@1cy_$jYr4mooe-LY)@ODQ{$nAg1#!ttUaW>7n(XC$ZYb z?}9YAg;3FdCHbpTpdlH=obZOk3r2BL2G+Q#)EgCaUTJvQ8xkTi-MTXMQ zcbG2ImH4#IJo6rcTS};UY8=KS`oH~?o25ckSX3~7C@I*73LPfCprLVv_i!wL+>J)Q zWa;Oo4t8Utt+uQp`NPgOIic93FT*n|tYB;Se4Fi@Mf67^TVkWkrMo=if*uXxxlglQ z#D1FV@Y%pi{P3X~IE*lkukgkL$a1Fff^R08&LVSRnG>Iivk(W zgg*5Sw|MaCovH_l#^&XX3SbAS6xX5Tu*S1T7h2df@Lbf4H5lGQ4W*(+1SnBcGCmLJ zfKA8AD}esq=)TT5l|J-b6JaZtP2z?JbkYFiqBU?_xN)fCE!mBSSF(EQ-9mSu0ywY- z7V_|^wqzYu1QyyxZ?>;m_WGMY@waxnl3Y{LIOkO1kra44+pnaA?Z9z zXTP4Kf!>CW*lEH!(15%_72Q20^}f$*uuop7ytF_QM(_m3|Ez!nwu~~z!>$<@shc+p zPNqtnjS=H~<(cQ_b}nWX1gU_z7ecuaatvmVYByoKPJ@2DAfi1yth(3I3qf*v(wr1R zy+{AUk5hM)``L11RCbFS+JOJ>I=dm;6Q4mF7(Z zcS~WP0-p)`ne8MQm(|^ zScE#{uXvRB>h%igjWA|NB;xq*b8zoQJ|%(pKl zIHof!?8Y8^&Lko{M0yLx%j3xzx$f?OwQf_OV5!gSeyLlYeXw&A^pEE$9R&PW=OPr{BG3{VV%2G#KU*!b-+kY{nmK~+XDBU zV?b*0t*9bNbpc_?5Axiv^&`$j;kKF~HBux+iucCgB80WpPoS*#yBlWOhS9x3p12k7 z)V>*YL~b_7)2CNe>YN-x%#%;WO@I7@nr-7;&Jf1UD`1q zeJbW>04ZsePgMs8N^7#qt^c^2u+iT{II>15nO?6+Uh{ zn?Sh^k;UrJ|IfT6=wmlzV6De+hQR7=42KRYth1QzP5OUYldjSrk7_F}_ZdP@m$84X*9rRDBoqcO4t8UhWSGY146IugIcR0Z=e#R&PKtt zPIHB_X`P}pI_|WIXNmnFYs{zGPCzFER9ZF2etuv{n2wfPp~`=oO3iyf9W(f)tk;nJ zf(C^aZRq(WeWIJQ&2f4ylBHc`GHZ>ZYvQa#eeRow;DR25aR6k^v zaU=>rr#2K0#C7NsvG=)iCC8*}NJFf%jmfRdu?13#?JECy*KM6mIo2M$riYcPO*F%G z9%Ey-DRy{34x344Yrjjmw#d&YiPX8Ryl4;6bKUUH#PtU)P433W1>zKk11ZyG#aHRB z1k+B4<)r`7pogR|WdJ+F2TTAzHjT>cHjplmhn}ev>&%xxlv1sV^|=ZI?4&p~t1C!j zb&?_YlLo*(V!_$v6}hqu9&^=oFT~Gi;H?%-`6DJ^0@Wo-Gu$|W{5JIW0GlAnnA`sz z`=4BCM1&ubG)E{19w8f2C2g}7*XZ+Fev*5tqjFKM$v~&y?%7=4hlv0CmR^r(zezDD z<`OL3+W9(CS_ZJ|&gUdRIE05oGNP#Mn2%m*^>e&P49FI5AOPmKbb77~<39E@!VmKN_y zC5X))K^7n>DwcL@7DIGbIV~fDI{tIIzEeC!h5WYr6XYgb2m;b%{vKDU4Lh|5AiRD0 zxDx0e1fvS^sK=b_hS{7sZlRfo(#~jKA}((~_yntP+*+#J1&3KJ`K{j8j*||E@S*haW4!~`TMR0EC>kWaxwC$`b#+1GJyM5zsd{# ztscig)-VNf3DQ&G;`;*W*hMx@pof(Epr}{F0@65BthApjy4a4u03j`)-*W#I5yMbC z6~r+yI{Z4I=77uM{Qw7tt<>*dFji7ps%5<>XIvQE=H{83tQ?pFSCtz|S*vgG#&Ulb5r#1`84!BSeL30*-#6K zx|iT%Q_Imi?I;py0SFl8f+sx6T+Wh4k7l(IaG7f~D-(6GLMGDla(Q$zPJfnZ&_JKL zu264LdDC9ur0Yf^j@P#aOtSvQQaVXnZ!=V<^yX%hBoMA(j!AC70eK!}4+et|ViIPL z4evPGW0T3-I~WQKD8WfZ6`6Vi1khv6nBz%0{@y*T#fe=-_j29hz%hvwDlLNyD93n2 zzJsQgL)YBdmf#OioSZ+2+X=bF=i99}1`3|#=Hux6b(oZm^_sru zq{g!&raVVn)wl3rBe3ag51c5hk8>K$_)v-jg3F1BQPF?px0jm;AXW?oFEns>ONDtV zT0s9Ga3yS+5`PC9bT^xi{GfJE)jhe&<~9c=IfJkcWjGTl_vGyU5+WA`V`Uhfyv&!= zb<=Lm7^qJ?umB_|-GP{|6ZsPMn~;-hVY`54gP5M93Z56eX4&l=2bdN^Q{EYuX^7@I z^ThXC8Y7b*a5f^*AYsLf68$Il6dx7(CJK=3)v7?3hTz%(MTYOqyP?$#PbGLW5@U9< z{`=1|Nb*wINkxei#8=id>U$W-DH_E*6rv_R_@^;q83`iS{BdwoEJ6WC7~BFVvp@D6 zyDyz9nP}?h>FTWC=t4uGWvxqG20`$^IA$f`<)Yi#G$oPNX%T%TDZjoFhL8~K#QULIpFyx~-TatzpqGgg5#ibIWOZvX_Z!qp09F~E8HPnQ{SK)&tY_8N~l04(+)7>S|GZF23 zi1yUs@AZp*GuKQD8p-7=utG0c>n&r)zD%iD~wpV1c46QY>XO3eh#I{Cw8w-vGKclo~R4fdU2isvg`6KO7a$xr7+T&`|5XYFAUvvi&>Dx?rI;{svZP-8Bm7SK z<&;n_m&AoBuz_*33$rUOgtT>{q{mEd=0A+Ve8Q*9TWM7s!M3T<$J`OTscKX{C6}kI zhxlKPqRgktGoY^6dM3svcG%p5i=9FXb;>rh+>ZEX!|{1!E*a0(DgM{7fm{l*10944 zfh}wSru#)IU!hZP7AKN`B*44u41Y7#V;8SJ6F=U?A+*L|sH8D+uWW9|s7gnrftTdm zcc6|ea|qcZpub}j`84q=!d`OL`0HvUUJ$ps`K)pCMM`%MFx zn?B*JQ#;#&P%nKNgcrnRQ@FEHpNzO#zE5KKEW@d2KN;nHm(nbQEh{aqBu$oEz*AhI z^`!mfD}lz4&%{3+P&!SY@60w+pRo{D;{X&ko9e)kwD>Sh+UWYHpW<=IL4^|bJ@{L^R%Pe+7HEzo!dK7J?1De#ZQ^;kmOLvEqJwDAb!CqiY$lf)S8>A8V z+XMgbvKb{*;+qp>^bZI8SOjz#+K$Bfd%67i@_SCetRMNF5W^|h#w)8RtuIh6aWhOF zu0!q98rg67JIdcaqpI_BHm{py%K!u+jUtwRiYtg_uYK__y!DWFAR& zvy_K)pB(8x zys|yuZ}$jGF1f%*R;hvXUw_oR`vm5A->_3K+E`MoN;|6yX?_(!`KN;*>*tQ=^WiE! z;s`S$8Ii;*N9hrHHM#S~lCO)+n{n4l2%}g)Hw-0NwlK<#tW_GY@I(1}^QB*VG5Zli z`vzurat@2Y)19z8i>wiAK>LCPo**};k2fJqy!<}TM{xZu)C@o}Nu1+e5RgLAS(XX4 zvV&+qf3SNjck7X_{lsSb>qKHdmq^Jf?C3nH^+wFb+sV4dWDzV#=TOsLFqi^CF4X)N zo6_CyhIr98@{;Y~)?z|*>)l99?=b8q178V_)k7ub4lJwRfeZ~n(AaI?y@9z!LWRL^ z=7rdaD-Ns#9T+U|K~JEjRH5>Bk>JW}b>W+`oPN(??DR}=`Juaz{pvtG49oNZuR%fQ zg~j-JlC**WK9-?udo> zZ3EwuL+x>GDc~@O0}heYzncoq_>k~l{7tw@=IAQl9umq2&^h5;odw5*r^U{*lUaI~ z)UCO-B!f*xT3NKmOkX;Qk99E?QLO-;;nICg=FeLyYJ}udKzFEK*we8{VBpBPUK^Fl z4dQjtufmV;QRr)`8}(9Gxlyv(axcvDtUe{0Rmcf%0wG`yQ&b~FHDutmOmL$5rx%sD z&O(3K_E%5p3$Ddi)TOFM)6UPwSAk>-M)pGt{4rjy0d|`@Cfh8yYQ@DMF$ECjUbj|H z7yWg&7^;Xvg^9ab-ttZ#hw%C4a%wVonU_nY74CldJa^`psmvcE7wbbQG^ z1=*wr;vr`dbaXBAdcYZO8C)4R=~@0?K0B@lzrVeD*@{xm@md&LL`;r6q`-_9&VE0V zKzxRk=^JW^-)B+^nqumw?`j5IWn8<}@Blu*X56wItkkysOpBxqcP+#55^jBcM^IU+%D=zxIOfj9y6 z_*9Ohf8zIy0Mr1f^Gp-hdvOFDKXsAGv;$3JTuZ%I7+xKDR^aKb156BZe<56^b}^vc_k? zSRtK&w_t7;h6sy6FDYTVgaS$$!nLM{C?6+}AdR}xgcTwkmO10`o=OB~ zSYkDxPr?QC2cQ+`x}7=37%a11=>*eyN>5ZLuH`&AxO2y5O%&ghWY!+kTTIwFVX-_| zCnRae@O9s0rsnN1zjpV3Yk$7E^r#74UGn;w*Vs?!J7>-xZeMPhWW}y0bRSakUP`L3 z0fg?qR$8>n@)|tF4bJ%aIqx~-ya?|n;!+ZG|d`8!1R@1aKbq9C0j;!h0=kj+nl?)8=2W5$Ppu$WW z0)&cZWns?QSm0$JKrKfusg68WCCNDssk3RrN1fJ$g?p!NXJ>?|$4xQ}G z=N4`m$xp=|k+Pjtqwq#e!*2nW4X)-=Jg)e@BAB~D-;w|9_1A5g_jGM-vV>fD zuHNbWm!2gcAlI6*gQqmnmf3XJq*~*j5(Uk$x^?9b8m7mxW=dF}yxqX*5TeFho8SQT z(;GZ1v7A%W-GVPH0dHt{(?Xr7Wlj4PkfHdaBwK3XAsl`D*4lW4#OkffsIwxi@l&4C zggt3Ay?+RT`3*g7^D7WLt(iL$2a}hR2`SznIXV6Tz=(u6T5d^0*-kHh69SGdY)wij z>WOIt3dmcK>Lzam$NEc3y`3Q&+h+4Z-?S(4TZvlZe+RBS2j>MQ>*2I|UV`{PnOaT$ z7z#JavymI^@9>zoY;lAYym-^iHxz9CDQj2>F0%3o<;#zidg5t1`ncmdzLNdLl1nSk znRA*CXVDnW(8cC)e+scv3a@#rE~@?D1nUoy=7zc(`=-0oTimJ+A&F4Xp=zD8z1G|L zk2o7}4jZ}X>&}_jjK$beAA2JSh%EkQ-JzH;vgYvTGB|NK)=m97&Yd4M5P>Qifdu!S zj4NYfdg~wDEH^$n=L`yPm+Idk)EO)7<(6osziQ4*`E7nq&?zkjricGaqG4A zDbyzX{BRG|^IxS_^PY(mnYkn8P#8+7^8jY+=KPCb8{Z9o=UWfAbLXnomRyK1lO?_o zGd*-!f-rX{`A|YSQkh%hYZ;oKS-2ZbC8FAB8}U1^D2La}9gbxg+P-p+0b`=n2sZ{1 zRt%}&G5UL8f2xR1W{#@b#g*0!aal!=D;$x@%j<{l8*iJ4cPU$5wGF^N?o|pP9vP$* zo4}R1@B%0Oy~{|ZFpgR9hIJcN+}Pt0WSDn9kB!m=9DQl=>D z*qhwVmBcMIj2sE9*Rxw?k!5gf>%@t_>$Yqlg&H%vs$oXSM4KZbOIh;}JmDuG?V7q? zWVR%9F;7wr3e1M%mI>**Feg&$hwN@Hw z#Acp7N}}EytHW+fh$~q*3?fnTa%I}^ewz72iE+-cd0kYwZh;wQ=BB^a93KbGdRPj- zXYZ4w9eAvHlygpQ#xh@e~=O#`0v!pyyUm_OCD1T1|3Jv@*Y7WoguAkkgAdqOc zBYQ)fg?Sh;Eu(#EeN<)yX8J5$dwqp>pQgGbFrp%u(-S7KjHhTowgYi@Pbk6bJh2$I zKplp}_nTVuOZ2YPK5OY>h5|dXfU{zVzc}g5u=UQQ!31*-S7y6tE7riB{Ka~$Q=V2J zZ}vL8gRS;vanGK}!LgcHvZf9nAAw6w$rwF6{oPd%ugstn))XEJim#x|m5FXRP(x(% z{$P7Nc*nWpf{tK{`h~eQ^;M7<9}Re6Gepa`o~<2~;fa%;o?%_}C=$ATS)ZRaP0e`gUS7AqKm9RKReUwOC*kjOyVV3=}Nn20Kz-q zw9ve77$1E_ua%ssm3_<_03y1~%P^F5BWtpn-L?nG-7>-G@nc!w4n43pN({fx%&jRv zX(O5Fm$dT_LFSH9aZ>QLPKZEuQaXLPC4SnQUo1+mB+ERrX`N~4Yt4M3t^uE5)M1;v zj9tMW5A9lP>xu6{MdUSgk3g$AAJwiftKf)NMT+^*n#+XF5vtV|S)K}$Dcm|V9o0vR z`!$*{GsRodc<6PCkaO6R`|$+ket!YP54BaS)+^Oq8e-7c$I?oN`6UadTHnD=b@J$H zPsN3DZS#N1K zb%m8HkF9MJij#=LW4t&>&VhbtGsehal3%3|2-XD9m(tO!#;i~W)r=mHm)(MHJu1S# z=(AR-r1!p2Q&h2iSIb%ht3&Ilf+Z#iva~e1a<&|LS3!cms`fw=DEci1mUxEtR#I4( zX#qtOAB(4kFW=AzT4z-O9;4Xy1i=qDaUOdl{3y`+W`}nOob=2oIJ%}`QThk5 zG|W2nZ}})E{59H@GWDP9O%M26xe7)!LkPHG@JUt(5jJq^OPXAd(?p`zz3!zqfv<@u z^L|gfkO5b#lM~-*zrF&B1U@pdUL7TR)(B!O4ynlv#%?xio|TC{Kw*Z0{WOmBtdj&BKUVBmS@4APkoi`s_n!2q1ky7So#E&s!2CuogwryK zNQWesiv&Wx!w(hZW|$}mdz|=d3scZRPcJsCFcIJu+q~qkFDVIhuAs{KUo-l{QY6=B z1*(`?UTl7)P<%q>E-+_YCdOo9_>KvS9Ba2pM3v7p;Oa5@NW66?i&ISS4$<@>ijA^H zq@v5@e-sc=YPPn(o(!EjH=j}3Vhsg8af}r!d5AZBZG$dsPoy#uwMyH;?EABQ9^yoAL$ zTOCjg-*kKY5RE0@KZyNr@-p&ifD&|U8AnqQOqfqMEIH7H0rK8+1Ao6q^{Kv;lW zr4EKd_o_DyEWsY-lPt6Nm2+|(#Jv2 zaMS~i3-KQOieL$3VduN8dj+j=DW?V2c0q1%5#sKiM`s3uT*2@mzzZo^GA)$?bXuzB z%79oeGj`Y8Z2H7Sh3_|dVu%9N0|8yuMOSd?bKIv(qOU74xdXF=@}*;6LtNUPXaVwN ze@%;KUSYGyeE_>tB=rh>=cOTycWGpo$@EH02t4@T{Nb`lbD>TWrjyd`5yJiu-uFOVqi z#p+Xem;@}Z_G$oQ0qq8~wdhRV7~#_7nTx^#BY64j7IY?uKnmNw+!jQl{D^vnh8ON{ zsC=Ted^TG-cwGL@*xkifZS@VPZOPpaGn`KZ8dhO&P5}aq{qoA^_&z2=t0~_0vN{Zt!2#Ah>{$ zYXI6G`50-&HGG?e66?ytFT&I^3wRkhTH&_V&HD>sLw@o{*;O0#;2;qbGRP7^ZJGG6=ZmKd^PO*<0izOeT2g84fpR7HvocNjx4$7Q=7qlV_i6FSwbWz_432=tQ zS(b@TgkPzwTU;=8&CG@;I`)fzFpkrRChH?>X1AM#>%Nd_r9ny~2EEV$Lvu~`x_#eS z4{V3-V*_VIRF?5`H6|}Fcd1KrNKUQ@oo~40Q^8MbjAL>!7d51hEod2zI0Rr+K2LE3 z2LOk4ZskL~R={bIKD+GL2&&1v+g;6)KN~d9X*y$My#nzYpsW9Ow$X?;co(W zzI*s^?*jjo-G+E#DkePAMQ6I?R7CDY1wYJ!%Q^%>=5#PYYAehcKnD8?m_y*QkS)uv zae>|k{_9tPSQLc31uC7{n-~^8NcX+Z_>SNw$Q;rFu z#9qMl4nn8+8uK%@9qR4@Y>r_~Lc~(tAEDH;jWw;7 zESL=rV;GICarvK^of1*IkqAdS`}|bxP|=V`e1Gx2Wc`#+ha6wNarJFR-24v00LvHae%p ziOcl@3WzIkY|0y62EHQZSy53|&(&3Znw(&-1QI}-=a5C{XYcLJ0T<+w9#5T**aB7c zHLa%>sYq_^LTcEOsA4whTP8qXLUAt-)}_=_u2JLBmCU4iaMT(!m2BDtca(pKvB%na zxFIln>DKs^{(8tHE*E%q9Xh!fTaHb`d4_m%+NV4R0Pi9xrUWUA%GanvsIOn7A55tS zGZxY0Ku{LLA^Nsf8MY3Px)iedN2*mu#E_vY05Cw3{WOUu#}H~#45z`##ixI4u_E#F zrndzF15$DJ0y}VmlMt0Ept=A@v_39*HMJFSvirrU?8aqT;J$u!Z?Yf^Thy#N%#Cii z6RgUGK57^l3_sqno}lxajRolWLph|LAQNOW9mef+b7rB0u%e4#@H(O^0k87x1udiU2e_BdIeXT>*?PEJ?nAWcfRlghV`TQXtje_g5I%z zV9Pdh3!H#JhRK%;CrT;7%Skk+GmW09X*2ff*^Xu<%%2yw+MSK z?4hU&7Wmo76~&%SYq_{4ZFEP?%r<|6O25X>yRknI@j?Nt95x~~y8`|R#ZESQd6S1w zUaus^rt|>86rJw}mZz;#O$BzB?6MdR6lh_aE*8+NL5hZmu>SU}CYr zsoBtGA2xsT0OMvO>EKk-VxDw(C~322a@QwB52{UYFyBhMiph_j{V9NF%K|c>_E2!U ztWzwda>2YeT#dvqh?|t@b_HeVMVySxG}CBd-K@L87gr6(X{N>gl5QMPiX34QKS&z;k#YMz9|E+ zhtamMWwdgU=T0zn3wo81sl8t$Cz7}pmJnsg>Un$-nFUL%I6<~nv@ncA(z~#LvD0qS z#-XQgjR&9N;gQ+Ia|~$hhbshDkmXL#+5peJHr(;oYG+>x*cm6b!N8aq{Ia zL^TUT8DT5ar_C(0IyK#;O5 z=eOeBam#DI0>m3@Mxk6rcHsS<9xCemkJBM(iJ!$Fps6w_A=OO?pW=WZ9 zLpIiNCa+S7zK(eoplNOpyBe(+feFN4=4B48iks<=Wh(H>Nl}g3$y^i`hF~7kI=OS8 zT}HUPP)+V^Mb>c>C(r+l1y&w<6TC75oYr!6d|Er-Kg>9$V_P7193kZ|IRRnZE|v$l zU~ML}i(|+1Uw~*{k8oH;u<=DFFloRCZIKDbwW3XYww|>@@TB0d0%1NcW>}1 zK&P6A5x4)?IFVYGkxs`_z~_vO3D>)ZB{WhVh7;wTK1u01@e&9yW*Vcqbag#n35=b! z7MC=e3}L6*tFP3>6YdAs!SQ-OZ_cU`EF6|D#qjOpQzb=n$7Ryj)tcan+nCvpQo3gB z#CqQ1fnt_SL6f+cIb|b>dxIg;C7RACSGBX(gufF0j@`;B7 zmGtjetYZyL%u_=)+c;%wVyAfjY`jij;Y1D&p$!S2;$B`` zWyov_)ZCBTQL}kdTQb~hJ(}4Vsd02iCu?{cDsH4?9j^PVq;kfC=*G|Wdy3%u z2W`g-+1JnSue_ANdXsFF-7mYVVB>DN>O6K9UJ#|3Pb4TRF)8puc7ne|If8(s-un+T zCO;oU1j4PqOt^8>f*wUlo*a*|&4Knil0^@RUKkKc9kmmPc&fB^sQZ-v+%aXCiDvOI z!Wf7p?&r((AgRpDc|Sw5PoJX;$w!Yb3=r82LCkgCjU$bv_)c?H8Ut z_5z>TI+JBq|Ausr9X6wW6mAyFz46?I0!T@NrM-?#Z_y_XzM@eNI}4XTI~-Fx_t znfX~=86xD($wZEb+Pm$zOpAV+t8yDEMV-~YzyJSS>D$QOZzJSED(N@H@hcx}e93YT z47?(U!fYo;=2vWWxR`Y=_V@*b>SVbX`{uJ|R)1CVZr0rEC5_B1$=ciHHx`GUFbqu3*q#Z~bVEC*ZLPsa)hW)#HUxD;LC zoljc6Wx>$l{dU$Eb=dl{%Xtysw$geRY)`%8BrvU<9;syZB_XTfXt{<}Ni$#-q@Al$ z;DWk~Ls|}Bpz7mkgz$N}(Ryj~Trc~9muHQ)#uPo(jLA}a7x&vAZ;W_Z4I6Uf=8Sz5 z-s&)Qb#Kx&nugwh@IoB-4m<(AWUGt|8Xyz>I*4Tm!8%)AX-g*MrY7u{Q1N>|s!*S0I>v zM3nLhIF{e5ULrf34i;xi$&BG7fpvZqLUX88$(UQd*TY!VLVbKc&%N}mTf>_KYWOgH{+t|( zs((ENhmMbNMDGm4Fb45|LBKm{_kf2r0|GV_g%UX9Q^9%M1lXVHs|h02jU9G_nr{>t zP$-XIIXs#?8Ek+1Zbh$Gz`)C5_P4^w^mO4koD}Axo3JtNkbxJ2`211R{?Oe%f_@4K zC*7d2SZyC^Yz!beHtH6Ol<&bNfT-B=t{XI%R#PzW*)P=-L7H`<<`2RO0w+XAkT){vA?nYty0S$kiPRP6I zRCCMmQfn%Ls%+?cMZ)c1mdo#@Z7mWwc{w6-9Sp+1dpA34S()N~jp%XePqKWWF?Sdg z-h@I<3+V^?MYA@xp{8byBadtVRE5{EV>~hJ3^G5&iYno4d(U!rjv|sv&`?pYvFB!> z%tPDBL*Q_y$mzj-Sgbn?kCptM;c-^3@2xQ9#H0;NByoFno{S{E=b0i=Sf&B#iWCe4 z*u^1(O!A*9bV79Bd1ytvuEb1seh(u}s?a9TDE+>Y5z?@W5x!uMHc%pu4Ed2z9|Ctw z9k|`m@lclptzVZ#S8OgS7%T7LXMNV1OOimYS@X%)1Mb>Z#I1L!>`D!PDVsJf zI<8&mw|_aSWJZDuOG7ewb<@2i;HEmXL>I%8g?h#v%45s?nlG@Atjgt@KVac*?t-ti zaQARF{%TJuR$#>FacptUG4unx@;+?uhC~7611p^z`IOfH-U@P)zmm)O z9p$1Xe+dHc9-V`w`c)QzmXYqS&BQf`+DZ7qp?y5!05gET8%{WRK?(s3LzVR+MPKS} zCVkEay2;Ofv75lvTgNVacYwxc07|+fiQs-cOCBES;NYZv**~b*cxrNNnV4g*7dRuN zZ{ZCOtGVDNBRKt?Lfs7Ra*9NGi+3Rg!3{VcE3GC9&cJRvaZO1>M$#1!p0)i1f5@VN<}GL44H2st|4ZUn$k z{nx|aHgLF!n`l1?ibyum#upvP>=RF^POLf+Bmmx*vxsncV;+?9+jO(#&EHnv!WuvC za5m1r=W|4nRX0Pb!}l^9V7A%vR_mfk*aDiMBb5>_9wr|>rL~^wiy8b=pjn(vKr-;~ z;Td=hmrt!b#(7cy(%+iui$wb)p9+I&^6ry`y?1SX=&o15F(3K3o ziC^WEI-o~lcLxDP0CKQi%-EZnu#{$vA6+$>%Maz}uYgzG_kYGN(O;lh1jc>tRdqfW zk#07RfzILuIyi?>^vVWpE(&Qoz0D#c?L~`rXZ5T107j#px>b4}nJCVYm0(7uI<1oygF1yvq2R?|q)1r{P@VP{idBvIpgRkR-*xy+i27(4F&34Hp=e#%3oL zAC&U%0NVgJ(gbULeRMe5y>sMV@V@N4NDi+d2-na}#4*`CT$ArVn2#>Y?CNrD$j>maVb;-cM5gm#CHs(nzyydexSblnKmne6?q;)Bysoi96}K@6u2C@W ziPT$5_f@+mNe(5zA#j3+#H>u+^b)E{D(&|<@b{Xo==1KzonR<=q++7Lg>iV^pBi8u zL46tQ)1XRKwJN`}npD}K!rG^h)vp6Yjq*Yon63i8B7+iOviWj{2;pzvrZ)>?d<0Vu zlgwz)0Mgh&UFD1Foiq5P7G{bXw}eNjxQZ{@Ejy(TsQ#vskRX3+5kQV^%?sx_s@iXiV|Oy`ZLJNB5~Vj%F%oY< zAbtV~^(J2$In_8k;EYGZR|;a!vXF-@{*cFfJht8})9$L~cOT!fk=+~t+{e^^7S}EG z7u-1e_~BX#iY+NAn)H&%MxZ_><4MfgXP;uW1jLa+S$>SHv!Mp4qj1NABplRg2T7^Q zIfwfw5kiXR%-Ii3@n+@c!|-CFkkrWWjvLer+6Y~t!1k)li8x3Npb=y@+nD?gY_l;^ zQzIGaoL$8b+dhZu2FPNV;3nBSCmIkxTc?O6(|NV73dK6co^Oi4LM4y1w%hAx8MdZY zf-e466VS?S$)6PQ)XYlVcG8_o9&aR%oFH?OYJ*nmA0foMeeI@>EXdsrZ!>O)2`x}4e+L%JjWE*;gwLfPls0PKb8#H0!OH||Nt zH8c)^$uwAdNZo>yOKmkhQAPM__sr##r=~BXov+)+_M>dQkaK&`BMIxG%Xqj1iqtxQcJ$H{e@?D;jbiqU@`%WzEMM z)U4rBxb*>uVKhnj;HrIH%RXDji)nVS4&sT-&`SI1VN5k{LST4Lg-cLB05jt!48C%f zS)&G9eVnAzQ@K6X;n-C@jraQM21ns!*2cb0j>0Vv*~pbN{AF7_0Z?D^d;u;d`x_iV zZ*FYRH1Zp_-}=dU1Z-4q4Ff6F2>|0Hx2LslJ^-zO_p=KB^gM6Y?3GmqoBp|Cl%uT5 zWk_erY5d}aIosPLS|7+uC`L&Ff0Ulayg<;ZvYW}KAZu?*2zxt1>w_eun?U6rHZC_b zv@gf(6jB_O)`?$}?NP75U^aBtl`Vvz4xEaN6Wp5B7la+(E_vNmWMTpOhj|47T6 z3>xExY(j8q2%wf;{>xnX_8B9S>cmT_`~p$AfyDVwbx+e=_U?0*VY?0cj2ruC)70xC zjb6QR?WgHAI%h-oE(3_N8rIufYte^-%k1WMf>e9;TnhG=Q4VY&B5Y~6e|Ezb6Ru5m zes|?PJx+eVr1hth>dgtQn$}W-wWrIH1@L3scTGg4dRB=m^mPt7zU%pfs2zY^1SQ%Q zV;nKfq|wC|LNK9Cq=}yBOJ}BP^FrgKU6S9YP2l|(tsF#<2(Qrr3CM& zu!=S^A*z1i9V*>`y`@hT7_+QYP-tj|usR6Z#Q-UM&BV(NK*7QGLooYB zdH0@qzge9ZKp@LljxS>7&5>*$CjMXq>N;R;kxp{9Ea8e1dK+EID5xO60&#)&9>FLE ze|!l?Xj%DF)<*!dJwaX8Exl6hVuzvTFC?cTD!D>$`sriwQ@-{^vCNqc@A{n5rL>cA zVDrlULv7_{Hw4}k!q4|jbw)Uk-zNWkT0G5N zyLdY{A-fzlO@xoCm%|Rz2b?z;ChjpPi|;2AVHfixA69+X7z$rIY9yClfEw2N+*gPb z`cJ&U@84|mNyQ7DZj~|qGa7ufeP3)z*_dY%w9fkY1s>mGUgV=|Hi$c{t@8PE`%mIi z=+>^IG9IkfirT$>SA-60Ks5R2hKQw`NLkn&RM?kG8*Dt6_8fXFswv##iZh?Q(4QhF z&FX-IOBZr$x~UT{GSEPi<`47d_P*HCO`x;Y|DtX4xF(5wm2KOEo2f**GmlJIx+<8C z0Y>v%YOMt)a~zN~ycT1KxV9upt^{qsuxz0->BKFW77-byzs55V<)3mi#dLq*`Ke=4v8v|I~$+X;JdL=oCg9{*#TB z;yK96dII#`>>sy;Qw|g_p74S@IOZLd2i~d%ry&i;GBmWB&efK79sBds$mytyNI!(^u}72 zg^-(5gSvut-{YHHoNj3pcPf91rqQ9*dT5j)&H1xuK~U*~w+02n9psPzV$mPBNqS>5 z?e@w$6V8dKWj1NMzOU_%fhmVZLeDdv9YLU!=`aS_NsSPtpSL!W){_t>M z8?;M{qbMB$sz?OhLqAfrg#CU1IuIE%`{&LBG0Xys9v#hHdHBVGWkD{Y&T;h4(}k_A zU!n|n?l(n$R9_mkaSu8Q74QwQErMS0_Eid{KjTX*%Lc(EslgHm9tag{vYDW(lLnE`?JrI;3&E+VYP#Jsv5NF#M8dJCHl%YO_ zlV|k6&(`KE-$X9ToQA#X`2Fz99m0Co2)#ds0!pv4ekcm7$=LK0V;xH^Ep;d zN%kB;!gR+FYa0*j(1;l=lBNM)%IncU9Z?4+MHPVoL}#A7^M5dFVKA3NFpbQ zq%`w44rdEvx2bn*mVc)jtK{{$6SJ#ZrjW zZ`2FOZi@FvBl?){k2|{V^t8(Y9`QGFhvG~cV#Au+-iB;`?Ua`U0~hE5Tn?C5BnY82 z4M&JN!r=<|Z!e(aIUl8J@ci*+<(0a4hED!mrN_pCrv~7Jo-hb`Z9`@MIYHoou-2O> zBdHvD6`AwOan}cPKQbbYCI1JX1B(f^c``-EB2~TU#jqs#gwhbw;5+=WefIQ&%+2_^ z%3`7(5y%<6aHvIfJ*>cZ!}AH{*k3zRzT}~uiZjk^5Y6vG+s|IdVNP&2GNZUzGx&7) z!hzrpwI~i^VSq};H(xd^pb4P5FA3TZUgSr{M0|HqYKglQ-$AqnKnjsexUa*!LO;&OAPE%DUl4VTk0s9@q`j9P!GV^=eM zpNRRg*O80rUtZcYRdx&6lt_|}-<&Vpcq@)SbeM$HBbfO9%`dm;*M3J@ziReoZ$0O4@w(E80AqiEve@n= zm&b3GO11}(D{z=w%#HczFJ*Hs06Rd$zhT*0XN9S<#YnyV=bg55SosXrF~HUt3Cwyn zpCndWS`YNWuHdKb))S1_@@aXFL5z@|JE7fpk`y<%BFTV8S@of=o!fHrF;*>o;V>ZIOmfsgZBF5^Hu(h1M;FQra}4CPv(>(oRU< z?@NiE3Lim{{)wdB*q2GL$kPmOz$V>KzE%R+XM@Fbj085IayHS# zd|tJq@U$bNSP7cOSQa2dygzE_OlNvMXLHZE=0FdYnYn$LUtnF2AH%Q>@WVa290FGD zA#JM#Fh2iZ<8^BbtvWlH`V&lKN>%R(!@h8N2_YDPDB5x9qmay#9npDI7aC>zWP0Ug z1fS#k7TIFia*4}1OrxS_u{vd?QNhvOv*NV4a&jqn_e=i5C0sr;=AY-Vp!rdJh}gkN z|J%q8;oE#xKq8D2$)1y6YsLUVfVW&6PUoYBLH@tRda0m55QKg|?Key0?y zq7tE|wC0Zw1%>fn+g#}dKDw!b&JZ%S4E3YqwnlkT%K~CCjMCyP_5ACOJtx;r3J!8F zf)IOl<_6Yvm0~hAzfepC>tuCzvq-BG{Zo>_UD@Oz_?KaOeQ_njtj~7UkS{wFKHSnW z0tq8v9C7mN>k;>GGM*7E>1Hb;u2H+yn*Y-b3WmVrF0&+r?Aykk$#6d6GGr~DUeh3* z$g(w~lhE=D8Y#JMh|I(}6B;!s^2V=*+(ug3{*9ceN4|xnbL#XtP(zZmTDh4x#tu!H zq|DNyHf^XF=5F1yD{!@p6fGzLDwnG4^Wsm8Z|$A0 z@vS$AvOjtsQ2+<*KRJJsjpNq+DMuh|Mb9G|bz%7U#|9(q{$XqR;a`nt&VLg5PefEg zFf0MzbWn;I`WE6ZWBH586n0%qwdy<@?B<^#9!3;i&^7a&Zyj)7YfWSEuUz?O3-34< z?EpYVIFt>(`6Kk~**MY;+vFoJv)c$^5G9L5P=rRrRHg`6MBY!!oq8**?Bs~$3;YyE zpzsxA{nJu-QifQ`ZB+NbYCmg9ExNi;=ZQ z4pFTEE$@cYK;jRemErx{EPQ;;~3CD+V`a{uxcNyqrK z{pfk=NCz~Sc z<)q!tfdL;1eTmKG4czAx*#&Rkjw@j%_ z2o6K~W{tCN=rrueu#sd8uH(Vz!$}$(RBL6JH2gcHy0Wcu{8>73mTFdG%*FC!o2>pc_WObmQljuPDe-UH z6|MDTdam7~0xHI1fOVG%jgb!a@Xo)>qovit;Do+IT?X`2=Ynh}Yn}RK2+GBmveLtr zQuUsmNcbO5bYKB1EAwT|S^wT*+!?N0CD$8+NDt$5-Cf^SLYuaXOP&FP>Hp6N<$Fo@ zhj9VK5$JxfB(km;Q}{yB2=TJfQTjF&uUnUY9+G` zgnN#}8M?}#9%n*?E0Z2?7ROR{VBP`#wzshZxU9)u&{+$Vp~l)M{E%9!%J}emsUB|O zZk4{ugiIC=9%H$i5Ll6}mhY@nDuiGCV!+_O`1GmKb^7j8X2V+9tm5TNAD51H z&K>S_?!yNy9ZDg^iLRV|(z+AWn-O3>R-E(kJFc4vDmAZVkT0ZQJrN60rX*~>n;TG# z&taqBd<9=v&(oEm4|>QT1{beRL6U}jmp=m}>oa`cg2}VdztWb3jX_rJM@K+Q{VamK zOCfQ_gul9W-Q0VyzQUDNd=QM;)|PG-k$0P3O5$g&QF?D4fs8?^0wmpL<*?wySV<<2yipB>ozvwRI&JdwmrytZjJtP83N2|W<7HTo9?fsbo z+`iwxgY{d`8PkYgz!0I;b+j(VC?j$^%i8!&xKguc%gZXL$f*^R4t#vYfE#(-1jHnj zE7^UQ!2`xk<`Gq+l?|YA+{Er%LQN% zWh3KW-A)D->0d1Yk1a%b2+srSDC^>@3~ariXRCG=aS%%^Q?@=)Z@@R6yU*?auzQJ-oo_D97zVLY!!au#NO7@VEUu{)O;6>T$svsZGMH3{byXsfTutKs#qCq z7cJGE!g3+7sYAz){c&5R@P3i@bhq-1>LZiKf<{0~QKv+dAIo7M|e$8FOPbkL)C_ zPP}t|8>GvW5>gAJ0gX&$>N=$rAf2ntF4q*U<<6zsw0 zU-zDyn&}jMK}$>etaQ>D@SP6TML_i}4P1i;@=wz$xH+H-2Z45tmGg;d@huG)E!-OJ zx79lvS|@*7M*AOle+o(&Cq@M>Y{Bwt5_NaSY`w>3PRl&edK%FE86DYyV=F4U9(`{Q z7lb(qdTl%5|D#(vWkn%3O((XVr4?4HEmL$MBzEb(k{;Ta(*IP@AW{`7c#@26x78O6 zG3G4+?T4`o0+aY-MS$u*3kwnhnb>%~J6CDRrBN=tKqkFT>ONt>OSqC-Dt+vVw6N69 zk!zVgNwO4;$HGfQUrmTBm5iclALex!;?r%;uR z#-d4&5S;oQ+L$3Gw)*pojfsx6EFS`;>+pDU%DCe648(eEox2QEw+$zO^x zZ_~-uemahquiFZNfDSA~wPnc$8Lfyt%-WL)j9AyO`d0wi4twl)#^oL(|Ed{*Tyh(E zgj{jl_4!@?n++?%wk1RqkdtQ95s%`AkdI*Ec7?9>Y7N;$r7UW`5t4$Hy_slm2i^_# zLVqn`>NQ3E@v&ByKU=XBpuf7R!7GS=cT|SE`^A)aJbDYcQ5q@UY>FztZ!=rrA9Bo2 zpZT7~S0S<*ZH`*)8uz5G%l>^TnZAmUx+!j7mFgaR8oi!f6>^7oH2R0kZbeUdI#q+6;oV4O z2Ym!*xs^G0E3*2r-Eaj0 z0_S21rR$iO*jgk+g)zr+&6@3L88INGtsGoo@08r|^pf(6XS?(L{m9@t^A+XNgFRDA z%6YE2JP$sW99Aa$jL?%n3g~4XKz$*)+TH%45yDo2zFE~?+We=)=3szyT|uMCVSKI> z@Q$u{^2*;SDv4BfQyv6hNyilv$bqo%Ar@nh2ZK9vOFmxQ`nz)Z{W@ zq59W{Cc&9%0u)Xnq9?Is-|M>(&q~o-8jd;s_*k_+0+(ymD#U26jec ze6OBR=}kBSt7CC%vr{v8E_k$iHF?>MyfX?cz$4rSSqeEMl_pq+EfTtatvQ_+T!T0XES49YthH)LSToXB|6T`{ZwORrM> z2hnu!hPah1i^T2R$+g9f($xHWIne-f1Wk@aA^{LXl2D603*brKq(*Q#+_252nkYb& zX(fuZ;>O_1Z%VPJVw%Zfs>RnJ3dVJhf?N5#f_|HXFQdIIK{E-8SpM}RPRYynE~pjh z2EdJW8bf@M872cXf=W#HW3VG0J$}_s#(be62N@UWe!C~t?Q*ls%c46titTX<`oI*O zZD3Mz9HtoK5ZM|j=rY-I{x^%J+@1@yLjYv)46t)(i`1TI^+5aE(1UC%-g&q&uX$XQ z2J{bdDb-^>>Q#?T#EO%MKJPDj+zGjifs|ff)|^Z>D$cFyPt*j+$X4d`M`{qLKJ=Ch z8f%&D(6}J2^wamfXOmt--ckovf2RJraxDE$tUsxk!vy)FMEULJ$!uV|7=%H18tVt zVCm}=F#d>L#9RlO7m2w&W@)XA2QrbsKvCDc)CyY$eeQ)+{lp5>(rNwYMprP&t(Evb zGMs5h)FeBdG^UEt39eR|x%MDNX1ao9F^gD3#yz;rhD`l}qiW$Gtw!Ij@{sRVsR$?5 z(drjY19cXTXdlas^4G5GhdYH=Mf!<*Sxj@Y-(6ghs@?QaAw!#ySG?A^$v~%MdQLh1 zKMy>w0D>=%h{zyEF<5(2U6XytE>g7Mnt~&ajwsAFyp+ z&_lBy6LHc9ef+7zJkqnmiB>ZM93F7UcR0B~v31Q|#;NYLDXKPZO)|#*J=%{w$K)~{ z>|sy5+vx@+EPtJj5RfSD$6f+Yk||AjQ(8`8oMJLI?E{xi4@BNRRk@0=R~{j=n92U^ zAb9-`(`<>*SX;4WMolEf3c?YPWoIC#uMGB=NR(VlWMZ??x)d=!tp&}j`Ya50KP)-M zVM1+jfz--V7$e0+h{Gh)h{B8|{o2gq%ONwO5on0O(vA@pp{=S!#>Pr7;M4X-?h2Dt zFsUwCn;K^X6@LUpm9h$(vnD(EE=_%Op7{nx3)$P8mXpXgavS-|rtyZ5uuq?M5a`g$PMQe9rG_`{eME5S3)ir)jle5z(=tmLgb+B)}3 z7kj-{c(me9P8_g*p6xcg`jsHcvh<=tG~I&r%0uB~K^WG!P<@u>2XolOX0rEgr(g5d zoAuTxG?)viK3BgNUoRv@AfzgE8nMfQ4LEToeMw2O2E4|O#m6!?vWZmrB2=8q0KBwj zovunbt3MMC^u&^)3PdJSvgIAZOcigOLUZW^SZq0hV(t`o-u^nEL_Kkh?p|_v6VU9con|&bU1X=4_35t=moob&n;04;TG->yE^xY1EMR zxqtDE1@il^HegaI{<&PI=MhFDvRo_D2UbKyw_pqIPRb~hEbBPl0!fRNTOG6}MWY0F zTRQ4VXzlR*by%`wL?bTdPBU@xLMPja)5kMKF>czoPjV@wA883Y!!3do&m3pKid$im zYo)I3a|gkIHsLs9PIl=nRR%S+=O(w>b?N4p-7 z&dv8GC>t83SE)sn`53A~gz2A{DXU1&h%Jl#NxMcFmZJUFi64?hmgxI{pG@u0d&lLR za?i$3R-K6?FI!fjyQNCm;_mSnpkObOj+R~$H zz>?WmDy{C&Hp1505qZc)X4{)okin6kYNR%C>`1$3H!6OE@G;{F55MZddw9NAm)RYzJ_p4Aq-c~-f9ya zM}%2oB{gCr;tSMa135zfxontIxWLv&!$b9tZNu`4i{<@D^7jt$k|0#dnbI9pr+JkV z<9-Jnl%0H_;0kh1J>)^_P9E1X3BE>xh)2#r&QyaoBGAp{nYTz=Xz!G2whKZt2jPO` zmnoA>u|M2)o%ZWDzvulPbM^GgASHN32O#}0lF)`_l|1^R;Z=^u3(!|4CexCithSax zH-#VOx$AQeFFLXO1E$Qv=-@(gY859y!=*JJ;e+*py)9Vd zLE5^TK%fzkHJe>C`y^5z*>d;^Sb|#vIkn$}T2kSdg)Vo;o+*=z#1EdZ;>9B||E3z6 z?N@2?v;LeHU?X?SDU>zpLl(Z~BJ3{XSIfghb^qq@$loo{+#{+-nxUO}H*RC8{>Y6m z#%o2I5jZ_#PGta1i8!n;*Qm=k-xj+l-xhSjOpd+Z%xo*R67}T1u>7ZYvy8w}K3hHw zK8>5<4D#5Tog00Zl?TM_A#(j}iz)J&H%BYYWoIA`P^!Pf z7^8PzIOo}EHAwVEFrEXTF-ZkHj7ZEAo%Y42CW2U+U)Skj*SyIMyy?+dSQ_kv6-PWm zPeP7wz4j47uiI(g&>4aAl*=Z%9$|bKfS58GCBoS#qV_fix=C4gy1);MTpPxwj3Fo7 z4r0m5WhV7#vb_9{JI3Rx^^h2K`MNz3)}2(i!Zx0{I5wG2;CbQO?a_i?cZ~_08WYYX zQ}n%<%7Ra5^2(P$pd4>LUWvnjH$gUsHwO9l^ z!-2)$SwI(O5w^vJFcaCg;2rT-ylxuT9h!MVyv@{9lfvk&gzg@gp3$5?`R=v5g1_tIANakQ$Cn~M|>z#jB_YqhKm(mCP{xQ|9Uh#z%TY?{Mv3!Y_oiwk=f6Ib)d=*7Ht4;(mg9fWsw zA*=MSZJ;IrvFAe17wmPPFk5@({&b@yKdoAcN?gnkt84zm?OfFf*;|o&mofeLR@**X z!tC*EBCxu2>uST7IgtTcOG$wS2HK>FGeeHuG~-_AGDg013pyAGA+{}O;qsWN$+i3; zqRQ#;i*g~7N{3CildIeTdfNt@!U*{&Nt&(xih1WuWl39gYo{atcdq;y=#&%Er&WGp zB&mNIT3&kc|0OsYbUF7509`Zo@7|ikT&mpkoss@7BEm-G5n6=CIMvZ=V0{_B0X*5x zgNKNeCqsU~;M2n}$4*~m?7l5fm61a$cTKU^P;hkKE`Ek>slqfw`j|Gxb0r9>K{3BMLC*-R}C zIXzOm@Ypf1vYC?nlCfD-KyxQ6H?-ZW>bAWLF0vR`d5H_fiiN5Bevv*BAb+9yv56T+ zKz6@fX|v3$c*_PxXEj|yhbJSO_t4^duoE(5T4s?w-Agf-q5k-x?eG7}Lu`wMaNFTG zsony~24={}^y0hmkSZs!@}Ke|)8F8r94?~GcDeBuXCm7elE?!X-xb#K8FBiO1gDH2 zs%R@wgtGgc?SKLjmjN#oHUYD&XOUOVhmrrRxsU2OMpc9}>1oM{hi<9wHuvMs+LL2a zP={JYXct$u-xl6d%2p<&g{PlEehAtFtJx8x*`^oXq+PP|2F4_!hWK5+vBF5v4$XZ} zKzwTay6kKncpbUXc97K$jo+86#VIP)W_{(-sz>KMYvK+Is86mw=2A~HhN!@TNA2U% z)a%%<^lbq^-RsEZ`WU~Kcy&xlmg%h&UBXPU0UZ4QzON%2wYG~ZFU zh1(zCgp#u$b3nhTo9$Mp?z}W6fKdV-0U2J&H%1p`DdmONI!2181c9!x4#+*UE{qNw zR?Jjp_ZX$pY!Ea=Vmz13K;$}5<>hzLB2y*ASpmxV^1?F@&K)@|tNL$z6niG!tDH-u zZvdCG`Oa4ISZs$d8r(nG-Bw>YMCaM6qdvT1!&`lc`^Jflfu?f|@3?_ndo0X5)uX=B zfAw1zk35w0Jpx&Rjc?vsAw>6ijp<6_d*H(yOWQtyoTOcF8{jkdoUBa-4c(#T(lp8Clx(9>!5#WOPv65Ki3E8Y`tEQn-jl- zHa$(O%>&>gV^YWfLQPn++7hX)YUXy?nPqC!-kKPUo;K&6I9xvJITbY^XKNv&FL|f<$RBoiP$2@bV!-jFpFlfA!lB}L z$erF&7_`7(RIf0`5%pkBMS@RLdIkFW`7>y=1|f|fow9dXp`<@K0lGX<_4tmGG6(u->HaY zDv??hCdaSHN5|9o$4iuFquL6831``de9h}Yu^@Ein6IrG6$*?D*OE{1hn)+Wk9}%i zmsFH@BM$7u`kvOdZI1Op+;FgC7}|$s9taTG!DuOvX(W})ZaS!X3XmS(#Mt_#{xmY+ z9_G!kE?f>g=H(E$i_5Tzsu4OJO=zsA!3eZoA*PR3XkHQV> zWbjdy@#`0e6&RWo>jl>ZeKW;U(tq}#DuS-S>-E|7my#& zl4-Xzqp&8OZ#GI@5mi#1V?}&8y$2a?Y1fmhi5Z%Dx0WO!fVswZen{{4mcEAVOs@{>cv=tY<>8PF6uJonbVmdhOXt)SeP$vDMu~5+{saLQ;>@=W{t8q~_c_ zQUdK8Ty`r>For^7J7c!*jHC6CYkqXS#H&-=Km4Oa@3>nYEFjIM>}QLow=zcE=D%r6 z_D8cKwOnK3?|X6hSJ|5;S`Q;bAvsS3E{Qx8Q!Q|BV}}$2%bP7)I}hAkgpM-?=rO#i ze^ols#Wwwt7g@c~8H4rq^;vSWp-4x_j~8dK6(oKFW%8!pvV6_3biEi71XJ*M z7$j5yMe_bJ`Gz{zS;db9f6&3}S;bUJ1{YO9L!ur#Ekr?iJb(&`&qrsV$xNLdR6kzy zZ4cS8uq0JBl-L1qeC*vI^>=G|Z_a!=BqZRff~Y1dV$ScN#mqGEXcsf7($bvJgXtnf zxq5X1Abdj_$j+Q2-IgIP3yNrj&X}n^Bom-I1fXKshiD8I2EG`v*Qvww*yJ2?XlnOT zy$X4}?A(>MdY@wJ*HUWgYJ*Io{^c6adS~~Q;llSyxa8I@hRNI)kqj02AVk`{Y}g8k zyfVFs$;C*q-^PXlK~+qm*J1x)h$X_oD*YxgmA#f2^F;OlW4fdlP)jJ-I=~>Kb`94l zYl<};)!C|xQ6@37H!)463^mdd0}P6g6vG5oUc9U+SqAP}BxS1Fw!fuyka%8J(n?wr z!*!^GCt|d`2emryXGiyG!p%_7zo{EOZc2bJ^bUhos57u8Hap`(G6%q5B@bc(cNixY zrzFF%S)0i|j{b^;d7L1!P!B48cy$%#CW`i>-B1aoK)Q+~(MB=w^R6D%IgWVeQiRmT zF!-zWj|FXdrPD<4HCmg^BjK0ZDZAZs!(K!^zxtJeFc3Ljw4eR$Emn{g3B|O)TBStl-wHd*{#3@vi7# zvv2(K9M(OVc6wrJGm0PU=|Wh+xk}5F^cklNuqTOaW%SYc2p9oRCTR-;I0RuN&zG&gP`MEbv zn?fOCvd+#b1g`2&SLD2Buqw@-HcO^xm#o&%FlB=cs zwh&<&ppWj7@iujB%%??_S-FxqXas>D1*Ztz+^b;C*KsUxp?ZeXr=b-IEhm7Cco`?L z!<4UnTQ4*%cfA^aZE9$Q82I=gAvgrGC~|=Y<1l=FNTDACgqhV(?b>+ZS<_R;ne?0w zQ8m=_n#)UfS}!Lh%yWPTD?jJRnN26p%`a7g$01W z?~1@M3iFN+Z3D^Ro0X+NexSl8VO<{^hS=2ehHpy7OvUEpMqe@n)wLE3x+q`btXM=Z z=`{=uC_JRr#52?{Yk_LvbpnXZWOn^y#TdgVb#bo0k(j6a7 z>#10?PCBH2pEU?T@?Qf4-5Lu1lm=TtFD@Ac^5x6sq)Z$Z=_^wz1n`kjwc6*Hc=Oi6;5fj>^rbaC`)C)d}kg4F-=q6EF1E+4TE+119?F8Eg}O zCvoQ`QE$0H%h+kiv$0;npGWQ_D}ku(E90ZnN{~?xd07coOEMO-5DJ+`MEG-g-QE}) zqrF~rQrUn68X6+{ycnSc;JQniVx4+($k(E|BP)=|@zk>_siH6I7BZckjEblz{1z0l zF#ECq8^SuhsTv362qWiT41vA=YK)2qB+yN#)qQWZa2E)@#qB>T;nZXG9+o_0)YqPM zfPuzEFkmfk)y+w)I40g33Hl?9UUpc&-|4$%_V{4bnZC=eq9c;r$*}?Y5HY{Ba@o8x z_U0>|2Z((X>q^_v8*&gBSbhF{*;l5KjS79=EJAg07U_sDVmArMYw|iFI9@r7p^ovh zVNTJIEvH}~wN;Y;%91n=dt~OCKMLndJ%N$CJFXP^12@wGXU~s+=t^nRr62zobTnKKIW!>&Q;E`E%~ zrYKt#UuNIO>XQvT@aL77NU`yH;lQ3_Obz*>4xt#obyid{cReNy+<`ita;)?P8Zq72 zL-n(SZYVi))48{#;a#k1XDBkepP*jk(Ps{O~!7_jURl{ZImB(7zYkfJk$e! za^Rs;TR=yph(vC9L-0u7!732&reOgB1HYpA$)Fa(l7#LJlq*)wt}h&tVKZiiTwqsX z=uh-b_<2CLv!waeXGUbs&*`;Qu$^Lo-isBjiZS}={3Eq5v9}4@*3lfCFK>3-0&vF^ zqzo=7)90RDM`$C)XTUD^XhX>2GQ%AGb9;*~`|`gbAPg!sE*DP4k4I03%`GZjHnI6Z zUWu$p{q876d{CNs;Hx~02Vn2Z#<`4{*>?)#(MP?YIv9=RDM|y{!0-spB?Tqw@=;%Z z{Hp-I*ffw6tFk(YZ{Dm_Uco4`JT(F6gUO0_aPNH6W^M94eL!nvGf@X67(WMk*Fdtc zwPD=N5gfpYB2rC+u~o&^*Tbl#6Sv-NSCVk%B7nXH1LJtBTY*a0AkTwBe<#-N&2K1i z!E9l|V7T^9P;P5RReeA9X+64^6$!X)Vr1f5t4;sVAM11fvIf%0cX(B{op>F@89e3A z8<6L-KYm(j=7I7uo`iH`=Wott;SMQaAx(P^Wg@zuHgVKhH(_;X$zt4Frph^>`O{X* z%b|d0@hubFJ7YEufdzqyU9D5Vr0F9G3X4kS>)l~bX!rBb5t}aF{SsNEaQoq zRsJE8!BM-CaT@4UB5q7;zLz$my2zwJM$gN3|HF;p8v-UW;Pk|;vKAWs=ZZFkjcdXy zSx4FzZPib^P{6^jizc7;m11GGBzvmRiEp)}NeRV=<9$irqDTh9lAJfZ*Sra)Ep!?e zPIeTe`-`TtV;rSMf)D@98R~1^EjNZuJ%+mVRB^HI2laA)B^gjVpl z!3Dzk8=k0F!F?Y@!19x8)s`ERZs94e6RbP$+mfH{b0K8&kHxN3-)UVcLj~RrC_LIo zu4I#TYWg_sR<=I>(1Yol!oBBjA89RY`>YyrFWRb*mho5_zZ*M{y#(t&+YoS}q{|3R zi*@NSd++l&DVp&Vhe7OIwHG?0&2Uy0Q44nQfOAJ~FFWVJ4yaNG8AA?wvi1W@y2%Kh z3)2bl8_GC|x(KoLuLTvQEGf%p9-ZT^KX(`|R8l<%(r#8bepS`lc349%O;VG8oxxs^xmue@Vz-C8Lha6dX7e7TtI5gN%EbYyRWySp5m8r7c8jve7s z%zwBG`L7E^v#SDyaH!esT_(M}9M?hXbjdMs;cY;%7H*-C-Mvef92uex_Vkc!isvY! zwAg^na`%cW{C5_2Bp|@(fZ(XPOOzY({xD|z15`Ifhd+kgB zn}sa{`>2>bx_jhV1QqeQ7Kc`sHB_w%Z@%0pzW9j=&w|68Bu+iza7nAG!O+6;3@1W~ z20hDP<<&2xG$`tbU?Xn+?#8chGc@e_=a<9z8%}?d z$1{pnaDn7tdX8Z`Z~8K<*3VaACQ-h-qbJp9?B#$oH2|Qdio~?0jw~11l|&U7_MEB* zdMS-5Ub!%(OX0)}0VCYGa=RhO^dSJ2-+)UPOf99m3%vC2Zr;pRx8K0i)Vc4f7s*g6 z`-j_x1J7lxaY1$OnmiSt4=g8K{D-15s`=Ns$^<=hg`BkK3S>^4OMv^i+bg|1nZA(J z^voEG1MPr;T0IwQE+jLKgqBc?t>NqzW`v)yZ4%Y zd$BN%XXL=Goiap%(F;p!NWRh__eZ4R>s-vx8LhH;E}v9un9-ddDAWbGClhARW_imq zXO7P)0F}{Z=EJ{AFyV(H-uYG(^j(lK%XnfwPYqBvyltF}PHN}n&bdfTPG_%B-H0+q z^So8s1T9iLWK+)Qf|j3pqO_V{*vIANg6A+-_?D6wbyUJKDizX=)()Z$85^upOi|wB zgCfE)VpO#X$j?_van`ge4cr;#HpG#>6na6lB^GUqusc7H9dC9EsTbmh`{j)66M3`%A zx{h-vEom6cjlA-v&b0Voqug%|02267#4OmH<$tDQu#HgpOn0g_l?yrD6M?5v&P234 zsbNvIfIrx({$#`g=z_+%j1S_lLJ7(2<6YhN1H9o`rqD)UcN?wG5XxvDxPRS9K{i1u zq?YGmlnhd2p|gH?S&6F?il}1E+4e?FtHb2@JjzS$0ctuWmgZabg(F}1z=zeI+{g6n z^rCtIX8(PLoZxnG33NCa92<-AHv90~;dx!_sx)^6^DEQ$27@JL*P3{%1L~SCRtbS% zRU<0>AO3b?5hLkW3h$S?4Q(@aQQQ-a0qG{hb|*wUQU4gnw!9qK=IeyAJth=`9IHE^ zEtcqg&#B4Oc3SbgF$}VP;z0F@8O%3Pjb@zGdp6&8q{x4sxiT=jZpK`VV$4Cp96P_e zI2TIn5_EbH3!p6FOGIh2l4dNAGPazbB=Si)q`ABMAOKJxue!1#zN7r1DwHbzJvuM?Ko4UDx&p|PchC00?Bg$w5fTG%k2$4r z6We}!FS}PagF<-mjPT#menlAPEJ-f(+Zs4Wwvy5?XBd1q($#-jf_ z%vX7>-PGFf?tBFxWHVcGj>NRl*~@OnKPYi&W0dUhz=qhMtVgIWqA2u2Y~+&KPyPX1 zaJQOw^N17hJ0R~?D>)m((I<){WS6?ulAgB6!_99Z;_s!>@dc`{QpmkNdxD8mEUq|y zJ8=vLEUX@DNhKxsDKW2{X+jND=NC}(Hgo7S-(H#pj557s5aj4J7hi>(=4y4=I^`TEL=5KYsIa;9XX+FA*H( zMBS%iZ=wuhukN+vOWFVcdJH>&j=+ZE3XrfoXyQ$J91WO}`5!gL2q_NOT3ljDo13?v z#P_RYH35#QORC@IU6fV5(3u&*2`<6lSorFYf?)4-pqB?A!>IZ$khYWyt1b?*x9@Y; zU?+V+y!)MVJI$o?%Uk!^dno5)+;c-iNlMj1Kd-S#o?Gqscjyz>W^5#6Ef%>9ErF)I zzmr5+h^~e!E=vbz6j-Fj$0-KySsS63=L?CJ1gR%=V5`Y*Y>BT5C|}I&z0v^ zejm67qk6->o?TZm4z5T#gJ<;q$70UMA)2lK-NwEGK?;HHiGf+`kCgvc^n=@}5|E?f zJ)*Y{GD!d#sBw4zMMXf+BQ+4?fQz_le^Sm`RkqdA;aeTX%zaUnc$$$O2GkT_xw?d9 za-1jOelCrMh5qLQISo?XW|+)(S1UcWA{T6tL2VQ)dJt^sU8rHMCW$ZT`3v-7O4M)~ zAf&Q^dXCP2f{!XjU|OMc?r{O*)@a|P>J?2FI`oN=dCUlVqp|p3U0gCYrt9RM{qGYu ze8B1!z8?gx9`SHow{C=6xE84lTp`**7n9Xfxqp_?D=$rB69=XHvEHU{-R;D#Z2lv- zZFZa<`!`CuQl8c@}jEKrr%a-w|Vr6Rck zNt$Rxpj*Wq{yljFK-n1yFta@+S}Hv^7|P)jPo^o z5}>d+{>$qvZ3*|0y+lNV?8qeYaWl8mY>l|l1Zvdvk~HD3qDDfY%__w=>`;cj1;LoRzEPwHni3SKsL=aKE^ge<0^2~N%0dCvZM;tVmk#J#6VAxM zToYidn$ zpSYPKb>P}w)8w#%;Ct>ne$|ERZy%v2Ty08S5A zvk5H+gvigEu)zzG;@%YrTpX@MRwcN$-3s+Bkf?9&h<3}vEPcIr`ND+2Bi3CCBj#eG zXIci5?Bnt+$<3^Pwi5Tr37ETBHe+!~5Q0kRV;`x2^=I(cUl0s%$|cMFsRP6PZNH1w zhx5dVT-M8;$t%g6fvblnhuBlgoB$$$!FtT%3L>1_)%%=e%#jo8cfsaF`pbho(`)^~ za%1yiQ#;J)2gsfIi2Mmfk2V7B*Dmhd$_M-MEa8>!hwLQTJ#lD_RDq4sq+U0r+@L)M zcOL>hFq7+HGLvKD+_L2zOWX`<1iI%f%~K&4H29#ZPDEjL_oM45CSCCqsMF^M;*qc9 zVvr}JsQhUpj>-BM_oGPyqzgPoAVG-<1*ZwxJ9G|j4MsD$Yewdol+Ki75Q2%}wZ`B3 zp%#M^{ORl*jJ6_7HqH}TJrCLRpChL3)7*P(kOfT2tzsUC_yD;^FtTj##GV*<@*_t+ zBMUabgMC(zGq6||C~|L@#2>0;>*QY#S1XG+3WfVbEXkB50j1=nn)>VSJrsE`J#0|T z4uTYmxv5etgZ8LPg0Aev#3yE;3C0qtMtLQ>u`nz4G+>_wX2^KQDk=cP!@7yPOg@N4H}xBc(|Tp^7F3j;idsba2cD;dgu z((s7UoXdqe4Z&_1Ovo+4+7Y%Rj)3r0QfyTSVbGed3B?gYFZZU;4DS-+mpxEVvE^2% zU9mpG&Es{fuDKxI?;fUKuvQ51skPH@Y(0wPo0hr2?loK>(B}YT^)@D|fNMrF%H+v| zS)5aRP^(;6mhm|w$%T%*)dVk{YpHNk*F-_kUmUD*uG`C#yo3C{+Wk-T3! zco6)h2j(=+??lZWdd>t2dh}%}BO0xje88^clDtbYTV>+Pk$|a9u*c0JTP(OtCYCl> zOnP9L5v$z!d$@j#7giF zN!!*$=U~6zuwll5`+KwSPjJC-fv`L-J3I^G(C}xrLm4<(?F5bJCz~~r%te`GL@_Sj zWvy|Y{Os|_Qpcu1qru5hli`;p`3Kf3l^W`s&-|pLE#`=8^OIWErl@igmHFLZ!B6$epTd&{W%k(^a~ld zPn@~<=(Ym9uwKbWi>V>eu;dkD;xTLvUb|e1_;RY3|kQpg6Z-3MWM`e^99UhQ-TM+&KuV?R@55^SmPhL$o5ogC_&_*vW zE={vdfLsw+aTNFQvIP%kWoqMXt#~H{Q3iqDxoKIFeM$W+A+_2-wr~)WCpgN~-V;cN zXfAC~ebMGpEep&n&~zhBV;!C_py3q3^Y#3)*-fVHz345Y-&p;olRfGWMc~{!Tx*;< z{7bsZQL!ecV*dc`XHLq3?72o9QZlVj6oS4E9sq1_59z3McRD1{y4vrDjR6B*v$jUs zl+TiPMzDiF7G%?L+JdQ7Ds}QrJwcodv@|x$Wfu8=tI#zAO_Zy8u8N&Tw&Ay zm>4&PvFnmclvzBooM|3EKcK=kBdDrVSeVF0t|IKaSLmx{OnoCD253Fb`*gp7?FLjE zD5i#Vjfj-ESNzzS{M&2QmxDuLh*d0P1Y@g{%Sl(fyojkRqQy8v&24hW>zqSb*MADi z8_>0?(%FQ@<75JpvqmfV%jXdS4x*4WTUejCw$|!QER^uHI5;FtL|bA*GpG(`_l9Sg zoD53IjB4r_P2AQt%e=FA-qxSd#>O2{uDp!)9rL=JG^Wx1Lf!Wd2^U#u!-n(qXI{J4 zkCxU$9#Z4x?ENZ5spo2ws@Z5zc&~#)gRLcO-MsJoU!J^P)7NvDxXz3f&9*K8X)=-n zyak0bzp_xJS}KFkk+|kpXD>Nr;r?oR1Msmz@QfKA?UXs$KPoOd;($j(MSQdX=LIVV zZT5!aypi=*)fHLFgn~Le&KsuEVJo|R1yYI8|6Nn=CLuNxR)l9G7%+OQuYuc0^RuQ@ z)f6WNrgn5l)q=RQlt@iDn2g90j^}F;^1CF}SBKRCepOMmI1@dw=b}t# zoTY_+$Y45~zN0m1jLs{Hcr`vPetz?Uv`Pj{C3uCr1k|Yc@Z3G7&$sK}??d<7i)lqH zt1x%@uE_<1GoV0z?UR&FO?i1{5K-cHkDxFHulv4En0 z$I=`tJzI#;=}CWmy70ZX3Hc+6fTZnw&{Ba9meKp|J2sm#YSNQ z#U4L|<>=Y5$+ICW!K)3rw3&Wb6H|97Ri8NerC{Tkt`dMkR(8hh8)Eq&KK35w9pRKCf&Rqm3tLw)ogu@iK>(o=iqiw*P8|_(SvvBJ~39M8RZ*t6`&jss> zT%CzF7=^^0Hoxz5N3E6L}Rz2mZ$7$=$< zdvSUagn#q)X%EsI`-hhzia255)+Q+OzmKFf+4AA}+%Kh8*clq?c*S@;5aL2Vrq~#@(B_51AKZX#`-@w?_iJY{_N35GxrtszT2ly>}63PfX!Mg7vK3{k!7;=_k zYsgJ0VrA=3XEYB(rdhYSb2Z<>lwyG2w0bt0oZv2qGh%=uG|XJVsb2;>Pa5R^L-co8 z>yhL=TH!2vQ_9yH{|+Y_=&*$7OA4me0y#(M!8$$J8>ot7mB#XI>eH70n6bp12EP>`(t!mPPLHUeSN%&{NJuey zzvTJT!on!k==p6qKo({+FBQDmA3_IZ^RMjPcM+%#WnxEZNwr*XFwNdnXrNeur&LOhQLSMM?h zOU6CjyZbh}^mLTUu&dwbg0=I1YQn}guBK<5ed4oOYyyB%CNv`L_*c8F$?e2Z0FMBw zO-S{f&CQW=2zvZTpZ!q?^uN01=r{1al$dlqPcZ<&i0q7{WJPK)j0+}&VUnK`1=X=b z?Fxuh@Cc3uV5mP2qke+~P4IH@!Zax`T3ZmIz|Jo!KO=+=_Q%wpHhz#O`@U`C+X_js zj^YH~>}XiIN=$zrxK2G&aNpGyFLWU{sm1u)5cINPuv3ab>Czq_z^)f*bK}lOoVd!N zPpFO`)%%*T>xUb)R4lzcH=Q*WD(_CGanj5p!+OXwoPdr{wAnUSLqh{iIj1w1TBRK} zDv+jaCMH=pC>P$h{fYBIK*P=WpD$P*dR$ffkNc`@GVF<m!QP`2ESVGa$AmXcn<7`?^qqR6;tkI7+u$Z|ki! z2EY}vDP5nI1v$)@>-tkYUJ8Xcl0oLx$ACbq*(LhCz;Ek&<#Y%bo-)6QiK3yK_^eR# z`C0*?k#(nTApuw|JzN+i{PYYui5P5QB;69!2dwy#t7pa_^A-I_XyYodFd(^xBz3(q zXU96GON_a-_vSX~E~1^T7}&fpdaUGiaZXCRz@-1(YRZ1#z;@d9#gfax4ON#vKw1!i z$%(qmL;HlFO4hc^$H2dZp9ttdIn|)&AgkMEQO#K=T2zTR<-GH=A-sv>E(TlBj4h&} zXi!`*z9PUurfyRqTkulpD8ZY1#tlg~cYwk7qP*~&xl*^IcSK528kBZNq4qeKi=gwx zp2Rr)b@-4f7s(thw_?Fou?2Y|)X-T^zq1O9}u0D-HcqH-ci2(bB@G=PI< z8W(rh{GAB?+Or9o=Lg+!ypsGWLJ z3@GFOQdNav=K#$^(7IkVsU0YM9y2?QuJDS^C-q&DK)Z5{2ssg(TO@J;mubLj6_ow_0 zOqgceWmgvn`F4=&fnJhu5EX~VA7H#*4$8=#% z$?0hr)^q0!=LO@Tw;_zg48H?DQyA+GE4a=X#CtMBy>#ewf^n@ldymaf7O zvp?1N?}p;SwqOZF&1_{Wj|+aI;+c_o(?5N&AI+z(_6{2Wqvf zi0&IPYr529^Qc38H0bM|SNwoVc0=iTxsaj3uq!lweESscvTquMzAfzq&d%zZP&Q_y zV@aUZP2a(r)0lHOU)MAWx~$B6EYblU&BJvleKrJGt5oUBIS9WoK_t)_gcvjReHzQ7 zka@TzCOt={0k@bO-=(JHSnBm1>sLSz?Qlo9%1#X#1Hsb)Yl?7ZuBdxw0bafT{IAzlH=6*B&}@ z5SE=!W!e-?l|a+c-9H>gVpR~J zvI)bL{5a^ISc@|9Oru*TVB#DegO5p$Pd`{<2j(a>vwCe2-ZRF=2*T+mLex!2Gd{Rt zzt~)dcs4^<#^u>TRR71Vc@EO2JFU9M zFN%`@nI$e^==EKoBZ)V;lx4uyYM3^*laK(+!`zJ@CMv9b9r+vRVVLO9z?xA^0}YY= z6Z~Cf|3$S_+*pZZND@Ua;>MD<&GaMJj{^9DJJqri{Z=t%-Tw@MPT*u9Xn;r!q(b%! z_vF(x?=?R1)an6SUe`XA2obW0=&UnE`j+++6@9Ct93&~3lya76EwV6>HfF$3Vn!Fq z)ybg?7;e9*9j?ggp@P`*Vw+9u)XyRqdl$0yKt|q}88OCe@UGQ=PuEuOvOrlgy=O0C0mcD#D`9O$_aL~v!A=A)6IVxiGtHy9`(TENc+ zNx`H6hH9nO5*|K$M(F2Ws~B$0b%&K&+7_ntRG~)6m&qGwmAhAHa8hQlxG9fXvlG~U zNz#*9=$}DNRTPVI;i;r$!-3sjr>dtF#E!~0ds&kD7YzWG5GeNQ!zXlh#~W=K^A}4G z{kAy{0pu=r00Wy;CcwzV+-ZkFvFNx5!YNZ}>xKcJ>@yF+=W z7wB$4?%x!EYpR|FtElOxKx>jKIxS^-VVQZ;vcMA_<Gx#X=WG_cxaWjxIM1Ae%pw)c8FE$J>NN|S1*g89_?+JokE$Wz)Gd<90WuypE6 zkUW>qt)GPk_ja%pk+by!C z@Zfa3;GnUZ3d%cxey_xqS{ma%NgJDO-*>d%4<~u}8kVXFb=GO|_@AyahW_k1`z|<} zLaNRDdVo{|E%LD?m@9W1;J$nLr}5C)6fuq$6xeSqlL>49T0foSGk^LKXQ8&=U-2u9 z^QD7ARwU|boN|85cmA*bj>73b5>{@NVCNy~#7&5|THfJem;tari|ey1PYoFF&NA6G z?-zgFljRDte{;{#rFmej5hE&bM#7aiRT=jeJuSuNeVAh-CH)v^CnLX^nqhr@{92r0 z(&cVM5}Tc}z3rt+dcyCuE0+%JRwTK<%v8LNgP*N-AU)XkEV6r+ldHY93XE!9eTiAri|)ilO? z%RiL4*-dzmI~ZChOj)GNBMXNlo~JJg6zzeFPK~2zf1=#D*q<10V~Th*DpYvDIBa*W zgY3{Bn}|s=$8Xq@Gq!m|^FIz%?7!&^`_nZb!4=GJ$;J^Z{T%g5ry)Z0`0mlo zC~coKVrl=VqA7+sjjGC?ML!4k@8Ri7%i27v3BMf#Ms`Oy8|ykCvS$RmjvA2zM(BUi z2!L}TU-ofiSc!vGUVJ;3d^#tZ(qU=?RDN0Yu+F(9@iP<`6Uw+u4)@AqJ9m<8i*iW~ zvN;_HE%j+MLJ>9~K~U~O(f>z`?a>w0zIZ5x?3;P|$~bUr%Y*W6KSo6iu<6oM3kBii z>3@c>p+8QdY0^KvXUhNJ2f`W^3Gyw+`Q>kew@YP{&q(2b&oRR&c~3#B$;DC7UX5~x z&(`mCHxha@fVG{1a5#BX*CF@cdP4h*|6bUEXFJ3u&&!neO1(e5XyAk=w1C4#qxWxI zn6mXml6gKOO>uc!agT&bRUUuA&8TH2-!20f^oNmt{d?^XMpQ@s3EZclJN_^<5C


+?_c ze~89j!-V~G894T{J!;b%8p*rs;|T}@k@<80n@2Pl^keZ07?UiOFJTt)raY7iO4cgC zDCHUNMktP%>U3vLKriQ?5$BU=MJ0A&l@rT;BQtzuIcuZFQ~-j{b(bPaA;A5$4m~Ye zzmR^cJQQ`63Pe+`c2wwr;P^hU2#iCl+UL4TPEHUX0b~zPbU(w9{;hG7^?FG-2abzl zBwP`k$u4emph$rO^0`2~v}vX*FDsuLB1JU-qwqX#CRmT%2^cq%RvnQezvj0WfZLHL z`S0kCEo;?tK_2>7PQl({iUVS@b(3eqL>_Sf29976J6Y*^luYE@Je%uomVIAp$5pKCgM2FD z4EHhTBcYY9_pOp8>&@b7yc-40=^fdp#uAF0;QHc+Nk~yh&T+|@p{(gf(3_|=1u%BB z#^`H=MV-F!+5YhA0I&PEC2}DT%aBx@U(85az75mWo&;gJ zl?4fAba!pz^Q@8UXOt7csy8d!Tc^ja_v~3tqRMDnr(@;US41Et+cLp!g?N0?`8H>t zuy;Ne7JCL4jfUk5M*S!E*FV~<2Px}fo;z$rB2K3hZ5lXMwC5u`I*BK%=;yClqd;-m ze);vLHF(OXV^~|p8+mxO&u=az6qC$Vm;|5ZN}P~Zn;45iRhmq&cvLeaObaa}gD3U2 z@87;&c|Vp#?yUZWGiK;rW((skmZtay$Y>she?noiEWj=4?|Q+8{$vlS8chG`{BGFX z-r@=u^<;E$0?%1Yi5WJzdKfmpfIgPO{Un5`B(8`k15eOBfn;HDpm%00*hUEN4b)dU z$jsMWv^1imgIipaK-C)=hogy(gD14?%6R)@GV*#g=y#Up-ONmUe5bz@M-@>ylo?|z zF-GOcDUY@nz)%7R%2j6NL2M!&NxEsS}KTbodI7I)7*YGx5AI zMb@k?#&9~oo9E)whE{Q)FJnddwz~=4yu&ZWy4EQh=rrE2;8IbMVfv3+oCeb(VA&nk zRZ{fh-9*NObdO%^c7wWu^OV2AyCO8Kvdllk81^H~*cjpn)xJ(ru(fOV2i^#NvXw!Xoz${OV|wmCslGxhC}-s3K_LNF#ys!luo0#Lml%t^5BMpJ zvi$uw^=*lDWR*B^6EoEUe@iAmoFeCvc|QpJ=A!`Hd>Y$k;x9i$6)%sctb1JqY_tpX zf~^2ES@q|pMS%bV(t2H}$bvwH5h1qs`cM?5S@5uT>Fl~CA9f$-z?mBiy&~f*ox$F~ zepp~x=Ta|8I3Fj&2>j$)>m6lNdZ!>pgB1j z1#64Vt4@qDinE*mSBBN-mE6ny+I8a-S`{U!iyCY1y<$WE<94U973*-W8R$<*3|x8b zQ>u9vm>xdE{~iSqQS~q(&8tFXk{tdILQfdjY;=VFMP`#)(oXzy<`9?02V^QdtO4*L z^prOU2OPzK#nA#D?xJf=Mx5!XO~vbpJ}mkf!iA1xTi>3z{sO#PN2>zz|`6YIyC z+Og7b=n5BoU@&-`{Hd~JDw6&ap14{~$s+zmV`pBe(lEh20L|j%Y@9Odn7!g>6I-|w zBYue}Q|!eu8Lx?3HgRbDJ?%uW;jRxW!&?RWd2!{a7(4FngzS6wkWA4}$n6CSyw!(= zuIK|Lz_Eh!a4C@-MmCySCCPmUpcMevZGY5xHkAEQxeWs!-@t7FbrLl0qq?rQ)B%)R zZ;|+IR}UK(y9!~AOkR+am*^1xvs+%lcJpiauhb^IfVPLSh4rs?FMKn-^8`?dJevqs z6&<7XibmEmrLKADwvh)ky2zDz`-iUBdTTR6qL<5j`2OzT#KE$yyVgJ4Yvu-|A4UUdF{WZZ2S z%JQ^f2aFmx?Qj*-*pa;9-;JT%Szs|)wpU#!?p9{1+KG|y;W&61XfX8dwRkZaDH4oF zKE&f*Jk0wpBbJez-~eLkH2`ZHbI}7`#8gt&7RYT-H1S@&VH$r|Pn&JSR`1M|?W%3) z_E&*My_`=jG6l4}fGmfLZ3@ z9>^+{Nn1F%#&WXC+t_0@-|)Z_TkOd_)wDS=)*wTOTwCqpMR#Yv0kW;TyYjIawmjAru>VMGw$jF!8|5W&G)iNlY%2G)JAa6i}k_(%WvnCxGz{Y3E8>M3(-+k3wJSKNN;%u%tg}SZm_iw35 zt6)-U3krln6t5}%Mm7T#;iUkysAFW6YvgNoTUo8e(D!D9 z&k%!(EgH2K=)g+MBZu}~fBy9sx8TZwi`mwL>lV2cUGUyQRW&l%;UC2iY$FyS$@GVD z^mPj~r0QlbF*VH!ypr?WMF{Uj)P)%BtvZM$v#@xjD;EmwH|w2*|-~RRxG1@l)_ruzrv9 zHI0pOiP*x1p1c0;)S=s;`+@@0pK(=nm9Lhp&tZGh?c!A@pmZ8=2FZ}1QP8O!`7{7< zsY`GeBCc6a4LR20yBUWY%4V&I6T~=CX~E=hqkf?yVm+scRd!G@vBY@yEX?v9U{c5H zqY~t9=$yi8ZRkv?N2O3C6$l1Rz5!JmK1)o|^^Q^DVrO5di~=&2Yk1KT!9LC#77jt< z@M*;^Q|*45TxXr7NMJE7GQzV~Dxhvx7xMQL17sveP=&}DL5lhiLK-99{a@nygd2o( z*<77I$3!LOpb@rQv?io&cpoxM%n9DwEtwh(5Dra8xQ%A78?Ekg&cCt5&${y!MXV+*W64HtW?I?B&0He!?Ej4ISv~_Q|QPP z{-R4M_j>>wB+g5ZwW5@q`q4PR*EW5J_)K?JwOh!D?=(ijAlvmcT;@F{oqH z|H7rdC6NFvn+bF7OCa0Uzdanofyyd6T_a;BP@Ux|G&_`S-N)|ZK{bBN!thOg6q$uP zk9M63``N6+r$`ryW9nFVb^9q<0Kl`eKKPk6Qd;ZV9~F7t{~-Clkc_g1639OrTwavR zGYpUZaO_4RHW+ZET@OblC+@+AbwUunK!>=elIUO+lNp;`EcH(@x(*G{zy@N0o$UUe zWNG7`Ig`D?@L5RS;Q_MB<;GAz{1W^Gt#TJ`sJHQ*O!M^8Op_DFb*D2Rb5kg5v?2M; z8MElf=v+n>IOI_wLeELGC_pg)=2>H9sM*704XY?Vf0*2(?ple`*vv$rOQJj68B?_F zvau4op;D(x(ekc;w4xh|Utb4{89d0tfNgrxZWi}i1O%=1F6fHh!vDwmJ^kEG;-FZt zB{_!5-`wzb^lR~b zHA*bRI!)aWf+xxwQ5ye$7;~5te(q$8rcHgso5Hb(-28U}>1~bdnur`(y$wmBmcp&r zYfvkL#(X1oEy=mYg_U|dF01A>wJ&YL?QQLtF9qCk+Gse5xG9-;(Wrgk{-WV4ekJIu z1h7O za;Erbxb~dBWKm-lbp>afPwT$DXT~+p#~Ov`8x@dhl8NmdESPDG^Aj5mUFR;6(O<@t z1n>Xz*WVtR{)Ca5Zj)2I^OATTiP4%>_%p(8Q5F%iEQF{V`GDN^PiDr;x_PSMX^+kV z#c^Oz6q6Q7`Fup|S_|b?6iLpXpdO2-BRc@IIB5^&cEX_mpc)E@T%N_LXANC+m4gSi zpQt<>OG!@SIk{eh1|Ltzb?m%hkC)%;tUdJcx9d;*vYH66mDFSpKL>6!^CMiO(^XMo zTMy|rc>-g_jljiC4Qj>|@PVe(B*pJEcy%|!%N^L~xgOkxTzx-xlnDi5f%cjoFMn?7 z87?JGp-Y$CSluB_yjVMkDL;SKxf73dIF^)w8o(-}Mxh9HHwKV&SQ_>A$|kcy4ZPdm z$>@_^!?BswpKHodBf!Ep+7Y`M8|xiwI9$JhAfCH`J*ioahpHA$1YpfH2fMRysv92Vcq|eM0IK zT~m+MkV#KFM6sG}kabfUp}49Rpm5Rj9iJi@VJo#jTX!IU0U#5(zxSt1<-~`3_gfS` z!J5fCiK4iQzsqMj(+U!;XQdLj>iONV9nhmOA-{84zJ8?=tbGoD(x%$NxopSQXa~Iv z*&TG$g1je8Eq_Zna`B@oBu){+!zgUhIoET8+NMj}9=ReC+*U^|N*=#5dXS_0GU+m| zQ<@B}^}c88TAvYd{;SOQKIZ&s7Df(kOIb2f$@CK9vKvWk0II z)6{iTjoa=2phWcBvD_01H7*B1SyI;MXoM2IwaM%rLrsW&{*~g~pE%}HCOY8N8c`j0 zwm__PIJB*dxU3aZ3Qxk6TISBys?vA8&7G%r4!Q1Gyv5#AdTa`a`pixCHSMY@3k}03 zxeuS0Sdis+Q=oTvd;Jl~bo4@5M`=0lS|||qxN0>IGQ_5-H%O~}Vg}A58N8qv+j7<6 z8`}0ogVeowf!MnUNPDy=5~g5u#%B+v1kL1ryTPG4t>P*!FYi+>-tWC&@2q3!uF!v2 zfg!*bb_TKA;}7Ug-lMAhGZ2U*5$UrjN_U|{F*2bUyDpRmu#E|#Aj3bQOu_9JB;6tu zXCabnpYT8^@2*bt@b}g4TBmI`ML#l%_$q*x_YCt< zI^@9ZO9`PC)CKULF%hEAj@*QCbO4vU&DNgndmLVqWb3yX3wte6Xv9HM@z|FM7p%W; zN)+MK<)YA_kxO_$24G1Dtev8f*GR)){Ix)jz~(vL1+m7I=ivuO&LyDZ3jk)*dO-lF z4s`HpIQR=pUEc*AWMW5?=#rzxHfk9XSR#u$zURMUQlqwNfyvO@3)`p> zaRsc=7g9%a(lHOg-eOfm7?kmhh#^!Yjx_3kls7evA!!GsHYeN_k8@On$N6kxWM|L1Og^rg5Gu9Twaz> z;{f>{^E(%r4Vbhn(ff8S>)g%N z!2pOJH#s5e>fLqDBifIo(362=B0lc7qx^E|n&|wm^HeaJN(SA~<-mJIF@|TI5|uNX zVi8Z}2XUEs(O#A9<>`f6dOcg=Do$r@>ECD*SaFUZ=TE6n`-0dmHiwkoJ=(8>nRq^g zQHPwg=)aww;|Re1_c|0J8G}+^En;G}JlQU`j{8>CXn)MnNdlBfR0#D;R1s=ZW@$ zw}3l_pw}6}EFKy5o~L3dd{8AOP`^QWnO4n*Pn>wDj{#FbA6s5Wd-B(k-y^mra$ap@ zip-SL2hhBkVLq)Nb#59p4CV|#w&)2JZ%!{6acWu0nbU63{1;}0FSU|`_j5L)dHzt> zwKN}*?ZBoN&e63koQpI4*X)Twwywe;AaqWg5XwUYxAp?%pg$vWWKv?N({YpG^)aO< zano?sJAkRkrm?!Tpdrq`CA=JtOLiA!1o7VP^zq9DEKc=}YhDlRP6OV=0Xj6N8ui~M zP`fWVb{g8+vSvA%Qz<$VL;w#=Tyq$R9@?wwoH=c20_G>F0)uVtv-fm*Cb|(+F5#RZ zhpsm(_9?GP0`U%3@bg?OuYcRk2DZzBtzc>Y3-@%v=qR-a=>I063=YCEFfC8d7sU$` z6f66%n62l)vzuc1!6JRt)E}KOxa=lBdCX?V%V3w%wUy(kWBk5Xc^RQv|5DGzi4>zH z1^(0V*%6sB^rypuCWu_Y7*6zo_)!ChJJPg0TGj|in2c9!F4#JpJd9u1Z;GS>Y?DDE zYu2FZtNagBKxMOzo?j?m2hPibx<;@WWV73SV7h3}S7PW>Xs}lc$fEBZAK>sL>JoC& zc*?4>tVHf=#1JOF7olEl<1jED_aauJF$zri<>N$=*@_MXW!_c_Qr3^XL>TOFdZ(4mF$$y+AR?Ods2P{+xGyPU zm8XppBSyB9wGy!x5>KR2hwmB#!s+oYFPf7Erg}+Mh(=$7LhQaoBwoAZ_@J6DA-E8a zVlP^NZrcg_8M_VLfkSS{D}1d1rO&s;g0ijjYhn92|tV& zWRN+wgKXO(`dF1P4Mi}PvPV;`FIyMYBX;Klm%-MU%2Fod+ArOGjAa!+`9Hu>=K$?N_4aaNY=?c@ICYmuhf}B-eVbp;ZU{g22hY zk_<1`C%yd)8^t45g^?{JygsN~Uzv#xtJfFQNe`g^0N1HehWkBWTiDVI(5oFvlurCo z4N;B`AoSY?4j)9~S7Gz%g?fa#NtIg>2vZuv=OE7M!`GRGIdk8C%OHJmdjMtRnM8h9 z5|Zj+(=YyQ88{>6w;4*lPO*?eDU)hX*i2c>Wvy!VP#S-j*+lH`@)lp3TnA30T4q8Q zij$3TabH$>n0nAClbd4$gES;M-_rDTopc1e7bv3jnIb9$>MwE#zDt29J5~wnDIfRb zgc6e~m7Mz%iLuR#dPPXgWFk9L#$OwC(3Q|!LlyWtaSmR|L`DcvkT+o$8`guE6qT5| zpkI#UY_%=KrnUQGH!!k_L4RuNx0+*d`=c~uMCNYz8P9ZRSJmds&_jHq7PyJVl--T$ zeVPDnEa%fE4e3}+5T8tGi2E)fU#u>?_jx~6#G73GoKv|W;Q(a|38|Q!YC~=(#CzX( zD-7&e!yVp!YCEA8#WVwe6*-MVMJU2>9?WJ$T1H>US9j-fAX?-w4tb*&*XzIkFY4iL^DQ0+$ykyTLZKe zoW67IorQMz|NqkWc5p&#nX9Z^bJau+uZ1HZKI%`cOc8I=d?affwS-&OKZ-nP#Jz&K zEy?)M@Ionwf`4OLZn;StXh29>sGp#Xuy97VpK!_Bat%qos56LnEqq=gj$)z(fzwlZ zQv%P3g|i`E{GcUKjbdFlHI31ZT9m^S`|>wV$thNT07D8w5U4AIf&w!X!vKMuw*Xm_ z`wJ7)Ln@!O;t8%C<%7R8m;M`taw6k$tofia%{YFJn7#?us(z&vYEv=uEi(8tBf4JO zn*U8cb%BEos1@<|H}zh1FZXc9rMUW3CAiFxq;>jl5yC)t}+eE2V-;AiE`79PAvQ`Y!*XBRa5 zp$AG{Ljfr3lT|S1SRyOQGXIPl5?=nbj!yB*}#D!@U_p;`?Mz=jF4tVvnmQ2 zIr{J!YMkJ{V5Y{!xa>YBRO6Oz?Kz1{;g8n zreeZuiF`V1q>ELx*;=Yk@NzN;0~jI-cw^I^NoQ!wG?j7To`xNZizyDSqnVn{1?Ggm z5)xtSlwjR%ShTE*sJ)$S8ta3V#1g2X>{bqBLC)lmGnss!W9{ya;G;gZa>%kuduV6Z z*`Cx`wFK#B5`#aT2=ZQH!8pA%-8IQ#TszO);!ZMTH$!bl0hF9VX3#TEkJLt_-Sz#L zq0j}D_Z=BGo%=dy3_u7u=xmdzO@r|APE}7tZ|tHnp6jXeDaC7kK2eoO+fc+euZ8;I zv%qdo=UtR@WF*l8%5<6EquuGekuais!$HBXu4s7H*d+xuujb%-V>Dm1LN#=A3z>V9h4?i zZQUM+1YYWW6Ym({1K{Htj(HfXgQ29FA`7U00q61X)$3gbv~32ULtq5CAWRzUJ z&>03u=8mcXRV?d2RX#hGy$3ACfL*00j45<2PGoy#4_+I-`7h`Q>|Nv#RAO@nGc@Zg z;s{eO&q{Cbhz4<<0mul(b(rTh4T#uRaY+5_XX)&N4f!;obNg;1zja`3({^A`CET#+Uo;=dFp_llcs`)UT0-|BikKHEIj3yI zRXy>8D({mUljB^(wRG}ENSoUz<_5GDHn04=QJ$1>OXyQD2Qb~O+Dc!*94uUoZOWk0 zisRqB&KiV!R;JQxTod590I$#rkt7Pnrz2aDy6wT|tkwjPNA-|CIhJQbi8>koPFcp( zO!@dwR~wWBrrN0Cw;2uG?G>RJ0ylLGjaBkO;&u)feMzPp?J=I1qTN>YmlFSW_{mr^ zWqcV3UJ+Tpq{gE@zO&U(vY*+pK$hLGa|ZOtx2wa{pKHU|-(1n7$CS|B68Y&p%6Qev z_fm1LOYKp1Ye8%fmol{QKK5rQXv?y1ek_dYTWrtP@AxKCX+79qxt--nJ$WR8ui7d* zlHuhwq`juEogX?1rVhv;?=uoh0-?pDJpgfPC}^@t_QEIdgja#Tr6LUx;thF@`3f}K zFp`%e*07PB%R_jy;?XXu3`isB?%A*A%(pZv&dXEsX0%%6?Krlp$hA3u=$+DETd>o? zp=+M;*Y>2L2!2KH@e~}`gA9d@^3n-sw024^Pc0!Cz|#L-*RTWsIlsrCGA&llM5}a* zlJf%Z(<$gc3qaHB11--YG=}?&St1E<_dv9FEjEZPG!vcIb4d1!baK5>kFYSx=fn)x z-;8L2_L^kFp~{D`^ubTu8r&kYki&3cP&!4?mTkmyQ^Y639C7f~L)m&gxs@;{{9>7M zu%jqzR4G*L3+0&aAlF-;(izAug<|XdZ&DLnx+|1MWz*UwzNuv9K5Z_*=*LB=Pmog( z8ibylV!9p5|4=~ThEP3ZrL2N`Kln0vr<4^_<(jCKxp%TD`k9%b$1u{cJzRm zg3;?TeujO3#!2auSu;~G{wAJVB&mYOUoe~#=k~oT)xTky?8C2z@)$9|?CW##J#LLIicwEVGRvRvBG{0VZh-tsoAu&7(Ga+hU%Or zwSTQK#(dJO4+sbAgHF739km zfrALU+@0|buTPpEv1hyH-t%)^To1e%+}!t^l%VD^LjdE993E8X7d(XpYvy<+Xj^*ijn6Jauhe zC4Ou6zL#pDE9x-&w~96@T_oFYEx}Or7eN8Jad3X18s$S@jtld>U_N(uRM!8vN2d{A zmFSduN>PAzK-<7{+9ERmpjnm)PX80VEU&A?m?cx{uz!`8qKFXH{1K%C6J@7znt6v| zG1H_I1`nrVIBNQ;bn?s=icxDnrc3c$qmlXYzbpk_wTTp+S2e3Tw9xpDEQz}9%_OJE)+y}-=kTp2Eo3mw7}IVD5&8=Aw|Sn>7!236bQ{(W&jKw`*kPk*=Rewv=Nuw zqOl=-CFl~VlcC-2y*Fboc=Qr#N#g_WVqaHL5tSm0Q9`ir0V_Q*cDVH~IbB6}23(~J z8OoKu1aAz42U*>X35-&g+!X28+zUG zVJq}1FoP4S$JfqOq6N<( zf5wUiZS?=W7gr$+B#>aVRLEepD8kBokQpPO@-MkG?fCrBE(4luX&M;2QhrD7}L~8 z8ErKSSGbp<`5`8lJ-nTCV$A+Sy8YUlzxV5He#ypyq_Rrrp{VyLDmIAObn|#n0i$`C zO06YKN2BpJJVp78bN~{e-lbvoj9D=Zn~HE52?zq)*3)KW}sD?_QQOQ_eTA=o|Vp*KEn7xH7YW++QGFy?O|3(HPc++SWV zRc|%J$ET@@RsfPbqX&VHj~G+}qAQsvBPy)6nG6K85flwfBV>Hk(+?hcJ~eA?}58u)Zv?Oi_XZ&U`Nl|D$s>}L0ABIKQ#4upcp_ebfBal0^yTgW0o(m4^KvH z@u!6C#GZ-L;aYeut^#=>x%BpeP^gHJHB17Ti3;Rc|GO-<2OV~5gAp>1zsF_`hso-! z2X5pweARo{Mk}z>_k$gZ5)x&$^~X8AQpuo&T%+W62Vy*2{=@y@Zhn_bS>Gml6GGU% zoGp}KufT$`*VW18DtsbHcO?qx;N~FNKICL8s{#t;MFm2T%ST+YT|$(f6z1KLh#@ zzwh^|hJs*(R^qf~D>37wJa5#=s>3{1tM6M=4iG?rx&4W#U0~!Sr;zt4z@qfC@6(2c z*!3K`X=f* zauCQtL2IY!&iR&Vn|0R6!o3NwNeUlqjK zB&;9UJ?F1;01n>5B6&-`(l1Kx-LxJg4z^Jo9IJm7uUY!E{Ds-X#pF7gOoa%LT@GyS zAWNH}sa97LH9N;7k}gr;GRh9B-2h93M`nIlaPa9sp5 z`2Vw7*U7o+F`dCop#uu(%T((_$MPj2VN@5hMwpt$_Kg8e>$X|y+hGzkrLUV}5Y{l& zlykGDJDKa6KT$zPgGBnb523jxr6SGfMuc6zvl>=Vg8&+b;-~5g6;%{;?3|bn`L;1^ z65FM$)LWu7viHZ*$NSfK7i~bOQku6Qo|=W|#*zOlU3m{Q43fx>tkYaeorauT;H*jb z@GzhheQ{w`#kTmK0ZH>3wL{YvA|q+KyN;|s!hdQ7GacTq5AdUuaQ|8Sn0|R;PqZS| z?k%*u+}xGD&e68}(Ag2^EuBYm6~Q%S*ez^IpG;~Wv^s|YRR#5Kp#}u@w&(l3J}g}k zKhRl86iiT8?Hf~6Zq06turc%E65tso>6EvgsA^|?TfB^db>qRS=b`@70!LT9rj*k^$z~xgFu6v!eQmy z3ttNX=R3!?pbN4#=hJ3+wyMosQFqe`j)SB?XMr&wa{h!L&-_LaLvk&oXqJ8mq03gG zWd$o#bz@#5kaB)qIM&#~i;!5(ow{AZySmkxgUCFTAg~T9LeB)ilP7HElKke#*1Nkd zV~N{lkV+j%$mPP}h9Lkd9x4`_)hTVBWBPG&xUIDcV>D?>9NQoQsCywopp2apzm3&kz%)0n!*S`du4%f$cX9k@TK z>S^x?F1IWlu+kr(pbHV3^8$6T0K~yv0FUBy(9kjgHKOy!>`$*OV9Yc!70O zoen;XdX812zsgDgJwXQ3i}W(h16Y$Rwy-&ZN!l)p*r)ZU662*aXN6%!3COU@3WTF- zw!I*oP(#hwe}#1a2g&SY#nJhZ_jnMpZoi|JAz@RVDrm=t1{YoIb9oUq3b`a zCOI=`-^mX0nV|2E7f{<7FrNg%JxsLdUekX)RAZ}l#v)6Pfcp`@h_Etk&*8{cWf(8Z z;+&5x5HqwAu7g{CMY-7T9NPZKK1!GSUOgKPFA^)pDqmI@!tv$#O#aAV712LW^W7&m z>PxD*{VB1h(2Sh{4eDy(V>a6H59)k*F#R-k7}L)>?!fS#o%+8?Z)bv$cIW+U{k7)~ z-{M?(+#5up!rw7?&yv#hbTrrbyzzxl6Y9XyAIX_<$4WO5C=n!jtb@&|$7>bF(g`q| z6bAi)pf8Rv%c;_jd~9n-jk{BUP4zw&r8$HspoI=_6xz4@5;--!cAwA`nsh?iO`88) z^|(u{Zl$gF79=-p)vLCssvAZR*H_1&L<8n6?R6(T^MEqnhX~kD8smauZV&5+%6RJ8 zv1s^)@$8X&;4HdLZ}b(X4hz}^2_KEzQ)2#Q5oC_=KBRrM7R+D(yw4$bC9+|M#XA2% z+guU4Z>sX>9A)Wo_ppGx`rh*yt-!3}GmH`NC_l|{jlXamV0Mcs+Fu_0KCzGHi2y-= zWv}~I1J$-pWv$<`x}{4p0P%gAFeFG8@}2S%It_TUrOnafP5vw1qYeK>omlM=h+8r?@_Lp9$UmAq`+x9XhByEgGH$8qw_?lwA_zRYj zsA4^D*a*W?mPQ^K_vlgqvLLdS10%OxW!wC>Mw#Nr9`AdSC;t+NUao)F(>&!wqZAvOlX*ECfmi~*|w2T2NyHU`$7p>JcUu5)1ox@ zhBlc1o~F*Dz=%zRv$m|I@-m|(U;2Qkk!W}x>O=cQ22|i@_WeT!s4NiCPao#IPwLFc z%F0RxvnLx6Bc{GrcE%h`>+~#?bjJ!w&o)%ljO$&7Q6<6UUE!JYG@jPjg`z>MF0(S7 z60l&i@${5-93!YtNGm8-=(bf%^{7%n7LklBqRc-YXv?8Ift8o|eeRUN6Iyc*uR?-nAVjvovgCw+ZIGym6&Q?4FM zO*rOh$`p&F_!1~CJf6-qFtcB7Q=w&SrJGGCynLWA}& zcQWaJT8gRv^`6-ba|!*Z6CuMI;i(_X9Zyo;{AcNI_3Gc;iEv3SP;g;dVZe-8FzR4< zqyG!?;%(EtAB-u`(`W7Bwil`iB3V=WK;DQ1R&S7-aK~))Tkw+yXj~)GQZz1{P+}*N zMlcE3GQz1y6vo!98jg0ir=o2Lv5Gtqz?2L48Ul8(+yMuYe2X5{n8RB0vzLr zb?Rp4!}}PR%*1syC=>#Ac0D^RWrPas5stwo&;2q!Fpd=RuHQT~?S<5=Z71u|^=%qlsr;_FBB&Kf=sqE_WyAR55)Xm; z$9s;l=)6d$^WOYv*hX{(TaJYB?Mr)k(P~TG(*DGCUfFY@M(SKELblbj<*}S8Vab&v zsg|pV(CSF~QeoG|M21TKAEl7| zN%Ly}!4^GTJDJs%k((Fz3Egur_rR{M8Z!k7dg_w+yma419c01CC@lftTKBDWaT`3< zh$Y<>r@wPzKp=#H0*}VH?&SIrEkS$(GW0=J+qjf%& z1Qlv`mM|@NX0_s;wwFmc=YzGGjP6nx#PFUB1htO~pZKQu)j^G;GA^l|>M#U%E2FvB zdydxsP7qB>$xZ@2Ua63bN8CsQUs`g|PD}qp?I19PV-TXldq!6{LyoUckIO{w9)YvV zqDR|^ix;4$1m<4^KQ?na(zpXE#zoG(?mT2pJVG8r2q5k{q~4GxLJ9IV^iyq}m9hs^ z*D`;s?ZwL5!MXc}0!r{S6d2)lZf=%9g;qd#PnlLuD#d+}mN!pd+<=N`FTjtYi**9- z3+Zs)JR2p9-C9yXm6zRJDKzcYTnZ1Cex$b0@wR zK)d~)&#hr$l?RPETDGXWhZBvuUsZcNo0xk;tv#s^Zur!|lG%sEuaX$|Y2S=v7sZm&|o zF+OrO`y6rA zl^K-h*8b(xB(r>s(wLYYzqFsx+67cZaUvGdwlC|}!xxWQkNrR{v0sJ={6rur%wli6sdx!Bwi3c z!M^v>VRpR4M)v|YW0Vqfj(}<&SMBEd_mGpP!hR$B>hDyhG2hD14j zzL&=8FIb%Bd#;}NVmnt-TO3u6TQG~*7{2$iYBevBZK2T7C_mPvDA0iRBr6-%k4+Ww zWuv@_cWn=E#7LNxGtujs)*+@Sr!>m2jFXe0cMCKrJ)@O*#=HrdO`l0slW_trou1E^iH7gV;>wb;OCB+~0$ z^nn=^e|{EDfaXxS9C41B^MG4qiG)m0_#}8{)85xM+rHu~{uZJ@yE6ALTbmUHsFYGi z)v8M#!?VI>OJ)>1SWa}-c#INtR7|}wUc&qZT5rbrzz5P~FD(Ki?y>xrVV%IqP!`U_ zX%V6?ZKjP34FCsV3Scy;-dRR^wq!pmc)kOQnnHTni(l1+L?aOttIZ2o%NKJK%vI1! z`@0Ylb!KjJ<7gX~X&eovI+T$D_7Qo2|8s)d5<#;sVbN-XHD7nF@Z_TGi0I}fV{)Ul zBih{P2D?0%rO!!3W`@q$j@p3$zqK2FZ`X*zW?zR|j+DHk3bdRTi@}3n5X8SkYLMEi@DNZ`d_(C~h>UO*2KlBuW!!l>ENgJ%Vb5XP+5}U9(p?Hz(qnhukbgd@zW~WNe)X z6Uk`79~=USFu_(cr#~yCPO!%=qxVh?T!7aD(bjK79F~^s z%To7yZFjkdQY-$NGf9EsY+0vUF#NTz*Y|uDw$tw?Vp%Wf_KYZYC$fuQQhtyqoVLSg zNr9z>e(4FKru)Yf!=E2p8K2Wy{PvW$+Ryh7$OTAEj*5m+nID<~#wKVl<}CC%Ch51& zIbqpVDZRL>`ya?w4xhfE%OZ>H&yW|=6g_osm(fU`&Z|%lSGa#@d>VbwA1#uAM!il` z!kTZ_%1xpN94~`OL4CqInpomwe>ifutJX*TCPx{}iKy}0w8X@8tf&|e3BFhr_yyC; zGmh99Ysnk<*vpxrhiEd}gCGEn<8TVI?*U(pjoQ^^W1q+=O{-+U>%#g5@>sstX_sHQ z8~khc#-i9^`S2ixnn_0~OdDhE`;ea9+KWkBT_&jc?=>PkYhE#N#7xk?@7FM^lzz$4 zZ#)y_xWl6lN=( zz=$TTmA3te^gt+4kIOyjN-d^k1bI`zr!-z^k!*97^Sq&VM3Xh^n~?LC`>W&-WL+hr5H_?EB2FJk z1X5(-2TPTtZ26(Nse9$s%wmjh{*Vc~gt)SsIhtLUDal|rI5Z)o?Em8#N#`G&_oPt~ zjxB@yhs%T;jLq0m@dfK|Kzf_gh)o;7$Sr5VY&>(R^VT;-zMhpo!Pj@`5tjt&}8KHGAa z#<;?oYC)s&m}UEru-6vgndV9#5`dz??*#xOE z7DMoUcs*WOJv4T4+sbo({?uDrdfC-C|HPe%3z?uUrq+C&9un)(iZ>g z?EDIeEAHLhS1Fylo5k4wYlW34%K;8g`Y%E&(1=p(~80(jb<|xe(jE zOcp<>2>MVg46}08z!I|Ff1%30)$!(Rx+?pp#XhbJB=xljVG-^&#+LtQ&>HrNLqfntZ;#i{aI4E5mb{I1Usn4%3S{1c5Uecjvhu z{i|;43_E$+W5!JC?HMu8cfE0s+;Z{}lx!Q_c-+uC8g4p@ZCOJc*z$DXat>3L>eaOW z3RqvJ5`ZD4Ht6lJni#IuoA{7szPF>v0=cx7e(9Hl<@IueyqK%n&~)(I22C5s3(_?( zlJEF|`B#&5*x~#K&p2RjAW6qumh;Q!#yHV8qdBw!{~R*Y9*D3st_9A-rlHgDY$DNT zRV#>=ol)xcIJQHALZUH-mJ%IQ<%%0(ZjiJr=@f>p^Vg>^(?&4qgt*=9zytd4n(-gd zS=qFdSua_@j4T;zV;QW0Rgr29u;bRzZxI$P4!&-d2EA@kJtp)Y@ykN> z(6i6Ac50Z6E$!c4C@4^O0Q2WIaI26q%(KFLust@uS%KX8?xrNA3BsCa=1_pS1?aeb zN#{TGFvEA;@Ix-^W+wBa@v6#8TId7#@?10@qcxzfZLef|{Z^|jiiaqxec&A!8snRe z8B-rF6(@nIoZROGJr|K`AaX><-ZA1Lp&>spH^nB4M7uph|R8cN`thz`0ZA1l4M1D^TBGmwGI zbZd2g)-&kt3`-rT`a4hUbGAz69n%Kp{)YvCZa)Q9ZEmxLtDN8?I!D1f$8G&0ew`kC zR(J0&iaUv9VunrkGKcAFDRcsZF@wk^POI3(#AA4ulr@KfJ85)Gf4_;qD)GIcK~e#v zK16`KV|R9vNx+h@b!&ML1iK&_*AI@ez_QVia%iBwLP{FP5cnbLs0oMbB`6OOiG;m0 zp;}i(B%B{8?6+-dine>Z6nokB5SGhtvwU}T*oVUl7E{IR%u^}u<0Gg5R zYn&vfB!Z9K*T(H8By}A~+tv)PcUYvftLY`)>3?w(Ordgsx(4P?Jv5^FJ7Wb#XkR(F z6?6}JEY2yVO_Kb}*Y8_Iv1!Nhl^2rZlHs~3ZaSAEJr};N3b`p=SiUlgRzC+jj+N=W zNbxl=|5e!lUlE_S<}bsaQRPiiThP4j=bwb};V;na(M8jME*=tOsgT+_Y7|$H?v@Vw zcN*WOGpYCYa!>&;_ziHK&_PsN{PU7`TlXSa$ck9WJSFI zExE`~2(5p**;)P4*=IoPY!QGqpGx0$ROaDE3F3&TOJ+R8Yw902D{9BqphGtmW`RI@ zRsGSniU@$(|9arf{-BE4>A<;+ES7!FDRfbpNy?j<`pI2i_qgEbXf8Z4O=8_-C=3W8 zK=18oQJMvp(p}~%e@`l$&+Ur@o$447!picliSw9a{R0J13`(v;IbRh-%yMUNL)ISI zoxX!;q^&#G0W3|1tJ7(1rFPjs-ncuBo>+@XgCe0^>ch?jpr@FwG_HtFEefj*ZHCRW z=M#8I4-&^`FH|PLbL!E?{FN=Jl@#5LzWr%vK})Yz{?y7Mzr2kS7_`@QCXvI!uBU)H z?&>Y=M14Y6k|2lIixdrB{i^ImS$RyjH_uL=f2MD5lPid>b!n_V2j^w+cs<65+ke}^ z-X-Y;#s%c_iHOCl-}3-2OVJ+^Gyfm_tl-JYTP)1m>1}iz77qzQWK|S0M6{Os)!x!y z!()q-7laPwmkZlr95h4IK09hjhf(tM<%MR!Am(HiJvo2f3O4?42oPOrcL7q#e+F@% z0-0xYyZP@i#39P5)49g@3-64c!YP!_oTLi~A9Ojx_R-XP9+e0K&|8$wJ(MwZ-plno z%Ttij)80{Lm`Azt@HXC}CLq|Pd>vNf8+9DG-XU7!YMo-WwQZpxGh+Z8x**dPC`NiT z&P^tKbA7ctqXn%o#&1WPYMrYPr^WglZKTn)Bwc#L@~pH~1>=5LpZT|{?x~**2uICr z2?&81wqG-TWL-bXaV;w!F?8r&K3@@l+a8icXmahbxT1+k6W7s&KjErMJ@C8kH_Dk8 zDvzauUj%>M5$X%}YKTBoZ_DN>ASJ>%1IeOtXzisrEvsS2pRQ@$1=N7{^`Ng?ZgmZJ z&ezQdVjSxGDT*Q(mf|zXEq&0iT`rQ|kmtSi?x0@s>OdzQ$-OeRF(Ite*avN_$AIi; zQECkE7z%mm=~{Wnu6A2v=)M3p!rwrdp_LFhLjYOx76$FI>X!A&Znv?V8pH%S4Yaa1 zliproZt*-fEGZv8HYz_&w#lWS7t9J5>;mk;7oW5#J|;#!HpWj-6;^jZNq$5U(4~Pe zvG9M7yO>_l-G6z`S>Rk5KktKv?qGo7YcxoHKXodlW{1sm1wi*PS#mC&R9*d+%kl(< z$_|(*d10`q6Jd3B77?t;KZLkE#Yde}&7N=3Ns9kwApBx=`UbganX(kv=W_S!*WL<-Oxr3&{03|bRKCQQV^vuIE zzmwAPqGK^#bNvFDzcHx*SYK$srr1RdrIG~9!YpGwrOq(*g9DlTxCvP)!hE>{u9H+& ze$SI|O;3$F&7Ol!JL&{=pQI14Az_5uyfo*}Ue={lj$)Bkvg6?-S)+J74U zChj}78LsiTTootnqqSi6roA{ds$olDGk2Pq*T2Mf28cV2kWJ90YZ?PRPwLRJbmkBT z7bN9h&me739&+Pci!EETL6_DJ(gm^0V;{VhBK%NPYAp9Nk| z5xZ%)siji>)NFea6R%R2R5RE@X5vwcWQt4dI^kCQb1Wb~ou|3lDGENzRpcsj|Mugv zVJLK3m^E<19rL-%oZ#<&(jV`T3Q5OX0 zmN=35i#ydo?>PpF2!17|p47)uJQ&&>3}q$HujOWDtn^8{zZMod1nyF3>z)L*h@{md zz~*N!r)FlsqCD4ui-d`q8Qa%4hm%6Rp1)L&S-Kt5wru;$X)z1cl8R<)F-(0@#6=8eBedS0ODsQDs5L1W(f$2Ft`v&}03>i=NLsPYy2 z17sc{>OFu^6mG6n9;ZQV_(QQJQ}?jApldky%T`cLTEYM{YIPw^6$Oce-=`ERhX4eS z`h`Nqmo2!`%9?av!Po|b4jHfL<(F49*c7|`swyRXl7!#amPx2|n1O^06g^bov<_x@ zmw6j=4kFDyX0Br4a^%rUMiQ2#Von=;-sCeg%c^-t1X5;tv0UTnweuuhte;_q%ln}# zE#}u8{q@A&-UxurJa#hWwWNrN1^)70bp?TKDj9f{(;8Ryr?8a9$$Z6(y=Gh2PRM~K z#0f__U3xmX6-`d8+=|~r-}1!{+!O(ntqYZ8@qQ_g+H@ilBa+Su6nvYa{~gSAC2_#S zVK|Yw%_@Dx5q`s_Ub`dKQZ39gqPABZ^iT_J3IINk^-H@GM{D{GE>=m?#>(4*UyL=l z>fwXhxxAB3Z?*l*h$QE5GgvgQokRul%((bYIy=uqypQ0wC!{d=P0X=;%B=NCy%dbj zw7}GGq?`Wo`38&!Qhi_K59^^v01=O~-GG7BBN)ez9@-hn10JLHI`U zI#x7Dws&eS!o15f9Yt)ZnDXF5G%GLiuPVtM+&(K(kATiJbj``lyUJL$8g$IA3dLgJ z_K8WyYJRa82#MCaWf=ny=o^g(6}uGWZshK8b}&MFnR3|`kLHF=n$sAhO}-w>ucWrUy2cdidU0|f%tFD?ZLWuEQB1$?Q=-pI zVf^Ay-TCc^eOD4rkux(Bg6#)1V(pG#h}&sgC+;oMU7>C-+; zSjBywMxMhnwS@d}yZZbk^0_vY5ntakO&S4OfE;sd`>aAY>q`1O-`Hp}%aAl|P+qgu zr7j+!oaU}}C9=V(lbdthlZ-$-b@zYRtrYu2y_z^*>$)#(g~u2*MK zL2+xGIRtD1w=fNvhDxcEm5}!oinW~P&qUvw0zWK{Ht9go-7iE{sF8lIfiBTQB620v zw-wQJzW|Zku=iF7(rbFNY;iX=dWvzqP@VTr z#WQ*e^yZcV0_|egc@>v^vM8(#<#z%iCVpoM>%WP`1$vluCFt9f#*df69{$+;GQf6x6Jy8LZ zb+4h3Zy2`gY!(&dCJkwAuErnqs{6E3T4WR>EtY#0d7?j+b34-oR4^?V5YDl^K` zWxM*+tc;u3POGRpqFLjEifaa`@~fTWsXG0s8^Ady*txcd7pV#&mHHhdfcMZ_kSWGk zG9mEc2ef?Y3?AHo{|j>9?ViNBqUj}nbvbB&=|Gy)Q02zIO9JYXloV= zDW3%n1^lbfanQEeCz;h8GLOYr&~w|`0L1>8dw2inrN6?x!Cjiqvus%ngosRRUNft* z3gvp7sZN^G!06THv7mm_8{V8N%|4WEjax6Sh@fz}IS`e~8&keTC0t8x{m`P}%Pk|& z$h+gS(vn+t6C{~i>lfT8P&pfQgnL&QYVOUEb$2q+c8Qi|lCJW`2H+85HKxCIZt6SQ z<}x!%$c75`dkBj8^tZt|cTfG(30bWx&y$PktO;IS3R~8+?}k77}}0GKBM- zc|?LcctSf*xkFcuWEeAJ*i*s?&7;&lGp7lv{%1kHc<)fi7hrk5x7~9PEnjnBPJQmD zht7cPI7I5Y@r*f$1rh-vt+&D@yldJ3XD?!z(cMqLK#CvG!@r8jmU|;W^koYRES4Y%Qw*bJ-8!0eH3|Dh}O!dx8 zyuhZ}Pp2S*-FFrypTyHY_@|OYyPZ`Q`kbuwj;q}--pC#C2a$s;$%1LE%Ml$WG8Y2b zj1sgUnoJ0Z|e=Ip%#;+B_TInRt7plDLa*0Ef)kEhkDrZTl%_@y zrAe9J8IZ&|^21*tLa5$iY*Z_^fs3S6ArvU%(DI$uQ7qB=F%V{EqB z6_3mu2gt_#S)fAJ=*&*`iCo~LJ=8t+$Gk3AQX>_E6fp-40zfsPF|H~?PaVlqP}CHT z9m@JoYzjHz#bR@7j8B1)P9H`ESp}pqg%ty+t-^d*_^j& zj9>om>5K$0oiZdF?*-<^K~39*VtMy%@62^GN9?l|uMXd^hzp|kr&p_B1aNBu>Cng! z^qz&Gg)BMWUdKf;ahB;O2um-^jT$Xbiu9L+bKS8Q5zpmDvo7ZjN34P3xTs4)$+5NM z=2R6dd(Hq3k?U+(+Gm>_9>mQH-6h45adx-43K|{6T7R%N>ep7Msh!lM!t)DWGM)&i z8WHUQg`qt&%SLgT<`gP@DdER)1zdnGYB+$K-+W#yP~t*Jn}xYO0c1^aQr7djP3a%A zg|{0D(ugO|ByW6OyH#iY2UtE7K93traWGZ&8nxh85DC}`R;odNdckZfAK^!k({e{d z6TBwPY$;BBXJ_yG5Wn@Zr^)(9vul~7P??$MaPXgCC2=|VUe-G|irO9_b4Ha?=~9xb z;g5TI?8k=S`7KD8ZxZGcX0K!kWa3LMpQp#`%ER8 zj6W;hBi3mi`2AAPVFtA!Sw?^<9$^&JGQ?2LzsC)@t5UP`w!M%zxsD?`0n$pUZ(_XY z<4pzui1Mmti9{LhV|J~Uh0mhH+7pKFlvv{vQd3XxAmH=)=lTNrD)1#g0=O=9dCZ`J z`e%Yp+=fCNywi*>g}_}u4QfhWX{J@jxTovIRVhWqmsYu-H^+0Xj#n4fGGRbb|R7+Y%vQ2>F5Y4Cx59Mta(uqhA^hBu>AmZuGMh}*bc1XfP1 zXjIEJW3|Xa)@edYzOq8-_(c-9Do(8G^<$coLs-0_6*t=Fg${4+FZQD1Ht~`;qrL_Y zuPScamXm#h{Y1(dt>w1u$#VX5h}(A{>)ZS5yuZmSyYrJaH0IV~2ybH?){D&aTnsD2 z$&%J3!R@MR&xp8>*!iOBxsi>i04OH@AkXr#gFl1prS?QZa)OM`U~6x4;XibEYc6n= zga-Y}emuIJ-O9rQ6KmO+@Gmq`NvA60V9_f(z*gYOoWm01sc?ofe~U-)cx4X;48PDw zyxGUsG};&3%7P8SfGH*4wAygdxHMpdt_wUl&!qsZvvr-sf&ZTI3i-UMURo%okz+^< zL)X@iwREsSLhl#1MnI30_#*nD$;(U~D7s_6-VbT8h+_BA5!=UO3u9dkJ?k7@KQ?1| zTcHB^iIb<63akh*y3ysfWvsr*HRPi-F;V9jBCZI;g<2xVG&{g;?zgySQzC=omLD!B zIpwCbUky?jda3_bu2pl>qU+ZlB3`)nJxNpgho*ZI^}{iRV7G96Cff_-*H9+3m_yjY z4}Nh7`K;anNQ0tuxmCz!?4KI z#|^&gYQ&sGjG^6|lcIJ_UnI%!kp37aK+n^QE;3#9UprL}gMe?D5TNdqI)@&}#o&fvGK z=k8+8AauGk&5SF1b|C&@QOO#2(s8h=KD*eD4@f_T?SCefiVIZXXo@GD?i_2%#qX(f z>KPNCJ`5RI({rK7GZTLdM)ZewAwaGY@M#LI&$<#oiwbPX3?^4cLi4~F;kDs|MIgr2 zIHF?^07gK$zrF{cG3-_REIzPHsyqT09FL>f5j;*lkvli**nCw_dJCSJ35OMp zUGF})V@qzb`t5863kG7c-wE1{X{`o8h7iL$ARjdXvdI}1okUXwND%W9>OH#iL0yCd zUqhE5Sy)<2R+F~xr9b52+4Q4{?S0Z}vi}rf-?_jKCZKq3Sz1=rRL7R;&rb)2f{gn- zqh;I(K>gy5Rlf+?*;GUbg=g$Jjtqx0iVPK?LCoKlM!@_aIUz8Gu+A!g905KqzgEfR zDBiKg=*TT#Txz|xLwVtS)zPaR{dG*O1^j9v@*(Xa;|Ctc4rOQJV-{RkX+_RaFc}H< z5Wlb?3!r6X=o`_bo#CxgblaDQ zGh_EO2O-oTJZGNt7ud&p47RKEhPI98i&p)vf3>Neid;!!4D$BEupcrdIIu0a(P%+l z6VzKQP>Fz;H$e-s2*DxISsxULQ(vljtNlIdP0hJ8b_Bk6L@pp!C~@_K{aj@xK&i7*F6 z1?};7gf>q^ICfudT%}2FUCVVo_`eZw*tdJ?B{b5K!l_A%YdooKrNajwvF_S;C zhTJ91iu<2%VizeA>f`Nw&4l*hitXeziZ9Mj4Kc&b21CC8BsjzCbBA|O({cOt+FfYP zOkPA^lPqHwZn(*FV1+i4uzgx5tl<2k{yyLrIk z87wKJmNk@sgG8!h4DaiZZL^*6{;40rQq*`aas*~PwOU{_Z)~B;cBmUvPJ!Sm?{UP5NsYv250pZ*?N$54S)%T8+?26(3>$A>O}{MM_Qc+=C!NXO6=Cbb?Uka zMy_cDB4F06`r*Kyrg1LITs#0p?@RL^PEOom;&MGP(-Y2^{ClUg}u0l(QnekhC_o! z1+eBCp)#+kWGITo3gzdXQLYk0J{A&__=9AtrWw&i3a+=(Qc8zdoa`KGUk5{u-CI-a zPcDTOFQc;&G4kdFR^r^1n$t_}C0_&!+j=(sJZog7C41UR{-UbjxG509HPS+)S~Vy< z-Sfb>6QbmhAue&j)a@s0Ya5*SzrHT{y7A?u z1EImX8N>Y*5T1a14;rKzfrajEq6C>U1{`-@(etU*i+NqMe-?3)&_v1S)K{ccGR9c* z)_=}L zh;ezh@`L-UAW*L^lbL+^{{Q>w?9@V7&`{CrMA% zlLoJMW6N86PM;ClW5Asr#iKfZZV=Zq+&%c#wN%HvB3sZJJ>pgCCgk;+t8D> z@V{^|Yh2m&dqC5{Y>wKwnom%4vp(o-XegT$;{GW805Flyn--G8?6Hy0kbc!>gE>5c zc7Z_rmX$FM@;` z{%dijx1hL|p`&-78RT_Im$h$L+@BkVU1garSsMb}lS0vqEzG?!D3rll)5?_y^vo4J zZ9Pn@dPN$;O{XZPf(GWkzW~jp;a{%U@}cFL4M1NZpe(|ckBHV~kEFqqx0rX`b^MS&x|=asTciE+i(evQhj zi!rv=nWWUfunnGw8I-9kHa0P_-4S}3Gk?#uU*!08tt)1(Qm zn;Y+@(Z5DG@UFybV4fW(_uQB%Y(^X}6W?@F9h*j8kt^d?jcYAB)n9xh>4dkC5F5u- z91ICxX*cFU=zwp}u2XO!O!E*%FJs zh#efRI{7zrjXTamju3Oq!1OvyTpY()N%qAksAr+Te>Som_otM-mo|hmB9W@0Ao~2< zEbK^7J#6doX+BcbZC>c%Y$8A1Fx0+}4`9sc26iJKwz!WBCZ@Ge2YV=iarHJpgI^ja zT1a}~AaIb|$5M8oPz|Nb%yD$L7q=SB$(9sh7wn>DhDR0)y>DBhU*%m-*ZLI2DIdvw z&*IdGY|L%jcX$};dt0t5S%Uz%Y{QsFzN5W8W%mP8re zJFBtS?e@lU5ABQ!ED zW*D%owJsRFyAxK4fX7TURLHszNxV5?`tZqh-bqqv{8x-nB;%K7_C`92NSVBU{NI|W zzO{iMzpE34RKW-?94uzaH2VO#gNw5sOI|~4^ByAb!f_`!S0xT51^XU$AfEmt3Uw(?R<7S$^)R(T(@4qmU^CvYK%Hqr+K3gH)sQXyP zc-S>o70#kA?TMBk3Zj$xuwkk^Rts6duC~;fHJS+ddcJKg!}wFtxzJutVT?$*3F`3x zMv;JcAlY)x#cx{otDy(uG^)!RlAF1E5y64bc}awp!<%B#!tfi7+fC-086Qf~CeT(2 z;eyb7jlY{(V(J+(sMEz%fNQDQ$*|8RQUG1KO2EIkNW4!|+ztL8(hUR2zw5QY|ciwLsJ#NY>H<_od0L*h}aPNsa_;IxT% zd){qYDHQ!vIz%&B*B_E!ZOKTlrN!9sya>1*$)ATSBOO9qzrp0~_M3-1At$O^V`7c? z0M(!3vxk~C=lrgQOJZopO}6aJ@24)WUo2&szs9gM0n#z3!&bw8x2qmt>{V$dfG4CdEJV-v{Vt%e&yZu3b{SYeJ!G9+BFk1GpVDI zxga%uL@mcg+6kQSQXHLN9^WyU`ZK(G!SygP;C+*sGBC6>?ySzYEuOEkDGBj>kW14$ zc5e^+F7j{e0@xCtHgPjF#k)_1ywae+WvoLjOas+BF{$Ko{nLW5z|$j}ebZ9?(f&I) zHTVE}p4R~Wl=ST`)=|%@DnvJapcfykym!MnKct#Y&fEN&lYrH(WXrzMYPC&_9l@!c zdw;y_q=0opkZCBkE@h0Q1CxQ4jYH585wA_+6F2QaiH|5&M1t|<7A10OE)b^oB`1e_ zzbgi;>-LpFyZ+k}3O{F6i!FmfDUiy1K*&eCXB#=R5obT=*onXNhA1p#adcghNFd0_ z^5nTzC$(R^DLyH}!$x#VksU8gd&)Z=0cWKMBlBYLycFO>Cv>$!sYc(Iex)b)OZNzB zeQ_$-*WSgRmGA89NXPwfv979s)Y>GdREF#^GAwZDkUS zj#Fe+Gk9WX2~N`K{L96mv_swsBUVg$)o6c>u>(_9WTrAz_Mnk+ZTsv#J+6r1N?sCe zfDD8*mI)9HJP2nmU9KYw`&^vc0+N*kmlBGOH#I6=%54R);NBC#+-k0x$LP?54o-1k zGF7TEiAKz{D!5A%8Fzn0Ug4yMR|x-$_ZT^eR%3{ux5I@oY1S;W@lTsxvgix5XS~(H z#s0Q3TwzKoT=TAYL&pNkw(8Hso=1qj8t?-fmgm zograIu4bm@o8$hxhwNeP#6g{h7q_Nnq>Gd?ir1{D6?dn5MmK<3+3&XH=8+E(S=E;F z+zX^ue#VX)3Jh*dD^d%6U5k=F_cr>+LcWBDYTdKN0MJC93R#NWH47|WpMIzw$=~Z4=aE#h32GL*QGix1Kz}xVcj`cwrkZyr^O}MfaqZ@A~&jo zu@BVW(SbELaq0}aUv~FothDc0n+v=^jKYT&i@^031fL5nO3MKNY4bKV=)fwTjA69v zxsTsAKmZ(YDNg+c&*}UsXJ0KpEY%&X3k;2a=Dx^5f~vHwS0EhWG3QSlbhg-D!TeDK zf@$=_LwIM-&%jyi%J^j|I-vu&+Zc~fAqO8GERPMK&UR}Ywj{X^n7_*U{^9Lf$gfxWKB3OCdLmEmXa@2pf#X=s;fo#zs~2zY*j!wrANsv+tRR~-G-d*A zrUTEy-plJ?;r=qt`o1#Ec%XH*E{(NJ2Y}%fh&Y3*)hp7l=f6#m9z1;2zoQS1pQP9o zYQpo0;qYyy>Rp_!JO-yJvOkQr**);f|9fgUQ6gxqVQ2?nZ(a4;U{lUR6ITxan>xE- z`J9j}1wVgTjF^TsLS6PCNZv!dsl;{44e){7HQBUN@adxWGM9FyJw!9w7o^d?z|9VA zg@LLOr+_xP&d`Li%yI`ASsC?nR0{{n>K@uka6*p=;%dUN!rfOAE|~{ls;3GpUE8=T z7eo1A=*{cQXfmf_Fp6R#V`;98SUT^jWy4Ob)M2PiBW8mQF?x+(1;rFV zX9|_&*A+vJw^bZZOGa13a8A_+a>PqT{@pxvGT#kl&0p`b=6)i7m|8>N8lcDj)vxP< zgwMsgel;WnZkG-#-M14ti1ydzP6r0!_uqs-A+NK46Dx7S&YGIU(a$*(P6((xR|Ja= zoNn=>;;rjCY-2$~wTeCNGWGw{dXQ))KCSNOmW2@VXRe-JDfdy$&Xcn#VFKR$# zYvQO@hd6{Y*&Y76PDAW3i#?(UpK=`E)K3ikU3C)DP3#zRg3dUoug|70aEeiRC_Gby zDa^G-grOdVSOjUbe_9KjB|z-tkwhDfp#U~JkddiFr4k*Ln6Wj}a)v{_x@-WHexR4I zmOu=x+;~e9cgw80y8!Q;vmWd=XKG}cY3dRi7ztmVw-7B`Y5^F%d7vBV=7DjuwmxNu zJ9#>V3=Wg*!;_AQmR?n8gd5a|+lATQDd z>7VB=``Y52R%gTroR~mY)JdjIJ28Em1clk5RIN&lk;(ppmLVW3uA%ySg=km{41Ts2 zT^sKOqmzrG9`#Xc?BvF3k-_z~`#GQ29CvWflW4}&9BcR#M~ zQ~1;z%sp{-nV!2@%@qS&9>$^_Ro)ZDB_DTYgb1F4$G0rU0WU#$2*{OZ{b->xC)gcV z$+m*rkg&XEB!+Lat-znmHUTnQz8Z_254(y`q39kT``J-%$Xe?j>m!vk9M;wGnK@C< zCSuqNZx3GAhr-fp_#xz+Wz95$j+e_Y3+bMTg8~?;G*d1r3rO!H9R*+y{Ys}7-V0PM zjIp7oHmBhecUOsi8qp|@L73%4-t?=ACSOX!H?X@9n}CNUnBt$35YI!3M5ke)jrw)4 zR6DElFx(vFE$rXviRkrDE}MCbt8LEJDDm*c!pIVAV?E)aokt1bWCKP8iDl>zi7LtT zhnW>B;x*xSTmS>a2q*Hdd$_nfTvdV)eQ~ z^NwH^ip{BVZN11pa!=~3n}3nc5%bnW>jXkK)&hhMTBE7lj-8@?Cy~G6_9k#Qr2|s@ zax)y~`_jY?N@u%Ys|V15$fWah-O~xxn0t)8DqP}K1_TkH?TCyD%UW(h#X=WURx@IbJTyvq@N(a zABqI&VpC$8z`n-zXf! z!f#z^FkJ$;b$R>+4R;IkGAN#zH!u+&bUj)_8)sT={1kwDQhxpr_TarZ*MxVY-t7|= z?(-qA>*I^L%@;&}o2i>}`Oq^vGo90ZRVfJ44I@iLYu=k~H(k)Xn1MB^30i+xjCmGb zLaUXOtSw(h?V3#6OGya=K)-bE2JPnJCYVt&uBzjmlj&U5xBQab%$eWrP)~RAw zIZj;?+}hL%mSj7kNgI5Pg|ekh7|qCHhn?IXqJn8Jie$$@^m0h4mzD|-)fd*=0w?~( zgpie`wy-lY(JQRRsgO?>9KTDo0Uf~jV`JEL4{%4&1P_f*EVr5+hF}L_IToL}rLv}0}%=w(o)w8(?D_8;g{j~70EsDF+P{DRc zufW|Cw!UquFr{Ff4#Euqv2A!lQ_}U&v#}TXIT+*APDl73+(vKvtvhQ}^AHDHKz(5_ zzDb_#pqJ}j5APlj*uHrS+l6@EH|l%3N!|5@Q9HBZiy+!RKOcf*On~w7&!}~ldc*MJ z*9MxdV%~cVdJ@oJg;>J9*2A!z9|;?P0)cma>fGKmdej+ zy-+An6ZH`tHzg2jYp-j|U9 zFGcdBY55}bfGqWqvXXFdRI{p)7zE+s;TS%lv7%vv-I~9No9cwMrpqlK8qf{Hhki4P z-oI{xnZd3`gf#BLJ~FgcrH6pc__W&czCW+hEO*2k>rir-0C^I^!+SODiO%j$auph^ zf|4sdp1vcK5tS9^;w;3f9c-wEo)5#N$IRMENh@3DCi-r-9*hq=XiGa1v^&M|_)Bf- z+ig5i7vqe!$Y762bN*YpxOlQ^DXOEF6C5m6zm<&~s=YNYE_@@RDb^~)ZuCo>FjP+j%4FMVwgBvwQq%|JO3`w|HIw1?-Wx z;#CJ4#{WQPm+UzOv2LT@URxN}iSf}pr!7MO35 zFc(mK8RN)XlDW1|LnopX+8rl=o3_a@clt~>p~=E>{bg7U1~W7J=wEPX7vI5!0pBtj z)9LPJ{>Vm`i8*Lzx%IH{DNQ@89Ac3@a6i*R&t_Jzd`vwYoA5yhTvmXrQciBOQc%&b zR!i2`Q>U%nS@(*8yfBu6ZmfM!G@~IRt;ti+%cUmPQQWTYpk!B2z57ZuiM9SrbkJOi zZo_tPA%vUTGWB4<9~m~d{s~} z2Sy~C;t4ft;W77FC}!@%q8#P`_mr1qOZ)KiemQRZeFt#p1m;jEZ*2k0)xFf>iX%(S ztx8ShBr}l@&lQmGkCSbvEbmUrQ;Q$t-!jC(&%;H&dmNId zRjxfNBi^DBN4CcAaggB^4JHD_45F^zvt6v_dt{g#BRKA|(-kP=n#SIhWR4*XIm0>k ziu}OZy5q0yLN>fVRv||4-Mu<-6wI#pOR2}Is;h?boEI$|kW8)HaGgKYlO}|lgw6zm zJzDV)DA;(D_U*ud05o2-99>(gp`W|e16&Zz0;yhH$*Ys4IJm>Xy)Gl2;8!|>*}|yy zafv_%Cs3l9N%ucgXbtaHig{qloYx);JStYu(fdbOtT7=TP@b7^iJEb7LB=gVlm2Q+ zGCFzDZ(=gVn7*K2%z*j~-J#AXw~tXLAntY;`p3cbb8By5gf39J)jq(SMy`-;S@o00n-?7vt&qaX3>6(}Rd zBESh5{7Z!jjUw6 z(A17se2D8G!^?4>XV>mJhWpN0Q8slTE4qNhw@N6Z@ZM-0C^jmch~q-6r#7826KgJQ z;q$N2Pf88yqByt{4U|dTpKX3#y1IT%4{2|G6zn$ZMkB`}@slK&mS!Fo)0n?_H!7n! z%ck~M!2u3uCco$}eESnrGliDQ&3&m4E~whSO_q&1AD2Zw8C|e=`b};LPs0&52%hT` z1SYU58=?v8S2kZB1q}0@IQ#52aH)fKE;sw_zP{_*H_#z=McqZ9Za%Wz9}tFzJ8Hhc zn^>>hw%yIEYcqSB%0Iga9WdpL4<0IFgHQ!<_}*Rkg7|CM???Phcqj?nTR+SyhcY4cm1vd)%noDuK+UN{Jq5_=Dn8|oGK(8dNOtb>qHBic>b0(d452w znP)OCNRxprti=6Ba7^T4kvbi~F73CTKU-g?b>otJ_`T&P-+-(rw36gRZgAWW;193Z zSI)Y9e14m5I=Kv{VTn$bzwR5bxcW>jZq33r=zJPxe(pVo!Y_)Tmk5jp?ovatv-8{LBre-jCej5e=b5%u>w9Zp#$dwb-2OQZ-9RG? zCp%XSyg!LvF_=rDNU06dpRQh|pRN)QpgC|IS+wM<(kI0EC#Q*`=O|x2CqBVOaus$; z|4~AK&a0AC*PUWW+rwTZw2hK!76ai@+qA_YMG9?Y;s+=8P1PZ@K<%u5lBU1{RYMNG zs)x!XkIsDO2jTb&vhbKq3mc}|uV&!yu6VTrUnA-vgbTcGfQQGQj}2Vf;_=%Q)=SfEc%}F&;dbE%3p3(2vIQ>FX z5eAt^S{yb-rqerlrdw(8eN7(aGgQ+>IHy6MVK2(;Z6g`+1chTFrO=6b`h4=cH;dM-*2dXb*oJ*!tIy?y&W0na z9n`wFN?$k->87~XmfYSt8Hf?-3r=#azyaVu>=dQ0jp1ScfQww~nauB*-ctwjS_G43 zVikr}<^yQHWG(K9G)?LsjF2e6IL*Zs2WX@({5cx5B8E;yaHF}p?Q`XV16=?iu@KQ3 z6X$|QnVX1;y&!UN%(o=7MBe-P2TOd!|9TNqSH9S!fib|U`Dhta z^V%U!PgoBg{VKC5S8XXjmhuZeOB;X!1=5|#HGq?3vusl7GE8eS0M(F5ZiRdD88swvy)HrlzvuziD8fO z0(V}s#;gs%oMxc^)gg8`y4Z`Ah6UsSTMagTHE!XIbr*`#*`puNv*YEP*G~TmBMqA;mYRsx+#~&c4!x${mNIsPxw>CX=wZjiS>ghy=n&)Q1&RDNnD4mW1 z0vYHS8{?FRrGfVyFw!XOUdUc%h&;AmH}UqCD6Z7o7V(k-)Kd4YpuTg9`y$;5dB1r zpw67o)?v{1;dzycN8@pFOmaJ`Gz!}lHN3XD{p^V$;kXV+7vPNfvt5x&b0oA_6_<3z z{zWv9>NF;Yl739A_sheCs#OE9GC=|;`6RQ}+>b`pgGVQ{9^uGmHo1IbIHaf8tHy+3 zbxaD0+EQj5H1B-c-|HNEALa&3Mx4~jt)Wk-&0}|stnH2Y0u3e=UNNm*bT4qE75Grt6WAODLikKYR0HMThjqY)5OCS&9m1Cy8<6lM0ExC!~Vi1BiPVXAwh9XB;6eT~2C$Gy7 zg*1!EdraW=-a(-q5(sDFNVS&McF$`<;d%=cTHEjmr+S>X%kh6%XdU4%VqKXV;$AI; z?$?B> zPv8uv-glN&)VlqHd80%C_D zgNXh!sYY!UA$j&Y0;&l;qUrS@-`KSy_c$(s+ferQ@HsbFCZH@}g5EFs6LUT%-hz1R z1cDR7rM&v(PKySUo$&WFsA2!~)bfutR|$7BX!I(?ct41tlu|_wce)k55U!gOpp|3c zT8g*^jD$RG*>f9qGW-T_JqW)A=LG7TU+uqQzwfbB{*Ed+M!``ep@@O5cwZ=30-kQ$ z7OCvCTbjF?8fHjJI51fm?YN~Z7IYe3ikb3m=^K^jKe0)CB)25&(LVjeVehfgPP34P>$8n_qCY!>Vm>fd zwTd)ny^qR02o0@tQQBx&b@IfcselEbR(5}V1bSszz|B}2L=B}YDdH$0$fK;tX!=>R z&jngvLOttPSIB+*XODP4qt1?q2P(1vF^tOylmh>hlKz167gsIL6A4hJKJio(|7635 z9myIaC*j;T%h3+L|7jylkM@|?ho}^)lW+E@Drf`}V-n?)E=O&=N)2k8c0043K=US< z(UkY$?gaxV`h{mRw1SfT^+x*Ub`aTc8PTdUXH0hC8fE-EX);pru}#^2_z;BC{6@_e z9B@aPVWWa%8k?~E<7}0bKlL$#>c01hSqjU!!MHAwK$h$046ed&wC@w<9wXe2tekD) z$NZ!?i|&ZMyM+mAXCE$Uc59qA%)&&YQ*^O?q=W|Kq#{!NK$Z z^iv}br0UV0lp^nOM`2^%3p7?#}3uJE%Ri^<0$@s-Bq>{lja>Vt(3v(&$3n-A2qs4DQOLoV$^_t3c z#!F7yt~Vy~3NN9Ir_u@6neMmVZaLMbu){ijXG<*5r$!As)F#$yvnTAtv181CMEuE-0*$s|j!JcEun1VK(he)e*0BBz_#;L?hvNDvLhb=`h0E%S> z!JzkQh4=^A+qB~S-njFgads=77h_AbJ(+w?I&&{K)oSf?wk%3PQn2lQbh#!7dd@p8 zv&xGF*ETdu-%b~C%w!S=BgC3Yg-lI~Q}WycR$PLX-Dcf1ykPxj;58cE`5T#@$QY0d zq+~^9nki>wSOrMk6bYZB4G#=nXO#-4&<9Hot2N{%tSaXNGRqI%eg397^miXcslez6 z-dH<0_G;9F2fc@2$PS$ZlP+xd%scb_=8(bx2yN<;zaj9G*|LiFU*O`G2Wj+0+pJnW9j6BO#of7MLPf8u1$99a!&JdbPP}`-c#)|EMD}GkN!u#I(DF&oR z@M4UKGwq08xy0ACr8=ySHWIk~Uf4^AV>PN`_YXJMs86XCUCm1P`h*`;aHNBhI#wju zLK!&eyKe_W4@yeEt=MFiykX0T#KG5j?fW!ksmUOpCYqC=YuuGXLju1mScsG?CueBw zL+5kso7Wsrf55X0NnX)Oy>N!A>7Q+PE{IqT}fwFHP$Z>0s$dF(Fnl;yvAl_dH7-+Jp9|feo> zcQy2%ikI&YI!HpySg%5BYVvbNWrI{NuBkiJn zna~X>9(~U*dRSLCh&ray19mVGSKqYncpj#Zzpys^;N#w?uMTSL?7lBu2<_8G=`a20 z7QCNJGKbEx+Z6U|hIhc2Xg3Q@L6Xw4f_EM;n!#dOP(?{lGX5bhJ%w@Cv~X^m0CGD{ zD2owe@C0CLXLPS

Vo{miE_Bxq0;VgHDnh}jG1aM-WcK*!|3F3v6byiUgaCL z>QUS2G~Z8%sP3vZ1T;W;;Ras4L;sgGf6B#PJN^2NB@SET(@|c4`$C^<>3mRVRR`)==2 z=&kBkTY+l$Z($5onoFrfEsxDNo%1KE4|@`+Ejbdf-CL050DM4a%6RS!v+uY_PCl%$ zNjcB>UqREXCVifnO}k441vMl%E;}?1T(Ek8wjN|=rZ9a_EqUCcCXPz0v*iFHaFY$3tRxveI-4c8oY; zD`^O(*e5+8;@Y4xx^?&0ZIw4x6Xdk14Kfcj*_^*(IY~y(V9wCS?$qIc6lE3wK-}&= zRq`S##L-mChWQ-%>5v%r)DEbzD4UaQQXMZ$KtQy)v#ZHv;EV<+eVg6h<1p+Rwy#7+ zzX(Z#m%Kou-gcFiFWx9+X>Ee8huQJhz9G;Nu%Ma9IkCWK$6^0NBKzF$0fo2|@d_1M zG)Lc3o&6@`cFBY}6$%>oXnD1%HIO3p?PL&u*+GaQ9u#ZQ0h#n*N>SNc3pc%xdE;=D z{cQ|sYT&780JSwf6Hfa=l_Gtfrg#6A%a!+DXVFep4BB|RiY0Bq;c3FlZY-Qoi&~5R z7d%RFPYhRoi_Pe<8=rZ#uj|P?q1aBwx zdTB^9-MmO8ezlAI_>qIpgpGe`NYSgb9;+-8Zfr?jXI**s==)Dy?>^SWO2jywq7bix zI3Y%9pMq-+ASAn}-3?R~fcuz2htV-E#YdI{PIWQg`=+9+tB*5YiP?r0JXks_P@H?_ zU=DwQ^x-V%kBcg#Xr&(e@MgQwM3S^ijr9hS6w4uQy@hz=CP(>Mk~7aoh3^PL5C@Q% z0aCIJR5&FKeVDR&!T^&-qy}!(hiogmoL!$!ldi6ZuIPoW%Z@ONh?R0TX6nS;?BhOI z4iGpzN%ISmkmG8Wp5`e{>^Ohhausmxph=PeNVN*b?UTF8hSm^pCiMrmdJE-6TGk~) zO-`Y5-QFQ7AXC0j*)Tbn$=xd9T%#ca6qc4rVY4;QMg@+goM>lPzFs_~0{L^tH4h*% z4pocB2Q!0epF3iQET9EO3FVosf?OM#wl1gSqASL1Ea1BmfWnYz=~&ozVLsj2;e=1D zrJ8TttHOR;?-8gB(`#Mp`Jl$9)zX%kfRuUW5QAT&xLK6pv6aNKVfaXQ`BbPYG36*K zm(%+5`D%~xpK6R6)(>zUj-mx^7ag>!KIu2aeY1tj&}%sH+=8FChoW#5z@^dbNtI9E zA%~iS?NPsk*VrVnU-59PKm$P6%Y^#Plc$GzP;2<}Op>caUbtS6^w_!w=;flJWn3Mv z?2@!^lOM+)g}~Izf=m=18Or~?w#E9cez%z}Ted&4F2XVp79AcEXuPST%&6x*PBI+A z65hm{bT#(Z9=qP!>NTQ2tiX;Kc+L|`SD9gI zwJ!H>uz8gh0QQ8y@i?9RzO`gn4Y{pRqV3Ex)yD`^m(*DvX4zti1mFt?_v1g-Layb)_+xJ?o;km(1LrFdnsZn#9stt?6Dt}2OvH} z@}rQ1)WDaRvMVNOJgo2V|k)LFbZJhqr0YGk|1ofjI2* z(kwc;plT4rv(w8J;I8WlB}-VmKdYzA>K9g;n{_MRw%T|d>ths_|+e8f8ks0Mp%sfMx#z+?Ft zjPN?Hml*!n)Vp6L^~dKe%&m_*B}Yn++#e#CykoArSifSH4ote;hX+>Wj9wr(q6BKuR=!~MY-eXL2N z&|-s=pCEcMZkvW_P1)oI(+z1k!qqOJEf zJ%C)NZ(MJe5W!ZV9Pby2FaP^O@vEx2qu#^vpGX^TwNf2CfNspmg>}K|ouiZKeIcuY zioiJuXTl;{iO<9E=J$atm|t3i+`HM#Y0`+iXAit^?!qM-MlybiuxpoNizCP>87r6q zA};ZQ&Dog#E4)QnR$;RqcyqWzyTjZ;E_#II8wdpGUXr8wcyiwreP`)H3_XQK^u{*b z5rH%%;1q{+;rncBy2utR2EEPkat!-ZBpSt9P?+w1Wr&4_j%^`)1IQ)r$I;X;fbR&X zr&JJbk@;D1k06F9(zz=0e0y858Gv4uf0AET!f~HU2>j&{4pWi%+gp;<+fhC}-Zd~r z-P|7l%^OT0nEIwe^nIjuIvMOysf{c}6?Lr|k(=&}oiEK~=v{hy=_%E%cIDE@CY!8| zx>OTV4`%ML>Y>ypgS@GCZfha7^t@We+=6LT!d_ABVf7UcQZ z|A}EE^yvAMB6F$n&|3HOCf1NZPStdMYRs7HB{(T$t~18Ggsnxcoe+31SPh@S8i&lk zGwa|VR0FY;>Jdy+U_d4|B;!F&@aB9&0@ci<_+?X3ImiUMTX5}%xa5A&o1*fiq_686 zAIF&iA9CVbXl(})@?h}O!F(-{>!u?c&@`Cr6JL?jU>v7k3%^=oN0?Phq$m~0hqeh2 zy;NGbVr@Y{+E~gLSnl2l5U&GvSeQmN`TtBisr>LJ2!6o%D5))GjamGDC4{U#Ej7F> zC*`;CalbZK;|nOP=?4Pust7?v{a0k30*u>XrKo%bq1@2s!)a|H*`$VmDn+8g5gmtz zl61GVQv4Pd7Y$~|m3A@BL5TN?ly*tu`dC^TmFCbWE64bt=-JRJ=;vmxR@eSyXKKzA zh<5Wn@2#UB-KpTg7c??PFwq~U|9i=p67A^@SWe`gxogM}jXFU0qZ?9?(2$!)cJ|@P z`fd%o5)BP)n+$2q69N%IcR`sRCP6|%v-bLNMvHA~w6%GOsp|U$l;o`nsCmf49Q>R= zPPQOqBV7%Ac>XK@RqNv2D*76MP$!yS>v;^ROsAPFK9{--{~w=yp|CnR%D*G|`BqJG zYR9d<*L@%C6HbVH%3F(2?j%tA;rZ%9Tm4ZokQQ~s)HdGHO8G*Nlg<0+- zP#J%G9|k_CF%(t3l_z>nWEcCst!aJT7~TtUInx~}3vsl9We)So2n)b_uPdKEhgJX` ztSXj@yg+56j`InJCO6Aj=ONadw}8e*nXPT*1lu9D;PJfEXyLnqFtRP)7$ONyH;*o5i3Z0fsqIhgn zP&sLcvoJyF1isU>*i~qL zJ_pO9H`5+4!)-vw1FZeXA|`Nz;&-JA5$ok-+ZPnY?t^8AB)e|sr|Nbwv9yzYXI;+# z5WmkIgof4`wWK@gE5QoS;0!1V7OkC(JyiG(%6hCtUxN)`@*ps5Gx4q&!ZuDzMOfHJ zw0^ENNmmSX_KQDzPeN(nG9?c8Tn-m_^Q~k{_i)b+2U(uvCr8*2sf)Xk6~e-?F$}Z_ z5~RO8IFf4gQgQVDc4lcVPY{XN4yp6;2UVpZ(%!(N1%0wzrg6Ri?!4#hc2}MmKQvfg zKDP+N^A{CP=3Q&U#mF-xzzP5gJXtaiU|R%nZyEs=g;ty`Ej%~P*e}@xqa&nvB^#2X z?3Zd6c7qcD=`dT5%JpcZW1U_RA@EeFAGDD25B&tSB1!^x)sn33&E+D6twMCK#SDsr zfT3OtPT}!Yb^zJQwokmp8Q-kCFWQ$Z!iNth`?`4-oU^@03{MP~L^}u z6B&PdM#vN@lb4IGlf%b zMF^`8@`7~1y<(w*N&U5Vw{-&59KO2=9_WTDWS~4ao*a3R$iw)(JP7bc^Bm@k+QK^* z6P*&6AG|hPYdkf zL*gz3P8#B9?~4`1k)LHa>!MFB zp+!w)+k>q97&@g@wYecaL4j12+SZ#rTHw6B$*>>Q#IZ{db}_>6X@bQ^VH8&b{?tAM&Ran#yTY^v)(KWy$h~Z@2)X83A8;D#^faU98;uL zo@pz1W>ewnG|TdzpsWYsgpBx(@@zK}?MIu$R+VAdXnSHwN>5c;5Gmz_O{ve>x5~Q; z0uT%h3jw^)sJ>Mz;DV_CRt+vX&gcq&I5iz1>iAQ;JR08*M@PPSVNBSMiRP;S<|BkY zoLO0J-94qRFSL(cQnFbnHX|&5snef%%~$mJs4=&5TGrOCwE_w0%ATxa_K;Un`3(VV zMD_&F&_$4z#ab)uC{sBXxH-*g%zJ-KTkD3v0F-C{!SP*!Qb_GG8k-$xW^RcaOlccKO*~)}7o%G91kUTH}OnYh*fW zO>&p(Bl}t30E%ZKBx&B@z9&G05f-G)WyxS^yZ9MRra|+MX|L}ec9=oBkhu?*;~_m8 zy;7tlZ*BHMinqykxYp|}u~xusbs1Pf|@%}WZpR4jL9PVBh-Ks-CG|)fQ2qW z!dNO|5reFP`y9&$OYz?GO4Ln2eG#5d(lC@xtS5Kg)FibKw#tzt8Ms6ofeCPpX9Rhi zjq;M+sUy?Je2`lNFj9!SB|L-)YG85d2D(=mRf=5dMu;|Xa#S`V+I4fv@vR%Pt5-EFzXk7Ecxh$Oj=&e6a zms7YYY0EmT9x}^JsqUfzwfkR&E|w0;f5ViM`yUbm~Y+2m*1bMZ6VrvbQ`; zx5~KPZdmj5WR|5+jqc^QQ9awS=(@;iDOas=gI6IhTSC7aU*pChC8^=7?4~#d<3{%Y zf}w<)7z`Q=bx^@{Fn*}y2n@u}fTJAuxxr-LwAf!*G| zITLgzs^h=KkN+Gf0Q~273#Pwjjf#M9yCuIZW1#$1X7IrRN47cZ`u<~FIHL$~^%*$S zd>%E3WGfsX_=%rG+X@ViT@%r)LX?>y=v9CLwm>{K@=X}G$0IFONYOZl#RC1E1Z*yY z6lG$VAoMLf!E!n{E-8(bgw*weo>WwuP~QoXtmJxpXB!fuJU6+7m+K90y-s;k)2?GK z&4_YH_xR9?Aws!Ew7-v{iYTx~NeCY>^@)VH2UrlUS4;P7QHKwHv&<<7Scc8x+Yhtf z>81j$vJOCHS=^R23to?EIEW<>o2=2R2b8)yTQP>br`A4c6;K(|-|`;jXLhzN;aD0h zZw*m*#$a^%tJsSpTeX+S$NJ$lOTy(rp&lGkjigUjQjnD!ERz^9B z1tKx5=O;AG#aj$-g!buHZQNRRdO>n3ut~eX&r)Td&Y*P=nD!$FgMMQ0N01c7zWnqQ zY_RNf7Rs~(8qKFc*^L)lNe(CZAT@Z?kk8So2x%0jxBD_Mk!cLI-#0)ASK3w}hDlKu zv!)zYr@{QE`edo?{QZ`a=&3Ry^;B*Y&vIk4urb8wRTEC|H1L(K7j=knF5R{fwrwt(^sYtkx^@!H+Q{W{iqs#_uA#BklSbRNtA zIC-Z6@zMZRVFt9*IAQ5f-aDVb6LDGnT;YL#1vQ1}1bbw1>deQuaocMG{?V(s9J|g? ztws-ss`k{4c9&DdpJ;CZ+4@eM-`Z5g!DGralrFdL5IGgNnX?~(Jf4$O|Iz3Ji+U?s zg{^{Il#TeP4A+(0@pRe%lA>~%@yi%yH?Cb4T0rBbMs*ewns}3_j|Ff{&x0&#CzfZTG zggl)+&pzAG^AuMAc8|Q0kQn@s6q8rcD=1(`Nz%4K_)SvWYMZ)>w#wXHc4L$z1uOTW zz&C%mkR4&n|!W$a5IQFDb*+2pyT4J(`T zWx3R2{wLmv0hX!%fp%9x8Hm>HtiKnaUbx{c$TsvM^)xdV<7Q=x&LGfW2G0%%`R>cZ z*TSsMwDl!cI~ZRICIBJcz>vgVdvZKWz^RaE=nWbKK<%#&KW|48Lk3F$>iNC)CCBn zu+K3J0<+0C?k2Xh3DPTGAQ~5rDcK0S18*QXLk2w{Gr?S#Nh{Cu2@xG;d3tTRnuCL=R6bWkZBlPrnaNh|C54YNCw{Az7r%{(jDE%@#U zQ9{ksQ~6KoZ$$$;44`A33!7x#yQWREJr!jbJ6xB8(euhgIhi>*`X}!?8Xe z56YY#TWHUszo~nTpUAmC3TIPprA;t%5{kWQ(&Y*w>4e>#p7WW$T>bihGh_upYt?zd zm)0?<0XHbU7`L64vV25p{+bcj$d7=-YTfx>b6-<-t{4pzWB{a!MlItCKW&*;d=nVi z>((Hy_O?&=XsQH^Uic!O=VM(z$|*{#uG7kL4hT)B&YRNjCt?P(D#Wq9yL>@+7dyXG zcD76>k*TS;BDvd}DFxN>Gs($7IB_DQ&}|g<8Xr`C zC(B?KILqcVm>_n9BAL?qRvxduUaba?+jJe^=Ul9#HMhde=|GX!a-Kis zDG|b@R{eeoqg*rRT_zzFT!e_qT^qyby7mD#?NdlfJenua-!rW~*w|bJgSC?1Mg(XV z39&DApad0qZdS-(m4=`?|bU26ua@wg5@#-5;Z{iF^_>ljoKcwT&>!b4xS3&>sepaa-=3SBSn;4T!{gI>Tgn) zEcI_LL)l=+ZTnr6e0rPS|Er`06mPA{YBuxilBXc3yHVwOe6C~`88rf$P!z*La0 zNp6Z;lXqV4_$M9Iy;>HuPI6nMtytT1Si&szeW;w2EH}cAfRy3fN#0r4A6B>3$qTtx zz+h-%8zT^wvPi8Ra{uWmNiE<)_nyjxZstMz^MsfVAGr=d?^bdpfID`h!9v=nL}@&l)sdv2GY$}mT1i<9OIE@# zd9izRw2qg~y*YuYM>5y`RqHsE1*%q!bx5qPptGP*glYa%p2W!g11|k3Nv9?Df#K@V z04ZucM#~+@ouO19QyYYPCZd@-m1jFigOUHbIB~1W_B=sC8uN&`&v4Fnw~;RlUqs56 zw_gR`t|1Nx?8NhRpJlA!IdXzAqSaxwE{kOle5sNl4@@`f_^bA)^GLCiaZn*V2MkCg z|AQ(-cRBdpO1e8~Sy3Qg+GzX|;)|CIf|%}E(iBf{BC7R<^;XR3p28D%Zt!`rBSN$M zJ|(){E`l5p<+C0shTKL1iuztI-pX23z-x%sj3U5UZ&Nf-2(8spzVgr4Ka-yC)5@*= z$g@iwz^YJt&;6P1Tq?>}iUXV7_1svbgL2FJLR$+5XUm_ved#{my7JXBrj@jxmu zn=AIt3XfLfb6yXBPoc?}cZ^I~fO{F3TGu%$7Z>(c+2h=&B74=7E$hs2K&S7M?(f(D z)_b@eIKJh24p_DiWMQ>r6jyR%#}EfND{E@ZeY=sJAIK2Wg{cCrkM=eEENfmvVrbGP z){WeI6a|kf2p(9IX-gSZowT<9 zIxC0J`4%OJd&)Pntr9xhqB;sSxw9}4wAhT%Z8Z(mQVowS{4^vV9Vn-;5t)YoKyoez z;~$=UyoSAik6yR(ffE?Jxd}#^u}?@%x7R*I4^YaDDm`HTt5K6-3PH^T$-H<>!-o zxT?=lf-l&Kl?wBzV%9>G)fYtPh*5n??qqF9Ln6UzM^G>&P6-8f`F&?LEocwR%nw4+ zIuL%VM-GhQT6gBlArrRnLy#A<82Oz6EIKVQFsBo|wgXKFDHzKMwU&-1>3(B97?X?w zdu>Z(4`LylUIty`8p&r}M-2tMM5N)K+odfh5-BhEo=MtOs28M13~e=a9%?m*l4J#* z-)aH1(*BoxmQcxM+Ktb>NxESN5}9>W=YVo$5wu^K5F&}yooWCcM~wxj17AMd!+{wPRngr9X9-hnl$cn1xepx;ThGFQR*^sr;<=?5h1;%f>tQScRmJqjz zf<3^rlSyvdp}!hd9SqpyD9%%!H?%+7voO;#uZbnwcv30u{X4J;tNfMcZu@Ka#z#Oy zfz3Ev#j-%y>4{$TD);Hos;3%AIVl!ke$0Y;utjJMVGNQVH@EvEuY6y)V}ex-VZ^ks zpj9&s`^mxOh^~c_d7%V(l#Vzrxk68+Ce;Ajj|kRi3<96GB(gUjcM>3_b9`CQHK_rP z1eRSdfeG-K0jC2ilU3VXF)!Ss@fYw1TO7t0{{%AW|F2xwXCN@9fjL9^_5<0mq2lI2 zK5p|9Er}(vRKjo?SULdOKXe48$Jx@H#t^Qu(}~O(rT;(K zyyxyQx?g183+(^H1?H!{l{OelL*e~R&kLI$4?uBS)oD&l-s@nuE;V3(@1sZ3ZZ>(% z2W!YU1-O&}pQvv)4r|H43JNqPvc~5hfZbSXIo;-v=gMo(G+EE~fm<rx<-GOTHDEon-D&WIG0Pb|n)vdZEkiaU->hFMJs0b{cAi=CJNnFKU1bO}!#u_e zlF)s9>anlW=?$MQRwWPof2TRx_#B^zm8!8?7E+hXxB+e#i{I|3g*>J z)t=eKNe~)!cKxd%JX+jXR{A{sRY%^WsQ2Sd40WX&4$+Ew8bWpD^Taf>v6VxRMv;C>+ZlJHY04J$*Vg`~P z*%BZPrak$j*`_N8u5#n94wBBOu40)RiaK#MVut|;h;=F|xl*8<*bF-cW*=%Z zt|szbW?AWo(hl_IFzg$lF6rV8;PAmanA;76tHVY1ufH$`D!- zD{IVaVaxc~DD*7F(4n*Ap%|s8e%Z&9w>|fSsSr9X=<|HiA*_SltH(i3>JyUim8Y|C z{BnknltmKQlt8TBAk9XqHd$1~T_ijHn`BHJ=k?R7-3rVR1rw=+YT0qv=B3sTQNw!H z|F1$O;2aF0H#MRgKjppf-^$k2~dQS~q$A-K|tF?W!8eQIp~rZ-t<15+Ll|^uU-73Pa;|7ahk%3H>=4 zMHYC}s!#56MV>t>^L~wl7gEjmpQtE(XN@CEhF*eu8=L>v+8)b8pF#7ygQKkWsPm0( zmAG7(;5ejXcTY*yvl&o>cF8uI^_deE@USROqze3|4{3bP1pJd%I(IWyiqm*=1h)li zvwBHG$3XUcoUd^7-g8j*^Q`{PX>QYJ3*t(T85f!4e=(X1mo=s)<%Wd%>;!o4mu7iS z!wv>&OjB~mI9(Y~bOs|qQ(KqW=W9xuEgEhcZSl1jXC#J05y|NGZxx}zvx!sgpS*Lb z8@GyaiI>95EE!96+m<5GrZ=9mT#HJh1L$VICDejWc33mw%U#|n1l5oqyrk|?$7~~r z=r`JqSTCL-w}&mE>;mg!ZYB74q{)3{n@Pl!O0>iZ6d|15AJh_uR^$I740S1r;Bfv0 z!;hl%`XZ&*Zur-Phm|fsd011$v$19cq6UFvg$S*qjU)(%cs(08I0AI@4W#iD`=t!&^ z%-p3nCHxnDVM&pyb0RU6%7CK~$Yw$GM1cK=crRJZ7g6=^B>SiKMqZdy={ z1{eQSw}kO&R-Vp^a${g~i(g>8!;&5Pxx*s9IuAa7#C1K0PaQ5H!4lNur@(Uw*oI~K z_VN6Fbj9ivY;x^Nlce9|$tg(H0vfTZrh(n*&E+<*`YxKEFuVj#!6sa{YpHKVgd1csu399K+1 z^B4s`TBZWvqAlEPlZd!>N?BoDP|Gi!QKsn(M)9&9S9Fj#eVBXQRxPK>WEi>I)|W4? z_&1n+Oop?uKq#kDA__y#F-q_6g(A94%jWR{r(?blOljm~xbb)s7|zvY`0s{hvC`|Un*%=-)M znSuJudw&Ijb#l+BB!=S$Q1CK-)|nRdJ5zxzq+s~RD8fb%hQI* z&iUAvdsTp8Ao=LmZ#^*hR&@b2q$q6i=94M}73#06Y?#4ZDqeF8uIIIInp)!(Nt~WS z$%qLy38#f@d-Cdm$8&OTUef4;9)0X4n}iFJHRuK9Qi0p=y=+BZPjx6MFABDP1cI*UO7rcitSf3XlCvP##?oKgV=f%Y+ za;}#VE%hiMH{r#1q-PW&L8~GVw}ZhB%RTPjgv*}e1^z578%78lHlkehK%#?do&|xg ztTRdO|HFAr_l)NPn_r5gE}MvXeoocTS0cwF!9gHZpJbWLs?Dq52eXf6n4d30S0XK% zDn!7sf;Th8-DQ(r=+{t)FHxKmOZ{^ zYmZ4g-U;ps;Q2r?8VtjBeT4>TuQ%UfGnJyN=h-|IvO`po2$Uq*>YD5)l> z^Lq%z;plqEX7=6q0O%djN{NY0=sI~2nTAMXD#5diNWRMVFPmkdg3gXF1X_xstekxr z5qQl8UaqKl%Q8k$ut@ZRa5WB@*LxTH@XkepURx!C^d?_E&|i2&YGI9;(D zCR1lB>?+O&HdfE9mXr)oC$FL}g6<_&LLOYw5HKV`_0k~Kg1#Af1s6vSFe0A5@FAeL zM}*8D{QYc6s*Qj?&J^ScK*}qv{DTIx9w&;pw;1QCFYv=b>!Wg9@760uEb$hi2qq_D zHWzoL)WZD~dDD|#dYyK$d2HD=O~WR0nBsd=X9o*}8F~;YrOzl*5md@6;=^kMSK{h^VRK#y5|IcnPiSl!_F`1YDEo34qz_mxtFtnb{OWS*94Bi`?GWE=XI=<}4EVXWk38B0#D2Ra8&f+e1?i2W%b>k6RwD2O6BXti?wXF4Hsw zt%)!T!sDW-*N@lMsk`5Y#uSDwTkxX5k@nZIYY7@RPwvM!q)Jq|VqKmo=9KngE~`5E!)~BT=O+cTd^Z*W*TAw6n}E73>$c1?sQD>D&FU z$LeCbz`9Tq1OU*z2O&PgItMD+&l}K}GI*);>iFfKwJ4>;l;8*h{@c1o3=WV>-q*SYK2@D9Ph8);5RJq4VL9!w1pKMK3a;O)WgXlL zDKO7PvFyQa>Get~_A*b%GsLR5`T%b5d`&fEw+wF|f6-rGzDMxbvyQ#vxo3BJsv6p;vY7GK_OiTtBqmHujUzpE$3Ez)NknYys zOz*1Z_oz&(P=UE>Ee+GuLw-Fg#EL-4^^pM9gI5G>FQ0i5er_g}i+^_BNFK(<)Os-5^Z zz~8A{3doM_hsGXc;Tk@T&|qr7@guHzC%cD5J#Q56A*K9-zKby5=V7R=)cORn>@(_h zHb`oJ;)n%3_Z%8^>S_$sog-|8$FaU1wMfI$X-5#%jUAzB#CBA2fOIqXjnll;Aeywx z-<(;IVVAn&6mI4+l=WC$Y<(;z)v@i>T;k1*y_IJzs*KyN;KlE$WM_sszw*G{_VO>| z*r~!sKB>ZiPY8$Se);-d24}*zRwjm8P&jDYH0ZI<(VSqTsY6T!=-QST_@Df>pEviT zV1qv1r88s9|Mp0a#8rbw?gE+f(E0}8$+|LuLZLxC{^}gW3jTqem!hq*f`)p+sUNO2UR9C3zn$3D-^Xuk#S5 zH8EtZ3gI1mH50Q&K|euaAM&?cYHjwx#NGZx2=ccU9LrvsL7Kj^{s zD;g=4?vH$_xxXGz3FXrixIYqS)=N@taT9u#5CQ7e-V)&5dM|rP0HPO7BMXJr6n#1h zIA*ANvNivKGA|gsa*t0xve9@BziT!oS$7zrME@si7oRF%ZnOHcl=0qW1lnu~P)i$o zu->JKDoNVYXaf=HRgpTaw$e+=PnOy-$+U-}k}O1ZY-&GMB5g-<3~j+B5`z^Xh0sff3?tc`6%7g)?m7 z08V;S_D~d{cSyCmrxQ7|l{X%Xci|Kc>}TlyXWXCv_5kBUsKkWxiIREh>1s)kb0tTv zOStl~t~-je;a~916QGG;BN(hng2A-at=p_}FPQk+P#@{>WIGr^ci~Y`Shv7_OWip| z^jrNC(v#WPpxxgJIuzS3+9G-j4KbbKy7CvXlgQFM1wz(bkXNE3_4ml{JK^HpX!2YS z`X3$Jo8u@Lbpn+GLbNJDTqni^=pr4=W=?`1X*^%1Y1v@2wx}5`dL>krs4~b3Qw@!~o8>4|Xki(59Yra< zEL*+P{d3D9MyrYAUxRI~S76}_$_T4ND`$Qd+<8&RlM0zQwe!0oaitc)7)0e)v?jAy#%<7_s+ z!o3_bUQj8U+)t_P89_q=+GeKd?L;2Fs+OCG!OKRnE(4i4NKTsr9H!Y^oL&oC-5+cY zpu+om0%5_a!kZPzL<6qsYWgd#)BoU>9u_s55JhCDNa>ak-Eh51tYg}{`hp2t31xgq z{M>wF%&a8-EfwsM`u9=a-WjkK?5<>Kwd@#R21t-a7PV{*ag1CZ!WvWL8~_TbJrxCD zVx4-lYk|bWpxQHxWPKxC`9O1ef@3c0x_}WKNbMQt@p=?1`YP zx6<);ge{e7DD}Nk3gN->EzsU>Ds0Zg{V8hcdqtP=Rh=Zml2!D0meSOPVK~?v%2@l6 z4=yL8LXoyjbToOWGhui=?0!d%+?GUvUvF2F7;3Oy{CQxgea#9xBoC1kocqV@NAkyAFx@ziSF9SUa{;5u7h^6ZBVVh-s7`OM;M&BH zjFe<&h+p7^(UniO|NJ+-8JYc_!YOlg%aXb@B0CwvU8dmb3+wzwqwuzGs z0ZURIXE$IQ2jSWr^puK45Ex)DqR7`TH3;kg&#)a7DxTLTjg+8mLFXr^x^KsR z!`d&j>kcV|v0IMk9_GB^&74+Rf^v<&)d-}?Aiq^x;al*$aJ?eJ>UWuIOL&1&sG|$1fLxxb znOZ2yW1_Hsut9uQTpYz`0#-;{@qHgI7w%hPoAr);ex9u5#ZMs#J>C*n{iP~RRAJJ8 zNb})|T2k!op&Dquz!VWYxO~=an`c;ho;MX37U#jLL53MgQEn7;pZ(zP9{R*2ew=8K z{AMx#Qvx2Ev*r_}kxAXkFm5WPbl4>pla4SSE4#n1Q!xOZeBJ52f!}Iy2+}_$Jl>aH z6-#C&Fr(*O4ledbx~#hJf&NBh3Ki(6fP=zCaJWt7O==9YJjb0wiLzX6$e{a6r}(j* ziX{$$YmsXy%z3}lX;?7=!Q_fdgeQliE;g6zVz8BpKU9@$*5v0|1O}AYMfoN`KG;U% zya--J(OPG2)aq{>ErBmuSs$}gmR8yhuD}>=UST`p_j|qaRD#nX=KLc#ZB370EmM@{ zvK4>M0Byhj^gv&BZ7mye> zEAXldrA$CFH4te(t+IsLX@E8_ZV_^UAR{pYhwbFdK2p&AfexWI>`@60ayNSx*r+A2 zhmj&_f~#eRi;-}_XD!fjydE}?tyKtHaHWvw3H_)KC>q=hvBfuiYccO*P}UuML>Z0lVI@WV*W z>E~iSI7T2$dPv$;x@I~FPPy|QnU(hCH8PlJ&th@7S=;2}H#CU3PAe0+n89g8!Zs<> zUkYMheAP$>{dVh*fPwYtcEsFh{i3c2ca(L6HX1)JR%r7jk<}!u;n@g^KtM0qF`caw zv7CpI#h_$pmTiU!LD%yZL?gg{vr%9X4l_djOsiCvO4(G-l{DqQiPly(_}AAoVJ}pe zAm^KfUDR9onDns%N-x;uHF^?oKsY66)!?MNK>He~3=C|t&6H|U`~AHFiGZaO6+u*a z>bLAiT}|m(kMooO-pm|!W|zlhrMs==$A}a)H%7vi7~8NHAm4UajE+TD{Fl&DesW(F zY88*Jp_J=V*_D&VO%fbS&S7C=xnpW%GLEZaB_?2=NXbOyB4!h(vyfyPzd#F6{+rJp zHDJRM8RiHz>ro-LVn-=M?=HCexdDHbVG`+hts6V@hP7CGzJ-g zeCC*MMvueB7juvtG&GnOKv3+DIewNn{9C}P;gZC-O-C&|fF{oRtm})f$G_t+3YEU? zQC|(9#t7r-pAZsa@5hs%NyNr3k*gE>9?f{98n+NZ5`^B06v^8GVNc%Bfb~ zK+L~zVcXsQH}S)`BH9?2249l4HvsUte@N$1)mXze$5;}n%+^l&j}QzDl}R_?ourmN z6O=jBh=^J^EKP8h!A&qpgTJm#f20yM5gw!dToz}uNS&IO=6!8xi`Y|TMRN288~Nd6 zYMukd5*2fMQ^Kc0;#u{tkokG&+P*Gj~ zqD`6n)MxNqAvZEYHpzk`P++xEA-RxVl%Fxxs|4EWMC_iH@`Qo_obM5K0jeUMXFXhejoBxfM|n$Xp2{nCnLxgy zvr|Pe78ET?3McN>pcF&mI41Clsc=d~sR^6xBDq_ConVGYItw1z+2nPIf=msPRc1j_sItpilUYJPM;$IKueyJznx2h>46~NVtTeN? zh+L!f>gW)UEz&`^=P!pwwG{NuNy0+4^mw` za7ieYKqtEsr@8E2fh8M>u_;3!F~IMn|1$b9m@M9_H^geR8%R;&KulpAGnscd>ke$~ z;SBrT)~ta5Fry< zw3MW`Wmy#P!mqxb@4@kp(VtNiK?#?#G0PRc2G6cpu~ z6p?i7!!RkwxNFAwAj>?HRjXp*gAHDxux+@6Oi~RvR|Pc8r=7>(ULS`2t(dE`OujJM zIcp_+{5UP)Zwi2ght8+!N^3;}m=WU>UlK@h4`>jr?+=(cq^@ikF9myN&rHZgfdZo2 z#R4YiPSwgkEo=#LS>nl<^Fsn}&|3;df6>TMog}?_7az_e4VaRLGs_9I{lXsK22>SZ2 zHMHfZaW34fz1#W^_Zt1xvBZpOJcezuHNolN?~<58?Pa91kSP%vxtfcAuBv4fpO~c(N1m!;a!dicat6ZRh(B5%z52Zvp zTcZ_9Bm>}FZ*g*hZLdXzePFefGhe9CA-Pk+0D{8JVwQ}${`YYH0#oe71rl~Hni9>~xwKN* zV~}itQ!KQ4llI5zF8QPCX&bi`nF9u>vxsO(^}5KEQJjAZOZv6z(C1UkfqEUQ%3dtG z7z5%h_VBkJzTXq!MSWMQrJ|URpCbb(_=Y8~Aj=oH@7=CK*j$hDFp#o5$7knyEg>QeA?fA+< zM-#=vCa1Iacy7Q2ISk6^HRd{my4DJrF8f0+Jdn>$QhX4iMS8dSEIos&Q|hNkRw@H? zlgk%CF&@&(c%*N2SR!GG;?_7y(zWrebIM5zEDf0T0)K?2gPX4@A7L5u;+3&UtHRJc zN}*tb&WHD0-kf2e-e=j;U+oaXDs62sFC5@WWLXF0)7 zQp#T|U04s_*eO5Y52gx*bgF$ASOx%Zvejb_-9Cng-->PFt7Ez1n!5uoPS81QP1T3*j-SmFep^^XxA zos6UZJpktxs%$C2+zrC!!gyjATgqr*BC_T!#ICui3JNU>fNTX)>!Bh<7jrd4W{7NkMX=C? zU5|LYd%5dvSsVw56b4>%tCH>?a*(zfF$hKy6R%;uzw zX#X&GB5*vlqA(DOj^lw-pWGnhSrHIFGy(1b7cTd5%xFeoG#)ux?o}ik!bE+CsZ7d& zh&%~_fc$s*_K=gVmvSH0X2B!&XxdVd#jz1;ovZZ6CJ!kOsI2ve2Wz$s9l%2+Ry+({ zrP96GoF4-(@oZxL_KK@(XAuK*s{p5zMzY(y?E`a?%66e0%RnV><}A+`e@P zhmZyMseno)Wk<`GZ*9-hm1mphAG)NjnY+OorZ6wU7V_IN$lTF|J1ZGal5#Z1`d2M| zr7}baw)-rT9+m_{*n$L_Yt&eIb84QHj#MF&$mwMF8-pffM$Oe})m--L7U5=1equ#S zV3W_|4%TWS-{{;fQk0q0YuSBx@Wubox)I9)qV0US+s++q^~L&LrJa zDA-5agTKo?1It;bi*gm}fhv^~m-qu(c(SkXUmr~>YzF!79lv(ELd@t#5PR_>4 z3Vr>*OsIO*x}&k841TRsbgV+UP{S3nESh1~>vfcOT-*ceUE7nmF$3 zQ0(dZnSPDRnYm8h>^S8zI{v}03BgWb19Z()`u^1t} z<7XqXR57ZYB8bjQhQE~2|6GF&VG1H_^IPbDT@F4?s~{WB!%;+T-%5lRxE7`K-}0s{ zml6W0!U7+NT?5hd*sq+h52HV|Cb#ZzX|As3>Gl zCY&(npehw9I(wb=bLA}n&1S0#i1*W^V>;16Y;V`xKXP`Oi2t~dw#T98^AuxFY%g)) z=g8&}nAHMYCj2gMuI=0l2f-E~@ZUnS8Ha~Xak?fD6)ElFY;Ui!Oapx@SNgco%k#a? zDcPCNZ3)B$B+Z3$RIre){5)Ge*@o5d;u87wc zpvZj|SW+uG{f>r3^&S#JB|%!sKC>xA{|q!x@qzFi<^gHO>X;?X_V)&lSO|c2a>%Nw z+4m(d7Ant7Wn!$bN;cVOpL+NvnL0gqt|EA%B!s>yGlC2c^j-83;;DuX?zOReqEDYs zu68d?fD5B7c~}!a(uQL4yX#)1D-e%qFkS)L->&r;9~Ouwl@Yfi*@4ftoB$7#=rtv zoYhTbXKA*O!&CbeS<0&AX^D|Z8z&Sr?xb?~KzBNHQsl!*E@`Lu|J|JXvW;_Cdz-SH zCdPCVi2f|Q=CO%Z(6zL>wl};L@&Hg0))rM03}ClhUVqF2!jzUrUH|>2u$3b8hjJlT zxtXbnSusAjizeM%LndHvjl<*&U9DnnDiI(RQ0^Ocl>ykJQ{i#|p?RhQT3T0#E<9|4 zTfXjxH|+J>aiW>s2@}^V&IH<|{fw>}HQ|D?t#U)tp5+)4Z;)-7#Y3TmiJjMr8UD5Jza4R zd(im7K_Ji`v`UBrx}A@@)WoW4Ossu2ECt;lJT^cc4m(xn(M26Mc_J?P#~Vvlk-Iqs z#Hhuy{QJP8VQD5j7z@MCJl3SSJGJp48MLK)n^USyjMMP-!C;g0zMC8Wj<%mDKzN*N z;nKyMv?3C?>vRHYU*xSepFEo#X*~BYOK`-*J>VdJkkIMG<`;5B4x;4@O>ow&9XW^B zo(oh_a5sJRkIm?DNjZUCH1A5!n$N$yh#hlbYcHkVp$yC>B3=X&NIw{@( zJ5Kcu8>~Nl@|pxy8QlM3fzIqavA7R93I%XX-Uh#HBxPv4B!FEeq=~b`4}!VdUYAbs z3nzyvM$ZNI$M(p;PyKvZ-;*oq9Qf-(Y0Ic&%r_Pe_qm{u8y_fND99JAN{Uh3CNK1D z$0{?*s|}?3YdZ`CgTwc%Z)kLz6B9EWn)BW63iyfy&>iF%@PS3(6|OiY&VY;2#6fxi z?(Z~bSkVK=R_fwS9MrJwDVR`B0y`4H4;d<|2eZ^LiCQAHuFwGFLR8xkv+m2`-;a8< z#XA;5%4@+)Z#&69e~}Y3l$Q5U4Rxau(f=w|GL|^XAt!QW&?~bwxF&K4p$^DKH_qZ_>~ABZidtasKO)Nl zF`3q{|B;?RS^xW)hlZL5qfn{sg2j>x%ber-W~p?^53Xc@lh=92JPY{SE``V0GEpWU zYt|75P9|);dfu8fN~vf%r#;Je6D9HL6?HaDa?6#gi?{QX5v&Zt9+lJLol?$?Wwub1 zHmj8m^W=3kmBpVS)c@lwIeJW3&)tEf=^AuMLE2s8aKxyror>~WmU5g2@r)LE{g-G? zN1$n<3VxAysqKGz?OELCams;dB<1cr%3g~x$Ihxp>PN@5JMYv%4FAG*;zAbxkx_Y$ ziN8AQeeCjgS2%@D(1&Sc%sf7eUBP2)ypdbkTO55`RYzTuWaf`q|D#~9MmLz_?0nRRnnp#E8NA2=ivy1k%`;j){!#Y z{Ht6+bBddU+7>EL=f}Q58?ZoQa4TG&)U|V?>)0Hd{YbfJjt>(I?KeqmF@Yu_Y_Ou< zJSzOPMEspMPP4{HZ)!gmTo=@2lJPR~hB@*Z{5J-b2|`9bb(GUicg^q9@7T-6w>a%n zkMghKd(1*J9RolW@sMn$S&}`Rn1-z@fLe#7F#i|y-hc(cZ30qo|5u3S;q7xZOD8kl zawynI&*MZ#Oy&tRhp4*v>awBpFj;X0nQ+e_adm>-M0Kpd7BqmN1@bJKZdl(C7Zn3I z?0gwOH(=BU3!uXq^pvsm7YHZxIwEW6KK-Ms3xhf&2q@CVsy;RAHXI8mE%+hN4EaFA z8mOPGu_N$^^e(L3={g4r^nE!s|97p8t9lhUkvUb3*9p@x%+D`vCi0K}Gyz>!N}P1s zN#q@5Xg6(Nx@Xf@&llFle&oXu5yW!xVWO7T?Yo8UtK>rGu&p)f)!^_+vXS0Ooz`D1 z_j}t6@)UcT6RUDm>!L6l}ibHiSi~@nE)MG6A9ZF#K{b!dux9BV`wJlH9Y%_sn11}Q!_Eq`*#f~vcMWX zQVedkw-CAo&j?}ud7gXK=}4<5Pt(*=I6JY28t59%p4;QYsW>a#x&dK)JGnl9{nUQd zxTSz8i7&+99De6oZP|3PZgP!rturqpO9(hX>#{Jt6;ZNwMP=*~fLM;y=vzHg=i`qE zN^;MAO@%&Kep@llJqjTEcrsg?TlcLD*CKdss=M?nYds+Pl%{>kP~>9h_4hA9ZhMC$MLa_%T@9kuFWv@=O-l)L``o(pg3_Q^ovL78atafAkK5LcjZf|3IJgdM?qU8882d~e?%BBzPd!1q<_8&hx87-k+Kh81$8C!kbmn$AoJI7lT7v26`4$Fj*mL+j7M( z%gxHRKLzklX=B@Zue^5&iKYD5$du$+8&09v0Jg&=xG^_%$O2z?36%p+wJx@BfY($LhUC_#YQ3C*}(=oF_~8dP?3R2tu^UED5+)V?)_;L9uRdGc2s>9C6I zvG;Aki=8=%Y#D#fj=K4(`D&CtNhso;T`_in_Oy>-1~ses^$ICwGQ}`Ami$yd`vUyV z7R4sbNxnN2j6rd`{y`HcIm5SSF!8>c-h!}TOE6djf_pv%W_l`Uiik915!aOMpdE4w zty4&c%DJSSLy5S@bFFwOGyP+$MD3xM6a?cSwRGL(WO24^EXR4Ayc6`T0Z#+vt#a-F z?u-A;KZV-hx}0TysTOk@;jGr`Q4z$Nv<>>1>hNU<$tOBhAOJ#D=zBNTb=Uj?AtV`A zLH2BwuTvz#G~epUKwLHq^@jceX@sCS0f*Z6;a;)2BJr413IR z&q7Ms^F%;r^I24`+VtZv|ft;M!3~pD6Y!(*VqUWY25@2 zHndS@r{KIfS2`&Qx|S7}$WvlS6x!hUUyj0xnX(A|M5OjB?E?KI6N0cxVn`=-vHu}S zvxvr(lY8!uGMX^%2%qH|1nKg$P^Pu}>A8(AB&R$a>`onICjNe#DU%MPLn!>6g*;jS zrVHvsz6NPaz4_b#Ue#X3q%*!-DRfGMYd{oLAV4l+OmB*M&{sqY|?_bT(Jis#BhGX9nY+0vuIES&A?~=Z4}t($)as%k*vdZ z&#nVM=}8ZLo+%{H)$Av2(QQT%V}cRqemg*figHR0X4zkyx< zvNLHleA8s~)uk8Ogg_H_EJN4^snWYLpZW!S3e|QbVsKk@npD7XME~jwY{wVlot2$_ z0S3H07Vm0Wc4*BdG|q3YH3{&Oz%xo6^c~FrTz=l9>-(@pLv_dUMa%x3A1em;W@MJ< zA3=~3sgPwjFMI&WR-J~ygdpy|(n9AW_ue_FGgZCpQLO%hBz&p;t3Fl+jBW6P@&&)$ zh*RNo8GS-5H{3Nf(NX$Unz9i_(TIaamzy2tdQRSxfHdzSNHcEES2a z4}8nyl*vO{CraJQK36JefX(bzkD@6ncWY#$8*l`+D!i3+VST-Hs-$U@AOQ zoh=!MvIhPis6=K^NbWzYt8Ws?q6Po|@(rd-E9Brbq%r(ycukd6Wcev5xAYj2Z{o0a z8-du)A!GwiE_4+~%FL76Dn6zK!1t^uW~NA?A6Gu~FZJ7FXueHUioI8;U{zvq z-BPu8wP~pK*{&AI|5-2XeU@F==AYYGcJX#p~B9WP@Y zht=Zy%6_Xr1GIMBTGKXQYpU88%q%!%_q>d=#$emOEH6#)L2q}Ctrk4-45Q|C!uLlW zzHX+_Q)VCY2m+Mw`Fx%!r^&q}yC}@vT@q$E@rPjUrfBT^O->?WA=;C>nE=9cQU}ef zZMmADdPM_EbkM=amXc7wsK(aVLb^Lsw?@ML_KQ*nHzlD{e>{Gnmfock$5kRmne8Fb zcIXCUCLAS7KIMi-5UjpyXCFjx%*tt@ZkMS|i%&sw{4nA!QJjdt?jR#;>OATeAqm(F z3P)tO2H|lyZ{gW3F1e7A;vGAdwGvKqctZM{!(2Rh9<=zfYDZ>H&1}MpGbCRVP&DAI zvW-KbnOOQyEmxArnphfFMoBP~ZM{J&VwH;SI7ih-k5sCqf^Z6J6#9d{{J9R0`VlRx z9Mhb=uqo(EEa_RpWYEg-yj=Io;XO)8nrU!R+0b>3oK!yl3OQRM2~6U?{q%ZgZs@{& z{3~F2EzRg`wFtOI&!$~%CJX;LGCq1nnu!*;5KspCMJ?CAl5$tfQ?UKFKQlSyK)!F7?ro+TMaKTd*z#5eH+yI>XdG zA$|hMv*d~-`{5rUuT)Ux@fRU?1x@82o>Jn@>ExA?LIaEOw*-nO&LpOq(QM4#nsoXzx7BGBf6XEzqE&jbEeqnEg0*I2 zX*q@1s|ir@jS19xGgu6YvZE&6@m!l_>W|IvDYMqxzvqWANG*PR$6cut)X@7}G&D<3 zW^1z`n@M?IK0Hka|AYg8x`8(ddXB)+pp(*;h0|C>Ha@G-2Tm^?m=9%6)mR0hyPIg> zxSa(+rK5s@OT!Fy*Hw=P5kJhY>-rBkmHw{ZfY#MGOrO=V`r%C1^lKr;n8y7&-w*D# zE;m%V{LPRr@kUDT;`-G8?Ubh6(TTV#O@-M=rq_dtThHG-eOLEJb46bk8}MQ-Vn3Qm z#*DkDY|I^b%oV1dCqfn9V9MQ`xe!;0GV3((1VghB0FrB{_?;(V6_pr^{sBY3wUxqn zq<8u;{JxF&5=n(eG3!|RFx6&B6?-=S`9QssY>d2-UYIlOMmks?SStSRLW}nM@hojt zdq^DEq&V%<=MZa16`ZKM=1rWTGbxXFSDs3hql>U#=-lAF^F)6c&dq>4V&RRPC&%+e zv)^<7qG-1Sk|@&#g3F>|FO=1dTqR0K3VgKeHsdX5iMua4w91D7J5xwQ)}EMbjCtTf zax_kn%3g^oe%JAs;Y{vMUnrD`gQfFUWIWFd zC!(Q~s<-$?(;s5`Fwyr>eFN_;^NqqI%CLkp(%YFZ{@z)ClTE@ z3-UK-D1WP38V?aCKDnNc?-S>&D_g`mI0Z}%=BQ+q?&GOmX?zHYo|ID0@A8AdT=-`F zsR6mq{QO6;y>mp-I*;N3oL<0)IX`^h&F$L+-xg)j)I~{dzw*=Ig?n8HDd{N&O#&eP znuUxI^V~4#J)s~Ww96Nc-mM>Q-YV7s;%oSSyk|k1sCd~*&Z#j+0+pgN3uPE>Fb`BCn%39(~}8b*;tB#y?2WNGAGzn`_L$2W3^TFu?cjXwQZ1Wdhk?} z#`%15UN@SxAeZ{IiM`Ki=SiS4&xI}R$O+~oNMIC%X(YHoKHf;d15_M}No!=xvh8O? zx&5l%lEFrIqr?`p1{YgiOmxR)%cwIaEzT1_x<7J<;N*~t{d9X%;owsk_leRDI>l7G zhDP`R9TrOHjFau5ja#Hy4hQ}Zrs)6MC~MF)3%n?#TGbT|Kzb|0Z=0-MA0n5p4%Q2Z z{Iy`VFUmzmfv>EJGhSQD@$#4o4>Ry-PzC2&U%Pfsf1?H?5W;8npmtrWwX92*-kcb5 zodq*%Z*@NG5qW^rigxxS>p=1ipMBH;y#OLt?LtDBgR$~QRd|ty7e44qI5Joc0R-XI zd7a?Uj8lnYCNEqyLYPkc6f_npXE1k^flI_nL9imSqe$i}3kUXEmhRa(k2Ve&@GM5%G3 zUH_ssH?y!(GYj?Tie2(pJy}8E=M>+Hik3+wRUE4Xu9q3dZ{vF@deMD>re|BKOSK}O zuTC8@cHTAgI$}fH7@LG`5*{K*r}>cJqxDXxO}C!lei1%UKu)OvL&2G0>OX5x9JrXx zoE&-@uTqdgLqwm9G=e0~>K$K!bd4J5+4a_?J|@8k65p2p$!BRNmX2X-HOEP`O&=ep z-_nZd2gC5vO)EY6<=dZUmfBK&MvanCkwl$XIs~;?BpoyLK|+<97jY3dO6w|!JzHB;<$Os{iz;TH{X#@-p z{TyUdG7^5`o@aRzqIEREUQA$oVA%+JZbJDdkC4~u0u_z_qg0Dw6HkvKhE4RxlQjmH zXxsggK*AyjN^6#M2nBnPDq?+HNH2jd1~y$h9|4@^zbupELNiHcQFrRpPQkOH5eOpc z*zcB+9F*&cfeNR^ez*F($LFSKyth84#Clkv)O@0>z~f6zEiR;OQED zZyu=D!T8bW5=ZrL6Dyosh$i3=_Q*|p1%=Mef^y~@K){t3`kp|0aB!aVnT_n2$FFS{ zhE%LF$8NSkIocx8agjad)pCx1*-+Tg_SjdGKn5~Be@xUwyv%oj!UkrNSgn&P!kmql zofEyR(c~CRg(LcHFUfvdSZZZ^i*}G56qi@iFeXZY175mD5BsX6v{FNl={GLF94SHI zI&$yQ-2DBPvop*}ETLin{9qs%W_)>7zALwm9CWznl`r$kH^J7k@%Grbe1&C*S~#WYs; zj&q5g{P@3)CG^+=`{{~dCGiX2{#5Hl%fmVq=_Vl(*Rt~ubeV1ibOs27)@ zIz1t+J?if+{GoC!i`HYNo5qgJYc>;dLH~GiCT5e#uePaUm2$IHrtE(#+ir{Yt}*_< zp6YO1YzF(T7HBsB;;;!f{)pn837319gPc3iDk-I|m>GgW`!$nteZvtTVQ+_SE~`{k z+D|3NH_tz3=t}(+HSFG71L`#<{mf;I$<{@P@)A z=%r4Sh&Gp1i{<4Z$|cKjl%v|xdURSmMdtrQ_Dp%xHy~x-Ji)!R6)F;e|Is<)YQld3 zBpON=NyM8ieGO(m8{J5|1|W5V;v^q7K5Gf2g_v|ci%Z3k2u~0yG;RYr5oouOinfyi zxW>31=Gw0EEks@?&5Vk*;lsVkG4nmRo78e9H$^?Ew41^e@?UOh=4*x(HU`+S{QCw$ zuAJY`Hs&YSi7{tnEcccEzR(=<*~-dV8Oj1Ms*>X~7C<~2VRNCO*4}vv$m%pgQZoEn zjewJDTc(_n252+9Z5p;%cwA#VdAC_!d`$lo%Dxun*Ieub7zzR=h^OS-=1_DLlAv|J z))Y`@PjyrfHh+}7|0)oS^w=$vsH`hx;V~#fw{~Bct-{8&O4XH1L{fvr=fV3hE&IxuP8zm{(Rqec8&i%G` zEHcA{>BMxvEk*Py^S)d1vatl{-0dR|5Zs1B4Riq;cHz(?Nd_O~cP7-kRJ>VvHgfi^ z=9TU{(n8RyCyPv3?T>ZE4|e+_NcttIa1nr7?Kv#S0mMfWUov z6MlZKkDdDX50&l(|;Jy6%FQ+o%Hu? zA|KNJ)r!gfKc|RnmZPxgQo9dKR)m^n20yp6Q=D#XGeHPNtekOBZ36y0?{jV2IZ*|5 zevJJA%j2$OpsuBp5wQKb*dI~w0Vxrei}&W8s{iteJeo`wW+|vm#8yN8ODlk}T<-Hj zjVL6pUN>){br(g2-xcVd9uFx9c` z?fZ`hWujBc{YJ;*W^*o($xc57Xx7cV6*2mekQ%lgphDo+1hp4?(V42)9&EdpTyR-% zZN+NIrK?s&djx~CpwAUwX1b^kvb7o&KPM^?-viF3D zf+?(ouS4Ykz^5l+f9=@Zb>hyh=S{LmFxdXAh}z71^G%m}iZNWa9M|VK>Iw8<&>1AvA9dT#5+9?IY4%*IB+0XWeqSBdJyRv;7g@8~1y)nTRfSu&v zH-QC`Rj=+K@@K2NwT(DpAL4u~!m4Uk|9=v``5@=~w7xdeyDBF%&2WlU$%+#b>k}Ep zd4<8%COYi}_dX`W#ZY1cw$#u6imRUId|SEK-T{r0mO-l)ovd2Ou(|!<_Bv5;X(hf1 zP)1F*%qi@x4uhf@e$Ti#RHv49f@Dm@B3<*Ph!qD&< zOTK+BOvhbWb*kH`@{5~i%$^B7d{AM+q8UCbQoTCEf&BnOR{FC!IVl%B`C?8%q$r|R z39YkNd8L&%0CVXc*A=SK_88s27Bsrq&Tq6uY8!A4w1o(JtRczKLlQ!VkA&)@{!@N6 zz|OW46)oFn{bev;5C`vcPzKD{4W3gb5A-8XDj zdPgT1kkY+CoziHGJ2iAQ1gKqYUe3Ad&@UUncXcxDYoog-(lJrytX`q>FoUmbAaz%wn1z!VQx2{Isws% zDraZ&m;Wd)FZ@y9AYyw1Af%122>=DM^MPO#3#*g32^em>SbT{|_Mwe*Z?><48t}H1 z5#|6ma;h!B=9zMahh=kv8SfKk)H6Mx}jP3)`FV2K9Tzpr92oqA&d zne6Tf@d^ywic1M^NA#)GIBFn`@jRL)2I2TC!+hrXlLWjhv-iRCxS9=b-UME?be9wR zb+5OtMJZUBs((^J@Tk&`cP3lGIcBM-vkY@F9!9|;bky5HkXwTd^AEolp=uYSU?b-| zZ@}zuXM%2j1_v+8-at1r9hVbhpZWtgK-PFh=fzlnZId7G@4((OVgsk@E+gCGDF;-O zgi_M_F-OLlqs*A;ldYSK(5)@*uo<~ON)2|w(=Rj=IADFbS^9!)KYn=bqqTiY-J;4OlJkAQ5O=2%O(qZ-JJF!Wy;*O*gC(-rS8xNvyxsbVwiwZsAebI4M ziWPLCX}k6dec64{);WK-1g2QS#5Bxp0;Sut)ESr90zkQ78KY{c6E{wle%@T?1D6B$ z_`f;$!j_Qj#;;C%UD`R#ndP9zVS#B}c>jea@29jH)A;4>rNqkU2}jV!o6Vk_T3*J# z_&Uep8&Dm@Qt-sp9AB|kG4JZXIp}Bx+NPn+f0(eFWpHL9U7xzryI|e{+Fp`z7H?4{ zLitCn{x#3`w6mBd`qS}K179Ncx0ZNr32&yIG||X7LoQ=bCgyS%Uiv|Me^5kRK~j($Y3&V<)uck+yp&szHYVIk(Pl^}3dha9gg>2+d zRbD&-dA%n9!mQD#>nSFls^th^kKfrcT;|;2s2JgDeoO3*WEiR30vxgPXJv2qAQ1L{ zstwOEmPb2DC$~F@bU#hATeZskxgg0eDHR)+eAks`!L{I})ug-N$#Jq4ihj}#k*R5@ zVU78d!vMPrX6`*r&ZOK>4dFs6 zE74K#oA;?KkswF+yJzdw*Qc-`Lo?2zB^uXy%qA03ZSUoE!eMWap z1Ygd=WN@p_z#?LZ25NKt|3j5HnRka-x4QY@Q0w2L9 zy~a6Vy}@HHM5kF+*9K5+na3O^%oPK7RMl=Q8-b$J{0CP9^4Or_RiYbdfssRL=fmFf zxnZi-UQX8-pH#tGnk_l<3n09Gz_ zo$SkO3)G$eGbJFR$paK~xAqDEhNtBXcQw4SxpQI{Tl}sznS~JZ;vM$e@&H7Kuj|^_ zAuuD-Dd5#XXFO8f`KI70WGz(yUDZcWZRr%|`R2k_9F+>kQ&*IcjILZl(Br$V(Xgw( zywX#Ad9#m(@qw50f0k5R_&SvF#p#nMPoM8Iy+v~&taJo9^Wer_(tA33(#KCT^SM`$ zUa4tndOz7v*lGUK9~nC!&g=aJ;hj>bQ~#X{O;#vNJ}caMF7$3lzQQ(ZIkrwHX3*1} zR3b_in;lN7ZGt~TC^N$@N@kXHC$TI^HuhF&itq@eO!@+&pv6YA4fhxSf$Vr*iuFZI^k9A`GCv*9dH#FB!BV!QJK7z?f&oRDWUWfTDun=| zoPS2)CM~mr0&0~iKb|iM+8?il{y)Tg=;*_4G_e>+YLbn3q9EIi7;O=KH~fs9+1V<` z(TS=?6&^S&cBKF~&f#_Xw<|1ZJi}bN>a=^Y>4<}QTU%7f%2V3EE>5Pc!5B^QzVg`T zBLhVFtGbG$QcD1i$blc@qd(c0{eWkJ(CdjOu$d%6v@mbD*h7Qgn8-Tl&Gn_&kF#76 z6-oDV{$LjE@!cNaEQkCl?57pZ2z5BNFd1%M|G5tTM19%vlA^f}LjQTsm~8)t`AWO~ zFJ4Y!7IX~zJy{e4LPUzjDAk3M4~7^co9&(5Bx$Toq2JUa+nb)D5HbjznMh6|vYXlA zdNmL1Rl?H^g!oafG_tdMe8&>k*|gGBs*hyvESIu(B8OAPlGC|?33I#Aw$}VkXT<^f zeS%q^F2!)cv*Byy^gJ2^%1-{cWB!N~P43+V0O#(a+Wc7F@_PU+d979!g-zN#cBl5m z33yBb?S7tzAp?*k_NI6{nIL9Yc>U)nz}|S6dRoFABv0@D*~^uuPtNAEZe=_DF=OY) zL8Cf+;D~%~rQhAkBF>vQWfFnJV~tG;;~9LLg+PC8sJ@9b&4GnP4xmm_t2!hT&>^9- zrOc5zdNptxJUU_*kIuBLqwmaYt==>_Dj_Fmc-uX1Cl(n6c|?QEB+g}YXZ|Fa<{kL) z0;50M?4x^R;Hh1_Im+u@+4G8eO}2z2>*n~j-h2z}Ls$s*R%`S3ClhP8LFK2;S<;hX zu~*~!XgkvcWaYM*!YZ$g#`&Buv6&jKTc{2aEPUr}uS5bSpxx#rM*p31pMBwtgdBoNoTmM88uV6w!!M7-8De!p-~iaoimB`;}&>S3kvF+6Y0hqc)Rvd2T& z!E&y9``L`n25^S8Q0S=2z!}ICI(ze<>x!15zYl#CrBQy3XUV}t_owQVL#i#524F|| z)iGi=?uCn4JlUzX6Z5|;!mX}ST7%o&ohh0G?h;MSGi>vkS-XU4+^duwB1&|=H!+3U zmc0(q{x9Osw5tx+f)@kB z}pzw7?#$YB2V(eM?}Eb=Itu)ZA%qiftIIC*k&9Fk-Pnngi}mwJ_MLsJ`BD zUM(#mI1BQP$#bTuRX@he20T|S?MEM9GqM8i3h@+4CeN`Cn|{2Nk?jK}3Ht!d;{ztg z#r$^JB?DD;23_(0+gxHq((1|OKl#0}ZTb6oSG6@K&CXTY<0a*Y~I zod9-3aAI>C8kdiWaXDwzA&!NS(5g^fBF{83I5b^9g+3)xRaSF2WtAr}Q*}Af=0cxG z{wV5TXFMTR+1DA2F_X-qQ6`6umkzh-WJn*IJ$g`vDE2_J_0%HB#KI2V53L6306u(6 zkD&j=qi_lQA34JJ1hQ5`Uq?U_(`cfshjfo<4;=m7pm&3;asO|aFR@tn`4)Ge<{0m` zyhS1Y~c-TiSFc>i%7DFP^b$ zx_KYv*!6VR{&s0Us)`@fvaqi zaqLu0PBkoKTt%TEo1X7MN2Cjq(Vc>1h}*9w|6JIl57Z>AVJge>BEpQccN@U5x3(lG zOROBk_Uf*)+%xtB!%1TA=&39@-KwiTV$pFlHxpyN#(x~bs)9g9s3g|AMHH^*Tg!6^ zvp;8Npg6J0nkTf%|5brKlL#&tVC6^^mouq3t09KXcI7Uiw^sJ-3qXt?I>MKVm{QIV zoN#wm6(S#Xp$1UCpVN~!X`1wFPv*$lI^@XyNf=i=ODGVC)>!RVqntWku0wOi*FAHVVyTAsnoH zeJF-}Ge5P}j>_F|s5(z>Y1ot}5k-`T4*3Sem@i+;q^bMO#-xrnk&Tyhx1Mj7vB{O1 ztCqXBERQK;9@gkf111M3=sxW3J9^(>k1N$=N+;=kSD+G#N9KlB*m8wN8~PDtJ73ib zI_+-P)iz=keKS8EigkXVmZOoiR7FY)*P`tWGu^_A?4*!S^>)hi&LpAxg)F3$j9eiR zQIA*TKAY?2q6s<&dXnyw@4EVMfX zJI8*$OCAtoO8mJ6J>G=2C;e8S*TyTpGZIUq;S#L|^^P^{lR@iyv>}G3Soa?qIfXF3 zKm*6Q*0<1_tZ{Dh4sK3y z!V6J1na6p+yXwJdi{Jq@tGEySL!PHl|<94(=griJ-Q zA1ytrR>MuBX%yz|iuQw>E=Ht$5G>R=U`6Cmk*Lm$VR$>t%iL66oUTTFJO?fBL>6Nw zE&yAy^+uqr91=lpYdb6HXxqCLbl{cDaxqT+giH_cOumH}!utn<J#Jb|_K)^oMuaaNN0 z>`B>Wgql(eSg+gPH#TxodTX(rEU4F|(%I|QAO$)s#Y>ikQ zZiN{}>7`!|o6A&(XS~t<1gve>F#L~$nr9D9;{^w8_xaCAS}qw*Bx*dhFeQuWF(< zwje&RtG3U{S9-fWuOeA<5w#)_Zq=RjCd6wmjCYL)^Zg*jm>ko}mS&s@?{In$|cOt(88w5Ks&2I+4HP{T}{x2Q5rQG>-PougML zQQWKd17|hG-|{HmZb*sg6ERtjuNH#1wj_i^vBp=Ewyzgxv0LBQ?C+{24^^cMt zr1Q2S>kVmc1O{TH#r$0zjX_wST>Q!mE=LH>abrSSxpLDl*gi^-J<$lU$obfANa6yW zg9xa|HrxZ3SZty5o7d${g>civZ&$C6`rM8%-3uEtRmxHn;oq=nA#`nxW~@&b&TZlb zm_TJ_S;7f!@<31O@x5^c?Z{)NEzkC-Y*r6SXRxv?`qPM&UtYJD02rM{c>hDGzN@m! zhXi*4TZ<5(37apGkWNngj|f;Q>VMu$A+p`W2%+D0_Cqe7v+8XFLLK(NVRCEHT^*$E zVC#}6*3%JcW9K@~Q28OCF2k2X6*r;`RMGqj0nD~ku9KY#RgJGcoiH!j%u0^IO%7?n zxXKCKb%=yC`a6&Y#!_{zs<4oRs(b(|K-9nd27`ZxPF?c|(wbqp+(6wi1xaf_ah#me zDvp<$jtGfE`YqJT#jKHCvd^pRr{v!2!5Z{3sAu#?)V_iZ^Tw6py<2*3z$nd$=}DM+ zQSj4qP1h5Y3-4XNy-dGW`2-$wig`d?s=PhwK?f!boKs^NddfC(n|SBsrY%OBA-%k! zt6fZb@Gb~Hd})mf4*f>(^4k&GaB?N_{l3Y;UQ`RP6hWQJh~oxWqN*FWzsFFnUAdF~ zIVpwMr30W;t`S~1k&Vq4Xyc?)(fn`T+-eAJvY^`Tu4+`%p*BsP1(T1AtR4g(z)Dp` zSlK^@KWqfKI?iq&iDKwvZg%C7f2FjPig}_S!FQSTrx%1`(#}5wiV)Dj^+J9W(AEupXJ>$3QXO4DN z)76irCyjhJ@5moz1%KiR2meS{M5TKQc5H6S_L}6TD|YHtnHU4378f=)Lk4^?drzZx)`NO|w?%kGW zwGY}1K}!PLGYndph?TOjFaLGpcIr zNTJPH_za6y7x#rkba4w1yhlAOuLbvbV1-B@cv4`AE3#Up7uUNxb%ni)0>$^a7$eC9 zj1B(eq#3bg&ClUy9<@#>y})nDrKe-xdRI3{JTk~|0)%-F;lUjjAfJPS(372T%swEI z+={*kc<*w7seC*4ho&4g#zjJ70TTJHP0=(c>OKrqL&!8C90fHi0kE%l6JjDu(Ik%Y zrFtSo-}Q}VC^ySulo0c`&aXrSb_k9kC#&nik27-E8eT@~%<+zxGB`POb{wJ)9}$y2 z;)=?L%K@ujoAr$1y~2YOD_?p4qEFV^7`=Og1L+=Al!pwRI1}ISB)2IFP!ET-O}Wnu zimtm-=#Vcs%HG;6ga+PbB-2N_?%n}5RO-@XB4UfYmIt2{MBku72eeY)$W)z&M$f-j zUL*7C&q>I_UyShKhC7i_We)z8|G{k#gRX1fJfs^$;)MvF*CPh*=Ufbw`=&x!NUJNZ zdg+A!Xzc78{APJOH=qww!euiA(Y;j(jh(=Zu&xEN?j6EC_dSa<>>9C9%ImgLVg`Dp zX*?UEibBhNK7w0ZZCp3kMXn{fYb#(v`JFtW2LS`QJ+9epd+ZtS z{-Q?L(SahzI&q}F@2uGu4rfQQkSX5t!0_YUwB9Pq`?y25z43p#D^-WCdQa4JbKl`T zG4*KCe?j{2ym%LNA3;@=pj|E@arB4XhK+3h?ZnPYqcacpt)$g)lLm8Df?*KGG>Kdg z&7Z#Q4;6hpl^rc!RCXoYB^6e_N#HHInis`KQ<@&m+R}D}5sw%H6>rWb`hVFE53M zNkpI}B=|#0UKx@VUyJs%aNe)sRHnN$+m0V+Ffe-=ks4%4-}79nT#k_+A24m2lVHi~ z)NH5RgzFplklZY!LE$(3=aTaw2!Y%P>iD7+vQX~sg|pz^rwE9jCTOI42VFSPf4e79 z@|-7*oR5d{kIPy=$MOt%uusanP`J|X6!;*wRPrnzP)VuBm#`CILZN%ej!q_O4jzmecU z5F5rvC~E|6AX0`!7#4Izw}SxmxU8C3q}sDRZW78Y#ryAG0cCs&=D;m_Q9z#e9wZKm z&esLE0`4@sw7)Rk!3DJi*v+21KQeJ*maybMu7UD&LX4$#lP4zApyRlRJ19 zMU}v;BOm>Z$^ZZ%Qj(>a4NaXphK4}=Fae&16|6w9*rLpsR-EWqGuhFzyqW=Y!imon^;aOxOABkY z#!n&s4GFQNBp5(fn(lp`mh#g+gCSER2 zu9Nq|EK3C6KQGDPd)XZG<{hikdB*OJ&IZ0sB5Cj2Bb*r9%q_W%dWU*rVoz{&DH_f; zBs^jWKorV zUlb4|TVxUR4$p<|1PMfnC7YER7e|0DRIgS-B^^c(II?*(`PLAL`0Byjba_aaPeEv^ zSOU{F;IOI{&SpidIO-}M9xnvA2TTFfPeI5?y^J%FLrA!)=<{`;nBGi1Z&_&r6>YD_ zv$no~6csRU=B(pRu9Q5&$pD&E)s*_A|0=K$TF|=*t<6>_NZJ@n6j0q{CND#}==OaT z(E#VVnQoSN(*C1!I7at(ch@-DzOgOGBbjp9+^cDy(xAu6PTB|Ah*KvXbQ|B(5NdfN zv?>GzVKkCc4;w;$1*GvehZRNe(Im3FrB%2y7*<&L2C-m!uJJjh66^V?k7S|=*NgqK zsRNUZFqe8G2$5?$sfR|AdcSax?m z0H&}aL79L3PLt(KjCci8K|==l^LP>Fc{O$uBOm~dO{XRk%0fLjZK{%vU1bJLK1{KF z3o~nNg$Uw%#Zh~f;sG^z^zR$87{w;N?g^C7`N+FGB3<4A`PV8&@c_?BPK4@Ca zgpS}%s~M#mOi{@znLED*;vr(wOrN;<3;B`dH4cd?!{Q&Ozn&>}3v8skCq=?J?W?W) zNcWxD*!&qsh17mxxOcHy)bo+MIMZ3Z;GmI{EY_X?d%Nv9pQXf%5=gFbD+k0Ax7c}V z(Tq>T=_s{msRvt}i;99%5V{H^acg{l1Kzg)ii((Upah@>^Jbo9Xo~ad2Yp2XzRXt; z+NHe)lP0|ctFjwFx;9ZBeP;85PJjwk!r&mN1kJ;zLRIt+i&|5u+c7> zfeJueYo=`~A9SNOMOkjlDI*$$Pj+pYALdeM^fqtOq?Vl9lR{?^#EJv~#0;#Q5!)2E zl8@MmTCMnx>6KpbPylCW=9pIF^pv(BjCW0t2A?ALBc#SGh*et*RHkRW_<*Qw5pkdH zFDyh{Nxymr-|2&rRM&?QXRjsEg+Q|#m?&2UKO(5gK6BT!zACt*au@GAc)p_C5#gz; zakgP7=ii!+HN<~XsWpyv+hu(;z+n8kP&p3{4diw-4f{LE;(a>k#6>mb^`x57v?{4C z#zbS8qg8aTt@uD=OaPQYMhoJ|H19>%OrVJ-mv?Hz9Km%odk5TlHu-@dWLR9CkQ;XO zN#?Oavj$EK+=2o;SF%MSa`}8R5LAJqV&aH=2Fa@8seCZ$buakPOP{9vJ9s~ z7$2;#sb~#EUg3RiP(C3+)W5TCLJ1NDzqU(HmiDnal7Jb78TXk+oN!IrxwASUb_3TI zSLQ2*siJ)Z&ADZzSJmL1U$uU?LnS{fi@U^94vF!b0eKM(rhcAs)?IdQhP;3J@TuxP zo|cx~lPo?MqHyV4aCa+>Ia*o;a1G9x$%dKS-s3tnJg4l=OnH9WOr*`ld0ZJ&K>Erh zxHRJx5S8P_i|^bJS)-n6v;9)d3{S(&eP|CTwX{O5cwK)&Ujq8#4Xq^2n#VJM5|O%A zEESDZ1LPWFXZM*#E~n$ecU?bG$A##xpzoc^yWCXDbb*-&Rkf7N9;SNHp!C<5Mh}4$ zWZ|k-elqa$ zM!n@4TlEmGna@fZ(lc+VuP?YU|8;I{XGK}MqPkz5G?Y}Wt{tS~K>*m>s`?wwmn7kX zpFj-p${I&UT%6NSHQ*4JDxl(H@AI#~UgpBnwFYdIXjZUKJ_LAAYqi*y62C?<*#@&sg98ZNR0^=r(|$$f2;`W-PgZkUyVME!}jQ3di0oM&NxUOylv3Cr4EB!goHe}CIlOueGyN$X0bpqY1S4td1Q|d-TfGYI zFe*1@+Ug6QmZ@=sAvtSVn77(1rUL6;qK#7*0Y(p%g^L^Fqfy!(qWLT0r-fb# zSE7{_o!i^dPTX8!$zLC?R@>Q50Zun^o)5?@3_s}ULdazSHOdyIPNS}`Kn4OS<5u`a|Q zT^ev_LCeNw&MPZ8S+$?P8)Wsgwwp&yv@hTWTbROTjPZY3&qdd?A|i`n79{NX$8y!p zqfWh+`!zv_l}hhHx$}JzOh78~E8Q(jurGm15rppXJR;31f|CV0Q*0IU3P{7pzYjY? z^Nxo7^4fdw>Cj-8XBg%O)!Bi+g!1Ar>SMo8bt2DbKh7G^DfRwIjk?0-DceS+yaj_R@hBCw0K>$9sh+3kuTJF)IVf^y$7*_hf| z1O9_9q02{j_+9Cfd^g&h!m5N=q#5dqdTx#IslT4`JDIQOl~wgWS0uQw`Ed#bC>^2{~rhOU8R( z+u3QT1cQ)mcIwbHH~+3Z2X8epx8A!k%?z+s7y7vldZTD3L5?n4k(roOt(S9*;0~!yoL`~*#it7zB>NgbqEa&F6h}yqBERF4 zmzxJ%vB0pMV+W^IapRL_~|)VU!9$_1M7-Jj%*jS2??VLe|>8clNmr-Sj6 zA6hJhzgd-?;Sh&o7p5_T9dhU$bAz@Z2zP|R*#U03Ojw+&tze8@@dJjZgMq&BOO8KA zQaksUWhLgxT@s6`qjEyC|JXwzaq@AiaV~5`rw?{bGeYWK-96@eiLhLV47VaBK0wc2 zfsVHB(ZW2(Ss;|cuu0TblTJf_VmRum0YVGFzWY2d&*HvokNF*|)zYipUKpZ=Y_~uk~4fMK#zid*(LibR}h;Kra*; z4$aHo0<7`*Llt{r9DS9D$HkE8@6~)ryO_=$)W^z{3-Y1b5o4tBkvU$l77WTvv=beu zlVF%Q9f1(?lw$$miM50p<+2W#S4FYBZGEKKa%;C}YdiR#o(Th6JN~kL zF2}^AD$$0cgIx<8o|4D5ppv@Tm%CB&(R=wpbo3;(W;3AgV6Y6}zs0Go>$oMEAlu=; z659Da--IU>kdWG7O(T&P2kDM^#IQe0@h#G9og#*&iKp?UsniSOE<(Z2K(#P4BdY9G zmA4)AYzK-mJY>%9FzFBo9l0M8A=QwMxUp;I4qSTK8w?eL`ShS6;pNJx!;!hhN7EpN zNVa)|80b4bfLYk?CmWNUrR>Vk_&*zReIh&E#=vu)R4dnrg5K7*=skZny7HH8B#{jA zIfHaM9&`uD*kCo|>y=+?m0=@iOV-7T;TZI{iQ}%20X*vda;^VSTgfm(BaKJ07GThZ zkykz-|98AQr4jxykjqV_aSY#VO;8>SXKqF!=BZcjqPnw>z!%^MAXCo0jIZ5*&7>b zUSa0QWxWUF#-{h^pr zniEcDpnWY-i$!i=Ks(A?(wvN51r<5$MAc`h*_l*Z#h~lqO7P6-CyGO9F^p);{A2y{ z;ZOoEDkX9AourfqPDbJ9UVi=vl3;UhmjTg!g_;kxKHFeU_@|wkji<#d*GqhF0+dOs z-)+fI$n7iHvfQ3esmC=agn2t4H3JTM3TMI}r2l;ZeAuJT41cIG(Uke5zbroa-15Ey zr7SG5Qy9qgD#iHHW>)lLWRK$Aj zBfH4|%lejKPwl{MGO6u6!bRlQSEt2C`j-5qx+l2iu`38QM5Z#N&n|69UW{08{11{N zkve!btBHpOhLGUdrFH!{nc1O4app@|M65}D5ey#il1XC870xV(RQi+p@*Ul1n zqgOO387wORwyyxi$li#Ti>49v*~Z%C3uX>*$Hg;Z3JDWSFFi5%qY4hVS2hSV`6N&L zZJoDzD2Fem$ZMR0gpwXHo}DEgkBA{q+O?RPuzZGtD!f6ZJd4s}`3oSVm4Q0PwVPwiR}K?}~~F<+8hGtPjFcZIwjx*0Ez6}q7OF-wKh;xlaRpdyLI zwQ4mmok;shhRZ3G!JYSFXYc;331FBXsRABQ1+~}5YHZjDD7f9?{FUhpk}Xmyn}Ds> z80E-3L3mTIrv5c2rY&28{Zfe1YW|KQU0-iHKq@%p@Qn6Wv5ZEkXNZ^YFE&ai6>^Mg z4AgM{`_!MyvxvVCo+LXhja3^&2ZAaU>b0Fj>I@=wB`mnl@xN6s3IqJFlM^&yKdp8v zMq!k%5`?ZKCBp#K>~c%zSZUBoUV?$27SoY;nKVQs72B4$+&(Z+19d>oKqymj#$~S0 zVx&%1aVAuH$7V!kx09NT!M!FtFa;S1l z@&gQ)GYTQBxt{4|LHFp2t&12%GUmU$c0y96a_P5nCT}wdGDfc>WnjS*_?7|Jc~RYTxMV;*^~sbu6sVp}F^)K% zxD7P>*m0Fw28gpRQIRs7}GxJ9wjo(gi-G;*AH{bD*T z%Zy$5!_W7Ewj5}tH15@&Tk_A*pm-8H-SQFgu6#d&t|tvd<|447ze&uLMV5vYesfeA zCNra3Lt)e`Q$^Uqn^M&*yF=cXm_uId)4CL2ZbQJ@XkcC<{S5ojH=*I(g7v`!3GG3D zHwuKb(nik&#|eM7?+RkrAqEH#Q9->y#_c_^5=hR`i69-3Qn~FVukYEzrJ)0rj|-^H zW75x_Iinu>bvb?On|B7m$YZo~;>j|L<<T=sm$|30HG1EbdYsQMOqgY`~kU1r~lA|d$ryWaBFz^=%$ z5~#EHDd)7~^Kj~$5>^2z& zd8oh;*pKXQuwR-M!zYhJ;s=@&t3b;;NzXm3mVsS*<3E*Y+VM6h?qpt`f*Gl65BQSX z0dvA)Hr$pDIWDt&$M03q&kX?E4ZA{R<-Y#!C~43M7a;6aW5~0v>(vLv+mm-P3N?H51)P z$o4)1m=+e-J{YNp*{wW(ef=)MjVwfGgoBOPIMpca@n zvMjfrBBz-}OO{<@I+x2JQ2Mwn#Q^9~%noFKMuEvgEmP9Vh*>6WYv%JrHbm=65~;aP zptp0B5@J0GqgidYqn!r{FwG;bU`YP+c}&DZV$7@LV$)vr(pkAfMJR38j2fmu6pyY7 zgYcB&BDph1$U+Ax0bzvWO=OXz!*I&UN zF01PT`7`B|(Asbh*ZM~W6I^XMN+P4$5Jr)vc8ALAhkwG%9`)kG-Vz=w0xtbSPOHZF z)}WuOU^aPi2Qz05Rl0n4>x9!|`KA_H`)Ds@K}URMY4Y(g5)Vq*Z!RTp@?93?eR)|E z&l@em*kLEylLXPDb`OtG9>#E9kBK)PVQS>d&r5`_fQ4?@=`Ft-i6@u{&OZ8RQ(GpW z400t0>BvIpHDD()Ib)<&6geY>sFrFJAe}W?;vTlFrQ@gJ^x|I(pXRhCG)O?#Q+ z9Rr7#i+><~fNl$C&Nld2et(YO9PjGl832Da)eI~7@_%kZN;)m$(_yZ9NiJoZ zOYPQ*Nqyfyzc1F_EQYNF(nYL90Eh}#x97RyS-aHZhnBmsqo1+i-!3C7s@-=LDuRnZ zxkR`$H0U$jAD#d7fM;!Mi%Rw;>Rsd4gXK6xs@LfoM<2@{lS|@wld(cC*~!GO^&@?= zaz6IDAX^mE;SXVdtHtc?oa;OaIO3pha!s>7^+GbBbt{~m%N{nRmpg8?V7)ZIvR&3p ztui&u9z0mK0t0LuUm`rdE>`(8G$tJIhynUkMO0CKoB~p4`3Q!~Hg^1j(6XL-6XHk2 za*>v>{h8X5`j!#qyQIf9qQs~ambw#b+e$du6Z+eX1x+FKUX~la5|d`pIeTa)Gzr(S znR!6Uhv%s~XVvkW_3f}!Pr#C%s~?uZUawEyEZAD`bwBFzjy`25BA;2KjMsdTjMOhB zJgY-Q7U2v|p$TslRWk`=V|u@ilJzZ&<&1ow)LHa}<#=<^I4tA3@7moZbKUA10frW9 zRmyE0_dPQ>=_nqCCF9`TmXF=DR6uA%9c6*LGW!=Lt1J&k0wj5|vSz-OEBxSzW5yK& znknYbi^pz3?{$NCME>rQzCB4_+<|%LybgXwCu?!??YO=qIMR(Nh{giXAZUay0`l-g z+}(>u^bqag7#7wwk|*S%e41~Jy!6y8Q84yfaJ=?zP|Y zZ}OY>(q6b5tlaSdDFAarO=#h!Xk9_&32bR+x-e+nM!Au`dUoV6*UD%YB78ww@<)Az z1XykGF^lV)>`V*^u{~34DbWNWTn|l{zU(v*W1wo89f0$6>yP2d*q>n{!u!`aYB(UP zz^zTc4&9MJwAP7?n7 zq2nhRef_40*)@-MmV_TTXypi#G19IW`qrTaRvlBWG60Q;;_8P)FmL8Exgzb+{)_kJ z*kZ*X0OD-5gdW=YHJEYOI>%|IL(y0{x$@EEIk`AC6&4=NbV`aXb@JZ+%y)exzREF2 zL6Au!wobhK9d7Qd2(f8Ic<97)=H4&UrQ2bK@n-k_dMbEbdi`hh^vBm${@7=?y?8oQpf-fLAH z1MUTDoKvVo`kP3{#&cpbjbCABzkCVfiVK?IKt%&Xv-R%xO%0Ynih9+inlOOk7}X8G zk`nU3>vMmtgZFs^0yk--wWsYZ;uyo2%o%aT$v$bPkm=ow@ z;*#blSJzpv_0-3nhCwskq=2`LQp?f)w`SdiRnaqXzGft05T1pXvb1eV2|9b>gZ^@{ z&a3RjRadFzhQU(Z$d`K?U<=1&!^Cfl9S3{C_}s5FDx3{bCx%j&DM39GVkR^T+Ma|p zUqmP8*)ulTY!>0K@Aq1M8AV1?C+W>a0hF{Of48iykx+BF@La7R{df)@YBhfdYA~Ps z#f|sr3<%y$gnP;AG(nAm0O3SKGD51LwUEGHF!TDAXkpV?J;M2LP@FkxW4yR)8Y9`9 zdZ~$HFbGdsB_h6Uz4DcSJWUZ+dZHvD9KB_+Hl{FF$@L%zmL{0Us`vZIR2`)`bIMi9 z$Ys29oF?X7MBkq(VHn?%CPDgP0d9oh-p>jT(Bn`vR+eCwU#2-2zHNpGsi&wg$nNfh zkgE7>3~{Tmw8=nTA@}cKm&A zLGK*s<@DGBV+mJ&A9$53b};tO4N0B`q5FB6y z%g1b&k7$#+`zK1Y4M3uaLV+-|9@1{*iNG;y2gT9Gyy9`oFAztVHC zYxrE>X z;Bn}}Ydw#cM%fF>|3JGF+tY#jAKsxY}#f=*l$7T&m^h3m@>~_2uLq- zn;W6gHEnV+i6`+@!#Hk`m6q2hkPoT>3euV~r9>?C&(NV1ohX^aNxfDd6u__z!rA*l^kxb(IvN zg{e&fFH?J-P`7He{pa|BoKI5e;_r*tl-LbR+l2hk>Eskgiu1O_mSORWKbX7$GG z#TB{kGY|(>wgB>;$@FBAZY7EQAbcWW8y!igZORstg1%p1d$ON0HC9x;iNx~wW!vIU zp}^-DQlUz&CPXw%xP4Kc!X`UT0Ir;y$m4n=L6!QAg>a7m=Su3?LZ$J%xh*eswAho3 z9Ng;hcMBBJ3KN|u?H_l=5Rcha7A{mHwt|e#l&t4rHB8e_Q`;=PT6vq|ZM5U!Hir3* zvn2#}T1;`n{v3v(i}bB5$ZAfGuf<%Rp7B~G3!vo<*EWxoZq$8>%-(1B%ok2h_vBtZ z>PP%6#_1ycxz*+T#D|`mc_S_gpW(v$XB83Q0&d}%ea3Y>Q+6Nz`TQuc8DDh0aB}=t z!-6+F_QmdE0m6PJ`HxZG11_21pG$59IHjaWutR1bjk%}*_#O$c@zXb->ix-KX>FJK zzL5IdMt%Anx+E7|hT+QNs)GkRZC+<4yP}$6y6}ZNr8er7)x|YEO%@o3!R?J_BL1 zr3n05{NT%BtZZJKsPb7h6m#o>Xm%E<8GovYgl?_7e(06AwesfLzj*47F!W1uaJQ}J zz$CFG{4|J`yzC7kH=W_fd-e4ngJE+%mltL}4x{I5Fse*9^j3|yHt^Qp z%M%@<1XxVD%R0L{M-{7#Pil}nTkE-;wcid^Vha~nchRJhDrl=`C!05UvT}^W5TUMo zgNcU+>-!GWk>iZadvP;FeeYHEK8_*~Br5m!r2>qU^=epn`=2QTl&We7jBhov%fwi$CW@ofFL!CMBoGr+FJ z|Glj1cK%URp!oX<@1sI|+d*qk+LLvo%bHe6tmA7)0w3%b+ydtdWd`b8+cq*H$5vNd zC`umUX`dv%>3t#{4F>#|e#p4t6Tfx(&glA;4c<4sLTLK!)_MSTT`#vN!uR13itG;I zk@O;&IVjvZRyph?UWh%sRloYf8|xp#EUAkdXb>p>EX8^b)=mvBOY1W8B)&qL%5id{ zK?r85N^Gde%477ye_J50kJ=dyON2U=AuY#5%NIr5(V$R(!)^0Nh5#V8F`hc-YBVy3 zcAN#ICXi{jESRkG(r`Ok$cc(^A2$19Sbe9wMF9MEcY&Abh-G|P_LRiYyR|?-q-3j4 z4qs=wpxat`b2eBk%&;*JIddc+Qi`v4Ks_5NZ8@y8eZ2Jhf1bbrR*=LC4{hb5xJ+Q) zZVVJ=OEs~2&zseBA8DOYwJ{!kqH(yUKAlP(4)tHiRSB|6 z6xLK>j%H?Slp@YCeLNpH_15Pr7UL4Hae@x~sbkX}@ljYg`fRM8=W0Wgge9>-4X(|~ z6>-YNS@Ou>fB5})hPA?@VRS*UlqOV*Mqm*^7HB0tuZq-y2(t1|EO}>$j>qD%w$D`w)1NX z8$Cq7rs|)d&YdT8#vI>BKq&KvomQk)IDUsH*oT-&clV5qygzy`@-_tGp{Em0x8#1L z&I?Q};ZIUE5Ed^-?=0C6y9ouMA(NKJ8m7BT+O%uA3Lu=Cx3WUuH8IR0Gt2ohk# z8mh#+GDatqDiO0qF6U)S@@41#n=UCv3ueO0JAqI4uSF20R%dlbSuNXQio)^DooeoM z3Ne4;pxdgNK|2 zh(1$+zmZ-Wa|o>32@kfTP#>kidGqN7!sv>N3~hZKl2UGSqT4-@e_ro9SpddxTPl@) zfqO{orKU#1?ilL!SiJRgH#xy9`iuu(&MJcAz-ysU4yv~bHkndtRn;)6F=~-hsDgvgjnF~uZl(-NwGQTrb)q<5t?zIi_d-Ypn3tBZz1GW&dZ&i zKM`QR*bt~&T#X{BU(Lus!yD2+ag`Bs!ix342kYxGPhDkhwZk_v4(_N@?UNX>?=o-6 z3w=CvgSW%yI3_vGolF?SY&s+4H5w@gho0ume*xZcBEFq!NRUTBRs~|4<{O%jCc2bK ziD_Dhm##%{93PlyeIa||tHbD8t;gh_!$u-n`lDQ`ie<~wtL!e%wcfPR^a2D{)`F0t zyz+fLLyw#NuHKTLD+jI6m!*W+ba?@6CwVvQpb2TVF9PvIj7cAc`0D43uu7Og7jH0- z@kz}RVyn6HRSZ0FBWPdqXzUxf|Lno0$Y`(9wDFom3^WIk0O7P}L#?ZG{qKUMiW?}f zyqES)ICsJyvx^Cl&;OI4{HUOx$9<>qBu)aLY95_D5Uk6_dze&f_fa;@{RA&i#R5)| zHxE!q3f!N1A;4>4pq}62>nuoe41K@+_EQ`98rObfN-*+w*<@;yaDX5|^Q9>}JWlp< z8TuMwH`*UaB$AZ+2DG6@k*$0`LrZsQQe5gJazRhF!s%#-!hejvUPS3w5Qdo=QP@zUM&n z$AEtauKhBaICn@gt)m*Mo=-bVHhSz!BOeD&8O>&?pzqFo8EKrlC9xblq0o3-!XX&+ zk|5I*cR^XpquIpkwoi+^duJcVS-3}-3uitciZ1nQS5VsdL3(7N82X1 z4G}D9=^3J7|IdwIBD2phx~JRNjh^{w@PAi-dM}SZGU2FDW(I(V7)hITK~JhC)4o61 zNiHi$X_1{&cfClH)`*}@@U?{Y*kZ*ne$kUvJ)93T!rz_eYmA~3#U63x3hW8rXXFoa zOpboM7uFoG!lm7tqlF>$V0@FmD%XOFZwvNWQSeAgm3ye(H-!AUQF3jN@ex9T z%{v>Mvq}ohKgE)4`T$q7AGc(d!emI!e~pNS`jKI>r`IJNV0hWP&* zYPsNrb@tsu{2Y)dDSVl`T{1n|O1&3Yq){wRDo(xg5p6o;obGsl##-AjFUgY6NMm+} zxE-nsVBS@n$8-FjdmWw`+8((ktKK9|7`mug*by+!xC6oNvLW~#gCy;Yu%8%)h^c!j zC`Q*pRG))=wr8(@RrscfMJj~Q8I?)nW4ke34G&MMrB?G^66x@Jzf=aY^3#@icbhZ!4y%w?#x?#b>G_?-%94{ zit7Ha>#jW?IiCs0`uHS6<)%e<)?jX~#k(0K8ez}#U8uDJZjS=#yxJMFuSE}8=O((p zRtv#W=$bu85ggR*OVI3DYXH=GUzaksT(o;KFyi$T!SC80cHE7p^ocs+=)vfXuFnNO z_;3sC(g&94FN)#8VaDqjl z9rAw$l+JCL-dO{m=7|T)<-Oh7s-u#vh=8gSwHpG4dSKGJM$!E45cI9FVy2~E&zktV zSL#@M66)uL*PCXB3%GAL4R2TF3wD-uZd?Ap}!#6-CsI;mTvU|u=tUz)r%iBq4mqHijE?QPm;yftlV+UD@Sg55h8 z*LRi7N@7ULO=94s-B^aE8VIB4E0a&hD}q|Z(y{M;#Hz!9@BJ5(M~=Hu`}1{PoraUQ zAp1V*}{S(k4fZZzym=@EdFBFUP9z-LPol}%dCY67;?)X4&j&`>NScY5LH$Lh2~Sv`3Y3lirAca zrEZY-66!0B&Z}pv^_b|Q3>+G=vbV5QNmS}q%1@!CQwIw;-Aoeq0w>nCG=2@*0Di!A z#lATBP1r_yn{Fo2+H<(-F{bP8ITm=KO)k$J-|LUnzu*NE70-VrGA;{Rt~g9mOPz?_ ze(LIFUZaDaC115tUcY2+2Ca!RuDUW;7YJS893dv;?{rY zT$=aF88)vB=@o&&tNrJndj~e*70&+QK^0Sr19xBU?}Lx=p;a-FLyG{Q15MSg*yjO$ z%%LS#(Z&7mtv10Dz0v2V=*09=|2F8%#$zYP1Xy#3qOI}havfwcl;%$)x*z(+5TJ$# zGA&M}87W!!JuaU1%XO=QMcKQ7c0gxq`vdlNjGYPY`t;`@TZlJKzLQ3CtvKLEpkzv7 z!h=y>5E=85Loba>onK2Lonjs&`7LD3)4XT3(w}-p@98L%U*R4j>rdJMOB4_P z#QHkZzzy=m+oTqWa}??_&9>MD<_QJBnc}s9IPApY5-Am_f7=K>A&qX#8(Sy)qoKd4 zmae{4U^-#I(kq*T&4&P``tqfG$V-LPI zU9*_a0G)hCjr~x`1|FKw@8)Za zQE$Y9l;MbnQbZ-q81^WSIWO8S;H}X@#g=)wakVCA;qE+&m-3&a3!^I7Ue!IBF#jAQ zwWcCqHYwosGwIAFH=#Z&t>Yom5IJhT9R+>7{qy&JTc(bb4VEQL%;FEM@5UyK3tztr zG}Uc@u$y1HEZkDLvx+>dwXwD&En$=yEgJNeFF8AbTjb#z-(eth$yQ?zQYp3Y{kAEe zB)xi3z&pbyKcIx%^Mq=7hyAb$j1Z&_6(c+QBr7-QAvYhaEZFE*J=vMW)O&=DQ)70vq&yhNw^@BW4{2$Qk!lPVO4bUy z<3C^uK-k&?3jcV%pPjMK>2OvQ4sk|X^M47O2TSYII(>C+D|TbBV2({uR((X}IY^LS zR6*69NTghzZq24h{OoVq7hn;P%Ect#kgadqDK<_9678%!08sZIx?#)<*qbiV)Nj&US{9_!GU63^h8xV}xwLs1-x2-uW_OgD zEto{o8l4U{0}%ANLQ3qsW*Cm^XTcqz;|0;JWkjwMGN=!8Z#!c}Td9fqWEJ0UAycou zac47!-E8>jhh=z^+X{ePxx}yS-Ie?Mg1b-+lmz)Dnz){u#uJTp6EukFa$?3aFXP_d zos{KIF3$rL^u_3(o=GO?0WxHafH26iSWv~c`!EQ;LJ+bQ{WMKnT+T2wnjp`ur)YC3o{uChptSui{l|H>Fc8KJObUe8_d$LHeY1KBTYgKq z2q)I;r@nN$HP%0dCV(@hZ)DiJ48qII?Q{i;&Gz7x&{z8)Sg363YfuN`duqDIl&h*@qXBk}X-+?F;6(wKNiKa)5O5PAq1Ddo%mv4z&Mgf>x z-2>bQRN8pql^XcAb_eKY0=$L4ARnWjS~D|GJsDGrXu z_8HBl`*K)h8yYP>3*wGu`GvK_}7x2atI}vdJ zrYYbxnH2og#akgMK6`aHp5h9VTu|-=%o3mLmsZ$aas}X%aFx8UqCjqaDV+MD@-$sCgtp8rg6N%9764uj`o?7;DkkEF6p|;6#l$2`+(YzAogubn}a0 zi}T`rN6}YjSFidR1Usb~h>{f&3DI~y)QEmZ}OFl*l>Ze9oo` zlYF%LT~Jj4lhkT?vmhBk~Q#Dmz{w9}Y^sk}{AdVot1nEuB?) zw8mV@hd9FLXIh4^Oa*ygn5dQ3&TN2n8gwQb91p~lW+mAg30Ng<^)#t{QWWsM*MBCb zYi&p*$V%+18o+}N=BZb?eL@9xWJ9tgx8;?>0YmEfp{UTlnu7GYQDoF&PK4zL`7@;$ zbkMIjTL zO$L<@ed6>_5iz6f(jFzT9O-O6VMw_E`ATym1imwmaRl@tHbM-3j!Y2tk;E(aWS_Uh zZ**76mg4D>xK4RPH!2E=fpSj#CJs5yoz!}Sgs>8d4w2V&=?^=NV1c!ow60l%2Pv0G zp8#3%M;p|IjKq#RFqamf=~4_Gt$P48K+C`5;z+*63UaQk;0F-iuv{k!#UNr)EP|TY zW7m1MB%$hVZ6dCUXnFd8M|9M8Yw?7L87XpJVV#Fn2QbSz(n`ge>(5?=g}FyDNKf5_ zbApRDR!hq}(g{7z4>+3t+?#f&;2zunMw3Zzc@!%HNOsXw?C!=^dpO!ZQ=oc?D$$0W zd?qUw5ff(;@i`R~NP>L*Vum^AGR3Eu;R=g%AsS)h?TAE`4(J4Wd{j)n$pK|QAksTM zPrtOdrPGgRkr8vr{pp3AQ))du&8G6S>mH@haeOi8eI;G*HJ^?3$ubw(vo4w4MFE7p z*9W*#Ts*$W`&3YaW(r64xoYkEswk~xmpZmJ+bQA+dY!+3oD2)ufBv7nIrRO;escD5 z>`$)8Ko8x30!YOI0)PIpfI21mjK@@RezC+_DRa0#_J8X}!0`)}+!jJn81;F{3T+O^ zW>X?zmCVO`OtqN9!bfI>Om>3N7hAmUTw4Wgx0Y@ryRWq-#4*L`rwIf?Q_5oVoBX^ek&v3y?yD+u5w86nsYbSU&Hx*X zI4QunZF?7UHSAdi0b=8uRnk?E_CGr32^ zyFtb2kEwy1cP#L-ArOhBu9Nw7C+?pJ*SC426`{1w((epR%JL2U_N`~gac(=yfK zilb>)9aL!Q`tnYQ!{&m`5$Ke8(zUkt-5EcosSyX24IdecsnON)2h?3z1{8tqRuuzR z8ZYSACvlD)3}GQ~q;GBa zrZKPjIQ7l|MZEAc@A*<)@fS8+uT%0rgKbA5Lw-~u#sEN=$ST-fNB8IOEjgzDG$*Vd z0P67PA4=4;vN@CFX)ceNX0HsrMkev8x3F7F;|!wT`~E>SCUd(x zh3#FpjgtxXHM%18M{MXb(mY7AqgBJ7`~OJgf@5J_E|un$J?DtPcLG=hBS-A~#{Eu# z>fVD6d0dq$%ap^T&l(_Og}sItqNw!egQc=N>|!!-g`$1@IUb=Nmo5qteVXz#AP>r7 zFU1J^7Md(97wV?RKJ3;sJ;u+!>}l8$Ho%QZsvX51NtXGMz~uR=O3&b|o2|yuKy2~3 zuv|kO70T-e&RXR%ulCTfH#<9qpEtemWAO4pVQ^%e%LCfT-fE^T5yxQ0*us4&_$*;j zMuUN$*x!rsh;)t0ByT4mmIVkzc)17E@Txa36G}uCPE-= zLLp_-A*3{&`+BjwA{b}c*FwKdVjmu-MfAJ_oTrLIPV>zGTb&YErJa8cx%1D6_mfKu z6wXLV;wYhnYpV(vGJb@;xwtCFui@|Vd5)?;(dUIIFC`!6-Tb-`O1E=<;_Qx|>;QzM zgxH|56BWQ&8+jq|DNKj=TyQL{lj=u3u=y6r1fdJyi^jh9&a@{HmJ6_Rwp)0G^YB8m zWclCfGyz+CbSkPI%2<39Zu&0JieBNK;Ov)z?#Q_55nMBL!*!=La=!Ee5B!aO;_^Z# z_HO{nFrWTEuhn@_)>;XN9c-AkH-_a^n}!BKzvnkbkV%@GW@(5kt-0j-6D@PE&(K>rZYN{E>%i_-3#JR zs1~Ky13c{q`BG63jOG&9J6Pk_1?Hd_qJ-;o2`ZH~s}BoteCLabi)P4uxZMxyHsR3s zk`D!~z8tq3k`{8~=#8`elI5`q1o&N??L&7t&}JdZOW1IfKWjP{FpkY70J%$ViQU~7 z)KJWHNxe2*gV1S}9}9N|Fl}E6yx-LuFTf?51Q-Q}7UMxeQ?NhUFq@{|x!v{Va}qF? zPW5ss5Zt^>Fnj?HJtfjx8iXX=r@ZAKj_0H6TRq+U;>le~$ubD5ap&k5L+ty95Gh7c zuk)w|Y|iFq;`BoUsP*$NGuqqhd?~If9~$pH&nq>0qn6)IKFsN9rNr69zCL)ykrLNO zRs-bhxm+C<4Fx{$H-SSRN6|at{_6@E@z1G}4(DELBTl0#b&!50=u2c;DP@!BerlJX z=L)*3bIR>oeUIC*D8;GFLl23A3S7?@<-f9$)yz&!kLX7ySv6RqvZGX)lYNVIb&}@r z#_FptG9!kcn@QPdDBjgsVlMI=HmUft!X~i87BOtJ?`_Rb}E+2vz5zrI(BH2kCV>!PJ!| z5_Fd^aS27~dcR&;HV~$ax6f6+R$JZ33vn#ZI7!vp4%7rH3rOXznK$YMyT`A|qJ!sW zUdw?7wo2tM8-I6PJ#I@3F~Uo0FRM}?wtInPzT|1z*co_$Z6mj-cym0veG%$BTBx?v zk%tLer)ba-XoEw6Ve+jgT-)>Dy1z*9;v~dzh~>t-R!Z%I1}>`A?Gw;PQv;TIN_`$^ zu*X`3)-(R4L_)V+=uVC7zSTc`uaWPcMJm>yhDnxNU$15A3`N1BgRTbWM=jO68@(l{ zj{v;$;>WwqkE>*j5d88eQF6SFYW^fM+8gAJ+kr47QI8{b-!YE^(vldO2f<4KQy>d? z4S0P$merN`B+kAx8@>m{klTTJ&5BO8JA-o$L^M5o?hXY&1{l?M27=N(KX=2OLw4}^$-|V+xL}scCADV^8_rhTA_L1!R7w$-wH>2!tUI}rJ!Vuf?EbWIck$c% zmJ$QD+JtbBwxyvqJS`5MF1&2b4ap`7UdF#A_s=%VSIsKISZAe9*zJtcFwoyom< z9vm7hed#nw`ypEW{B5frxNc>&yq14IjR$j{_{O5`?gWc5ppDYo!kE(z;pJp4Lo)2N z+WLK{E}`;50h8+>Nh$1=MzlJcm`p*s32!)VL7;7h;*d-!z$WmLB1^RR`|^T(X>Nl)lX#vm0Qg&`EN1Uvi@X!v|xK>VuR{Z1BZ z-Fr)cLx_8QMonPS!0XunpBjZHlB?`gvzG87^aFOHO7V#(b(z_5DxnQNWKZs*w`G%6 zpz(l~$0Bv|75xQLr6Sre+M6n@Fye8o%u02qg1@}^z@gAL*bHjwidMPKe2Bapc8mZ@ z1Q-K7N7#tXJg;Obz}l-Drr^_;y=AYXt->7!Dr-2I3cLWU=8pdMUTRKrk3nRB)(rj( z@hZ2`>=k?J$=SpIsZX~b*?RChCw5y#v%fvu`N@Is;F*pxu~s}wn?NHbK7{htdC~T( zqyZwhp%N@i{Uw>B1m14b$HqBD%wD}NLC&Ke*>okjm45|J*W6E$;*$vfg>mlAen2PA z5LnAA*EI^j|^p(cug1lYS~eL!MWNz?V+Nk4)qIY;R-k~TB+ndq3PC_IIvRe}k8nY&SXwg_T!wQk^k#`;!d1B%qp8sY=-z znMv?GR%u)y3qNEbO;+%?acW5DD0IycCnh!ZDGIKxU4bb%^1nwu@fjk7rr-{gu6&(@ z@eLJbKSn`T$RhL3=_w`3(kLrI%VV=@fhRpB=mqLkD*nzhj@?=y3b)fztI6 z@?(fEP*R}P)UKGenu?zjyt`?FXTe2kBkDI;Y03(TZ@?toRDDki>ti*Jhr+_B$<{8r zzW%7AH|Rg#F2IpIUd70!@UnH)AB0-vQ^cGVIVd?@+j2_Y%l`+A2L<=p1~rFd0k7wP zn?0jz9EqknIE?BH_{h{a)rC{$4A3=i!bK>VS=-ky0iFTvA>A@yV0gwHYH#{uCi&PX z7`?<y0`#igqAw$Y8`LHxjG$12 ztv960Y~(K=?AJd8zjExPG~#r?Sq4ez&0G3Y9uC;Pl;x^$7g!;eU`Rpe8csm}H04b< zcz&iUvP|{d-<)>CQW=2x+eHNNXOorPnLe+tzfo!XnRh3?k>1BozZNLwR5rqF9f`Qw4Tx6w18TA$+5r~8rqoi|qHW_K=82J%Se?tM*2OgQU zTCdlHkDv*XoHGyAPx7xMz-&8@X*t3K?mpKOK~x=Ja>asgO4vDaz+5{G^M*dE;K@JB zo0H0xw1<7$zBS&3)TbMMn!Cyv_nBzM@Wc{?L)r4IxM`c`GpHSFzXK2Alh*CPWR+Z3 zs{;A|El{K}Bym>0B37qe;3L^IbBXssp4}XA zS5kD;m(cPN3~1M=p*FTQVlIK1gfAlCr=x?L~uKNyPx>4UMB0yQ$h0bHIBs1Pi z;L2>b*^W|6wM-N1_8Fw}rSd#Jq33+Z9p;T==u72+(YFHlb+EZ^b-H=@RMuLC!}3(7 z^pKWQX`<#?A&^mmpyL0AEnrBaVf1{DKU!mhq{_BduyUT$|G7tj&!9?w9r^umVu4$z zMt2`w5-F&|F+h*(=wfww#h;k`U@Avq`d(p>^KHveMd3C^L=A4xON4h*9MTlao9%rs z){pe1*6e-j6q>SJdC6v$)|YqqPd}dh&y?Zes;cFPb-Hu6jTO???kE5uH;}1En)-V~ zaVn?|i@$hHZ3yxN5imWWm*{o9sj%t*3gYA(8NQfS-aohpn9+(G@_QX}H3=&ZYXEm(QFL#I?yeUf zV5tC&-5<-h;H%)Fw$YwkvMj@rdW)-}U>S(4;)*y+1+cX?#Wvl*B-RO)+~a4BZJL7U zt;`7tlluo`0fmq+hZcIzL*(w=TV@lX{I$UGEaK9L1NVm|I+b32@6dIfDoll{T(sf7 ztz@h3Y8NDrKnvUOWTK%T1szV+!EtN~jb`_%kA?~2x?6=+XDUz4SPX4E%IfxX3M7+h zP@TYY()x(K*i0eu%ml6>HO{kns)AGyEo2gAmUytL-GpvZrIXwG4n0e9aX}WWNmd0G@^epDe zVJf$~;-#K;RO_cWD5%#*CZc|=>R;2rJm91E3TiO7@Y4Stl98d-?*_!tz#L2iQ(;0w zA5PPPqS{w;9!w%1EmgWy!IFLU-HbfAcbxR+duUpuRX8v2gx?>>-g<+-^d1IUa3>cY zzArdNET56|R$cVc4Gh*D`?f2WX?es;sr!{ziv4O1e?|T{`;@&(3kn2Amt{IX^0Kbd z*K|YcO7hoyRJpqSiz;Iz@T-Y(O2o)@JJmyyxo&B~RwGpv!(0cC6ZZ6WzWy~zN!s<; z;N7DwBUu5Gu3Q39z+XqJXQ8sO1lWwp$DjRuZ@D~VWH9u0NLU{a86S~E(*#utH>=`%}Nl20__~=18e)W zh0u$0h3$QLG*6fIYK|%UrHegqnwnY0XKF*k`};DrEq8U`ZG9gdx1x?2bxTjzOLv+P zSj>`I3>DR=j@*P2avobY$}r{EM}f0!3P1!0GzVwZ7A#UE%oj^*rN0HXlhLi``=`?j ztQB>9nG}_`%A|_hRKim%`3Iz-E8}X!feUO5>=dk1M`d#Lx_fORtKJEAY7FHMzl|Q^ zAVqO0rnedD7Yi+JBP+w(nCl@pWmiz88F&D?HAor?GzUfE0m$0ow-`LEEfP>4StU<; z`)2MysFE2vqu;=0@Nb?IJNkLeP8=;T15fQt_*DV~Jz5wkReIk0yY4RopB{kHrkd*d zJ2L>ua1zkLO4{0Jq>WRiIL17MsmY!LG153jIRz+>3)7R@qR8Ot%0 z_Y)Id-;si-$d5YcPq!cu`-51dG8D|4E*1rFu<&UlSI-Q?w5_@`mOi%WA*aV-MU<`x zY?PDOqusj$0fwQxpefJ@Y+OzWZ$RU4`~g)!`M0Lr!_xSS>5uMwUwM5q4TjO*M7n1Z zSL@2p)D{v_1r z@T@syBE!Xkv@>tx6s&TQ?q<}dp)`wW+LZZhO2I9OupwDeg>O~g!B_21SN&+8W@oZ@ zlqm}1m7Whn#HwT2nP);UK_b&FFr!OgKR{P(r6*nfKXE|zBrp3PZmhT5z^utQ{E`|2>q}!&oBs70ZTsv(9a0ov*1ro4oW&EDWq5&`pQ9mon z;jd!Fu27M{e!V2^wZ#CJF91Hjl3g_q(vR|@ORb5rws8r;A z_rhQ(TKBM^kdbp)M*fThk_T9F!Cgk7cWPei0$XSjrAAv%>AXiYzvr9@VkTPo?CLYk z>!P*7^?*{`qol%Xm7PJ7(*~^#ZF(G8qzJ41Hz<-)f{_&OJsydZfA`IgJbp|w>gKwK zDiqc(U=~$It9yG&rvY-Q<4=A+l}0opVnHCEGA8_d4*M>~eYta`ib!V z-!{ntr#96|t~4YcO1@ntsTGa2t(7r<@JYJH_w+vZ0ydu@D+k85y#ZQZY_Lw>&vA33 zwzUZ&wJLYCa>w+Y>y{ z#UyPRq&;HIP0`EF%66O9ZbRc#r`2yz-F@pzvOsi_c#0_Sc&npGPo)Km(UWKPiPf;P zU*hK$23?s4Ry~j|!hogHGNL!mmKHyf>?-7kq{ul!bm#~*sR<}OGmaq_lyJtoOxWen z0OBC639a3o8`_DQ)F>jd{U~w=g-^5*FidCJ@H z*%^wEtPJ1_(nDlUjhG_UE&K~T9w7&9DH3l4!0j3A%q#yW;^Mhs<@9z(L!opmxuIF) z3%7%ix<;pe`J;B9P}m*__#mE$@*QNGFy(sdB3ng}2}NTlX(M33Ud41gP)oPf1HZEo zdR|nki~Vl(^H9uhfthCqrSRQrIvTs`WJ8%q^>Y?)8XWxQPWDHkisByL?(?7t2I=m# z?LN6pOM1I7Qh;poDb7!sMLyv~XgHaxzRmIxN8!vnjmr68xZukbXzXd%YHrCtiEF?Z zuCTilb`!HTUt>1X%37Yv-I=~OJ0awGF#0QP!F_#l-fUIb{306VRqqB9a~Af}xO6)z za!+zr6u+jqpl_qOTsWX-Iybzv?ZNaK?E!pn&yP0=khJaZ&4udE79po#BKHq9B+RQn!X;~(-&RBm zE&@zRAQD9a0c7%LHNLG=?%e{=y8e{@iQQr3**&islEZJMtKvE9`E$D^tgGK}uZ`thn(McuN63JDwhJX%7y$T*l^hmECfcx`7a%Q2mt4Pctx_4b(%wj}3 z>$h7@JRt-?`jS7tq}!BF6**!h39WL%WbceUeFo_d<1Wx~i)Bw1wkKZB8-zocZGu>1 zf5;i>;f@y3j1Q`TNsr8?Ti9F>_O_n_DBgTEi@-oPdza%|BY%t`3!d*^b)cTm=(0{~ zWaxIgJfllMkik0(EMS`auw-oE5@!ThVil4y{1Rj<4D|xim!Ti=MclitPCHqLOP{;D zh7hy@e^|w$3B+`75W1C@m8V;N+uQmW0}`Ds^AyU)g~l}saQ?B#rlw{iv@4}d=Hp%r z1=BiuU}Gwa9behCP}ISHAYU6TT^VE#BM;;d(2-#bm5c{pSsiDBK_|61!}hA2RLyZT58u0(QwoF5>9_cWh-X5Mm89+B z0b*an!kHY`0juqsJS>14QI>bB7+aw_&M zAQ&>Q@?3N(C4y|U_Pnqta<#-CTkfo;%o!^Dl_+FWm_jnvmX5?}2IJe9<>jdK{ob-p zXrxakg~mN@F8r(f97{XXrt(Oy-PE~)OC%euw46cO7Na4j2*eW~?QAX=ku~PIroQ;k z+-4;!HHJ;}?)KtUD2jtM2%KcqqT19E(2Pl>EJmf|6W!8=@IcM=8_^TWK)T))S_CFl zZBhR-pp8JXD_2dWN6ig_A4vOtt2Fqts+CFA9@b`l5hDy1B0T|svQgxi9z`oP%6~WD zlkA%Rs~xVp2i#hI?Y}vPl;tNqED25b^FUG+TST=;Gt_f}Pslv}{cAwjirv-}Ddi4`X^w@?V%>+|VDh^NqQ-kS%IT{yEZ2qgK3aKlK&!6aG@O0Q< z(v#_EgT3}Zbon#=XpvA$JQCfzo88vCL4fch?)=Wdp!DhvZkg~~krK{Y9wInk|09_~ zJnpmi@cJuMZ*l&%CcI0~47YxM#w#agL2#f$F-zvXC!gaIcnSmpw_aJ*EDXtW^ zvBsEWVeMzP9FjK`xU2(rlOzW9ZpX+{8asr; z=BZgsN=9f(cg$OEm$ty=_cjo#S?B8jm8Eir7T*cNpQz#*oy3r}Z`>^&p!W>3@Vs8~ z?~5L(ZcK?qeCc}oh|GQ35=nT|A$TE1ZFQm}0;Fh((4=3%2zdrl>O+xxQLdmgiS1dz zVeqC;!_siMUNJap2jQ>E(dUFbl=3H|2d!?C_5&)x+y-|Mq*-{1DFFbO*5nI5eae}& zR9@Z3+zjx6f>5aQ*bW$2ES;=a0q73JZ%2&oG@jxKHDBP|VY~%Dmq`JQh*@rqXgM}; z2jb`%;2WMq{`RM%SL3BvHU-NQ`Ld}G>GA*7V>m8JgF3M{xyt84h#FVh`%Y)ovK7HWTY5N{0y z`Uo_-`_gSYK*NTl@CGHt`=PTtt7{2e*&B4%)#DKLa=0!=d&q;#m9q0DVg#raaq}abHdyZEiAJ=TJNQQ2j4RQ4QB$6U{->R3gAn(f=^7wBf zlLm#N9S)nx?-KUwv*%xld)1a_qW8`ERXMku1k?CmrM$oighd_`+@q(EfneVTvGR3< zA(QLNLNOU}F!x>Olryg0nx>n|&7qk7NP?DJrE6&O89bbJEi~Y1!K9dUe^G-?imw%zSVPR2HlHUE#hn3`Fj_XthAeNQxu`nmxHMLm}j-dEeDZmDy->jz}4z_(N z@YlT2p*jez?e=3}(%6`8Z|n4a7h`;iTUcFVEC;ODY&2i$etLLExH zW$7T!MoV+HlXDzr=XI^d3IKk7Op`4j0Vcm3+Ja*r4(*r~r}bG4C*L5nBOa*Wr{dv| zut>t*0D=G?y;V(p5xkBsqLFk=&sq@Sa#9YHxXG0)37!`!F$)yXpIxGVG0c?cDc&mu z8Q6qxyunJ6jk#)(1ow9Mm3{792)?gz#E1HCXd}Iu-a=LY&JdLzDq!_5E)T{vNElbA zWA7I~fjt!p;Wlf%&)W`Fl7R|J6o{qQ?GGFpxA|+1Ag?L)`gsTEmm)?$56JM{D9FSZ z*&IiQ;J;~7D(?U*@achw_zW$)qwzu-h;9gt{h6$A30t{DUrKuuGa49fubQY1q0p6S zo1Sq_)WM3RQBsk!==Nmb9)7<-ZRpsOC(K`7>7YE%rEs_?OR>;DnlAyGtF4195d~;;>sPSfZ&fRHpT=f`^C_x$F$r}=h169s`^g~Is{0a-0|F+7(Vd!XS@jAK( zq{6w(gS$IwMH5H(*(z-2YrCnIh0YOisXhUtyZ!Mh|H9mjmH{3Wg}W?U6N}wtGPz!! z7n9;NBtry1T{b%G{NQ8GbeNU*xo8}>0=SMi_FY**$A4yN;6!56A8H)qPYx5^ndICv zhXtXgk3?YuQaOHn_x*J`3dVu%jty4@_a-B&XWDkL2}zjkp92*-5`@u$cO(sqo6GDg zh<)422#vvFZI+T51r!(0fDqml|!=nh(kM#Q209 z!F{}{y4f(QW}i}FBCMDlm~u{^+{_cD#$p>VDQL&h6|G0om;-?Noa`YB^@AKw@hDr3 z_obfVr2JZ%vX4?m@}9CSiTkXLAj*I3k?57plMm3hyLwBV0;rLe2>E3LbcDjkl zqotwT>47$#K<5TeZNUXNWILW4I{aV09Z|bv^3r{12J8IU1ZKE#T`o!TF*fQn{WkTO zajL@35T`Q*I{x6hd(^WKSA))dSF~Zc$oMT)J(Pz6oArUqL{+TVpw9@}Cf&kN#$e)& z0i@cp8m38@ZHTOLUcHrS^8XFF9xz;hECZf;E1>`!U#0dmK}4sjDiUm@zKyL2l>*C$ zePycl4S%z!aAxZDO#t=FJnQ7nY!BE@<)lJx5wKF6|2-LDwFqaC8l1ffa)Q0c>rqp} z+hp~tDd(%HZ2*%BK~Ifm^#n?zZqAR0k?#**`?4YCk3sq#C-MP*N&mHgZkKhJ(YqTnq(gqiD|!o@eQ) zP$smPRd)`v*))CoaVKDs79z#<$!AQzOl&@&dfoMoRpATBw{?1y!wQk85tc=s90duZ z&Qsb;qQ7qn*w|q=>@?)ko8<0@?O#HlZzW4zt38s)vg{O-yzFebp-e~5t$)5)5&iR4 z-gS+G5bt$FK~itdT1x%$R#?@jD>(0;FSuNpiX-X?e&q2YLPuKc^T;;}!=@02+ zeXS0Wm>uYi3F@x!Hr>1q#^f3DzS8ElYb*Vj1}2aAoex8p`Eci5h{oCm?%vs+Zn?ia zN!SNjHeZcywRb1&Tq9@$Xt3PMCCJKE$8JjbZl>REO*!B{{h*?ltX8diJW?RL0#9fW z@TM;BoXn_;zzggM?%dulp7%)0-AW>iiC$Corgfo}PB~BgrIGGDy#|28=QE}`Y;@Yp zr2{I*^=LUb(*ADUwF7L{zb~#(TN&qoBOs32N1Cqn?_6qoV~TNXky=vsml8CQP8>Pi z&tNAZxQ>v0n6Xx%R^Qu4-DA-o<)y#(=|mv~?y1B*rXMts*!F8SNHA`~@%r~FMtIJ( zUWtLfz6@ma37L7Aa`7iEcy`Fk%^RNboBXFk!);yzv~%!rs;5T0>#1Xz=3Z;u)JgeH zN6B`(PE4J*qC^6N?;$FXbB*ZqKw=@$YH5@u*5Be$OkU{ z_AMRM$`V+91|~a3C@(a0oW<9>%S(qY%9-)v)JFNP{>CRRV!^~-kC0`l-vMvnwdVHH z(>p`0kVWli)TO)yTAUb$Q`*FmqTWmUTl9h&@7T%~e|l8$nm$#qsNOpmn}Ha2-)b*D zSiNN9@4iTmpPah)z?~;z+%7|Mbw7ZWh)+VJaqZ3GuT3dAAf>nzh{Jad>^Jik$-40nmMiQ?*R3TMQDhJ zzH{2#1%p!Yk+cyQpJzFFKpVdFb|M{(8$)7WL7hUOr0_O^CBz3% zB}gR6p0uOiz=`eIzN~+6t?5jlnh3+4DXi5uu9*UiD|P5oGm z^=_*GS`b6k;)^*pBoScrFnc-7=H$FYpEVdrmLVBemCq8PC8oDenCv=$)etKq+vV%T ziqp>7hK6*lvPXOA&#$bV-&*2L_u=YAF6D?PR!I&$$%w?n1m2*G-~&4?SGD5Ur;Mjf zQ-O&r&X>fFS=6(WsV>Ui6BWazWTmlX&;KKKJx7|0(GBU`FP-17?!e)bE42)saE((8#QV^*D*Dv zNftW%?T_+5v?RS1wkjj`!7tkz2D2a*)OQ|g$1gOtP9S0>m@);!FP(Ue5*mFZy@fPd ziI}uo4up6tQYe|kAsK)p!fNsTLs+TLE9=1Ku-2ZAy+Tp?>bR=JoUlB^3Ioa;`nB5r z+-Cq#ueRjy(oC<14z0QN@=pyLOG$(gAJLbI!kOk{3HRUB*QTp8d3_v0=#a@kiVZ}u zYC0?o;hCegJTIiF`pwn8_TFgX`Y+@LWm5wZJD(s>e$H!&(XUkUj=m z3r|Fi)?xWoer0m4`vQMbHDxZmb_-B9zd>ahRzL7WN6>XGS>!w%Y6;6%W7}qhV~umq zd~U;YUAA>eCWizV3Xi977op=b`H0V3_SlcK`wn zc}Q~FNnG*!1081DWncfe{6ouqjohB{8lH4N`DP(xl+p2~a@`mMTPAB_GB6Ety9@xka7fZa)3QTZw62ldhwi%c!&N~GG1hDP_d^P5 zj4(b3A#*V56o@j>hnCHjREo$jR8V-hsW#9*ORjr6#S*j0n}7d%;BSQ2XFYPtQT@UW zoCTA#sRTrs03EgAbe)kWfCpqVudPC{a%~OI8?o2??)wgKSuGMrJ@$3lm`An3@rx;+{MR_HZkE4|21G z{~7ZD=DyBcts^fD^F&A9MYbhfoWgmwmJIIswpd<<$#`A?_d6;tw4(_>Jk4%-`%xGE zB{el*5xRUF32k^%!bmzuSzTla0EAmgEV#wvDmFdBA-9{I^(RPCkGo5YI=MUwH1cU| zf{j=w^|Y<`S#>jNYYIOH(*8O5NmGAQVq?yO)ac3ge88nG?7J8uze{LR} zqR6Z$!x3=KvrIp>NAP#j?WAjTXE}8YbcoMsU{^V$WKrpP#pP=67+=kBq01AJqwxG{Y9)op{Jw%N{`7 z#)LXMTR1UqadK}#`^ktMW8L`aJ_)D8zINd#bEx4OK-rt_r1)s zmtV7O=B`MY7F7UQs6D8G%s^UVWz$n>7kP{wP3PtDJ>ITbzU;e~F5wmB<9VB3c;;XU>G^cH9kW-&F z&WBo9RJ+dgU2=#F3`#WcoYyIlG!In%MU%=hEnIVS8ZL<};HucY-kl|ZT-m2`48}HH zxu0dBG1vY_jx}(Ingq>8nW zGpbPkyadTnd|A*?BfOl-4Em@;UVPa=ttn~-*+Qxkv*f7SF{PBQw2|HD??_TVeYY!sFOKlqIX}ndY6=b)!s?KmO zJ5wRYaDe3|*rfMo+wMXC2RJaN)^ZJAm?{WjXvjT&oPKY|CiAw&@(7K}$RuIk!*SCv zeFAoSJ>MN}iX^=i?epO8OEPkzMmS%=Die0n&$Med56am_Q^e8;zLEZn`8Z5&H8p$H z_*Xz<=gCkXO9Jm6i11jX$dk$YP~guK@{{<@qlv>XFZEN?_waxi#Ok zC#=AbMwqbIt#X-kbVIc1Hy-YN}h3|YJ(t30;2dJy35fAi$HBF!tX`PbyY-Ew_Ed>_(%w&E{g zQNCASi0+|)jq!HNxrI>q9Q;RCpnEnbJ>WQ)8C|!DXi5o2u|E$%`-a7;lQr!5^~D`x zIakqM`P*f*kS#bfz$1U}9vKFnvkGE_NiSVCqozt+$7YM7)6;TVz z^?(ffHvyYIgo4`^*NC-HV}qi*pa2MHhXi4qbEVbP`w zrv1Kp^prN!ck65h0A1$zA18|vn?<+oL3WI41xo#>BaNY`YZd*Mx6}+^hYy+(yn79r z0rSYXDpKG*ZrTZ1ybIy z^?2IcFcmr=)!vVXI9cVIewJNv0Y(-4%4#=3P?nPq6wE8@Og|iu@{7I7|jM z`*%pBUWz|rdgUtLHHzMz`%Kn#e{K?OO!>ne{fy#77ay^77P6dAqYF9+OY6W78rZWX zm121Us9#%gzJKFo33qG4Zig=WWIpl7Niy;t$I{ZG-r@u5h@~du!)4kU92#m`MqvJD zVF;R{Jq9w}thMwkxl_FtMaA`#SO+!W88RX}Cp1_AP)=!Y<#Fw5&-guDtvV8Wv%1=_ zj0-D4qUg(ck~{>;`SQIzB|PE|mQ&Wj)D5AykuQr{BVKf|t zV?M=K-o|JO_}8xU&EbKf=`O+>!yZy1^+xEAq84$u=Wn4(3&U8r%ZrItqh zL^ikwgHUFCylV4K+7WlG0O9|ZMkIB>aU+QxU`9Ad3v@sN7 zH+)Tc7Q3p+VW5!n@`_3JhuzhgX%#Wco(ilwZ7s5%6C9V7!0)6?bsl2l$>|4QEyc&J zoCcM+(X$e}<%=s3M8qg5dy}oJGkwC5Zb(It&mydIkyv+($~k_ux1ou16L9?(@m^{o zz^6Eq)d02sPw(iYz_8cLjegiFB;LE*pua$v)oA33C+b?iiB!DAQ`Mo(-zmb4J*nMD zJ^Y*&ZoeLbiRB!Z8knTs-Cp~B;H{q0-Sun0zwz;i(cVJY${U;wMNW9Wo7(_bXQ7k2 zzEa57y=iyJWcPVbRxyi`d4>5)GQKVa$&c}g?V*$&f_b*?{z`9qL~CT6>ZPgX#m*1R zA^NZ*_k8Qnl6qXub~#O(eojSi08aKN@wi=}yBiNi(#V^F&-{bHDw}Th7}0h#ekR|sf{|0% zTwFtieSTXoINK1xFF8r~7a`R%(|Rmm70_LIuUW7*fNbf3mhydN0n5%ek(z+UD@^ zI@@`4D}}c(K8UTnfLM=BqPo7D+adCJnqPPcYer7UM_Iz-F=+$An_m~by? z3j;EY1hY&CK;UHvAzIwhL2==oLySWBC?X4CaeNRqPwxb7S23o^kvjv`9)u*#2)j%6 zYT*|vX06TsSLSOoc+!kK<^CNC8o^`aaM9kujAyd|KS030K6Iz$mYn8t4aZAtj|6)o z&mLC0p~<||B*R@doeMC>wYM%15s-M! zmATrejV$4B=G@`Vs~*VN7g_No=vt7?}}UlI%1>wsI66jb`BXU-byQIra^^WqyA7ohTGk{S&*| zWj?gGU#iQ}5XE+YAEk#o#2bXI9&u@nU{io2Olfoy)19+DRCbMYkw+{*O=Pz$~YcXWF%$y`n~JATfx@u z7auY1Z6#r9pVqNEvOIn1mg%=K6BD%Lst0t6C7$veuqcf%2<8A9!Ew_Ki8Sd1|66m) zHHLmL;>PBR)(gVf@UB|=j1E95Ob1XUIvVkZ0Zt3CTEUZwN_0Q26crYGdGL@IxxuZz zif~e%E2wExk;?u$Z;hC-I~ zgo#@JuL-w)iZoyA)zF}NQR1?bR&eX)+Wk5;rA{tGZklCc#r3k8$cl=p<&kq{Q~}Rx z?a{&K1hL-n(&5Y3+|T++`YqLwG!1?3nx+l(zu-5q04k$|yX7Y0w48Ly@c}L^D$in# z!A}+BG|J!FZ3PvM7see#Bmo9DXs+I!I-3k~0>K>+NWMwz(~vvIyxp}RYy_~o3gK@`-?#?nYoAvj5Tc** zqOeCFz{$(1Rv;H7qgQAM{d=iGC!7ZWXa#`C&nVOKYBs+}(&jynnc;vQ;!!^GfW zNOGT*GlHasWrfRlci><)?TDas}A-IMS=28zuyjUVl_BNngdm z%Fzd&r1aicnZd|-R(1wy!VWGr-KeOUTKgn{Tlh}U6$F>aI(65IAzf+I_!T-6^37v| z6Ur-=CNJRby)R8Bk>=@$t%weR!YaJwDV6vla)+*qj5+rEZaPGp+DO7A5CosSfiyQK3g{# zYYS)41#^$@is4KB4Fc>V-RECtSDT}ynX2jI16jzL*lCm7o}+Rjz$s3opujD z3wT!gCx4k{sL!AUWa|#Snd63D12DFljsrI@Q6W72j`UZXi+BTgb17?W0m#!HXb+S{ z$Dyn%CR#gq8e|*ZBcEH-n`^+r)^p{=L`F3g9DF8IYIYplV|+*NNU&W)1GJan?$P%=u?s95NDdw|n=CmDxamZRO#&at8j zxY3s~J!5o5>lspERNbW2Bl?s>31s4f=wiw)92%FD4pPSMD8q#W-jL|RTPQ{%*oaR> z{L#CC00$SxPLW53!ozGPlCXsppx}h13|Wqulo7*`dRrPIxZh;T;3Fvun{C9T&1m-v zK@Hp(38oMxZ-;Ps)0Yx~kElC3+JgAop*aV4L;)+bf~_z?fUhs}_xUx-H@FQWksC67 zvjFNrMvv=cdAep8>m`hL=y%=R;5%`HXDnJXZ-=2!VVq~=NcSx6gXoULIka-_vHJYA zKAmZBSVP>GA4{XyH9rhQaKd6dfc@4_ysZ$+0jEckva@=ZPY6P1#TD9}?`VW#o@YZv zYI6wtHvs8eoKfZgBQ2>2eaK!9{hv%z-*u$wQc)uV>br0HMRZda(QJP>2OP|MI8-)s zY26U3BO;K>>7cv$>ZBW0XdQd1#tSbBR%U3m!pC2(Vm}3XfdBQ4!<^OXE9tkE@$K)A zbT6+I)VN@x_53bNPpWPgYu%Sn55`sD2ib+32iZO`%e~X~&Gk$JOb<0lKBZOCRiLAV zy*oKs=LkX&338hb(3M=C3Kz%}2%-Wx zeFwwTyGS&Jg{ic3wUx^T(ZFmJ`ES+$qz$~hLx}Q;AA5p+hgG0W2 zVH8gBTh^+`7ynwyOj^jO3jg8PI~rfa6&6Rq;mcNDdNc1LetkIk)CnN18F+x7VKmlj|5?9`~ZvW z`V%G9!obWug)qMzr@yK7*Cm0dpVuqd6GPi0Ef7ufRu4+!BF9X zmlJy1%I>Npf=<)%w4SH?4TeU%c_ZMK zn0)j)@Jzz>l~yVM7DHeV*?`dWn~4r~L!p2IPCJYgXECw|AL&9dxyGT zb(;ql_S7t4UAb5;jUK5IpSD+RUPJmFtF?{{PV0*)L5n)osb$g@)Ra^{PN{zlC}FrM zXnKJC!+vJu-M4Vp-Z&ZEKSuXTw<;GZO&nms&Oe!mlN3jI7M>;s^b}iOX)SVE)#KQ| zy^`Jz?cc6TF-($u_^!PRHC+b5C5-nVqUyV<=rZTD-qxGqx|^uhD$ zZhA&NsA2T)cU`DnZeCgPy3C4}UuF{mnd41WK{be8Z*7*&3T^Bo-%qeMzqN!Bj#LR` zyjOK?ht_i?z!S?`PQ%g-@@s}-k?w*d*n^I6_I3>|Qw%|ycSuiGNh3D=z65&Va`I?*n zo=`bGD@a=M#_#hWvdzjxyaS@`W=r3Bj|J&Mt#$pTap>dS-Emf1jl?3p& zO^;0Qv7{(a+$5$Y4?d%+Xt8jI8LCp5bSY9F37Fiya|Q}gqZ`F>z~O5zEwdR|oPVFB zPl{sRStjRj>CRO%ugqGKmc4$N%c6Qd{UYtIAH%U^W`yp@ChoWk4 z5g{EbDIN!ykxPXzYDG4T)yXvV@)WVI=o8xs28Pg=(5(xP$$I#vx{sChT}?*kVSQ~1 z0or*+01Z7;=Y>Q58+fLN5h5x%`upZ)E6OKES73g5DdP*|BuACBXUP4)uRdbc9=nOi z$|rZ``G}~x1K8$V(N{KMueM2eZ+ZJl@BS>&Q~*QgM!4N7>uS^XmL~|m;`)+NN5_aCJeY|!?ff3@?#A5|EJYc!L+i$5$y#ODs$8@nb(q!dPRU@qg?oXky zmSZ}l-Z4=GMJZy6H7nZw3@zbJ#Wkt8hxLe>7XWoe0cDfz9BQXUxL=4xd!}Bj%f&(c zxWOVDB~gqr==p6CHZ$#?if60cLdV z#5sJKUx*Ja15r2sC!$jqk5QyR&Ab0As7LSz%q%!%1GQV2yeqe3Q{O>)OFHYG35qWp zgaP;d9P+vAw#FQhV;hqTZ6v@`k1e@@2mfSzch+H=>d|Ewx;>8Wdz26^OB01sCI9jZNhaa^l( zi=5C*Yj+$XXBT@|TpMJHsMLywH$10W%)J)U@3jTcNZ(?K3*hC9mP=Ik}251ZReq4U|iGF>{C{j5vwNxIKg(WGEXA zCE*CzTxW2b5zMtsj~M({+Svn`YH*o46BetD01d@iBV~$=7>wOgu3&*UGg%WZ $~ zMoAi>?sq_b?5uGERJGbnU$9TJ9#=Q{1gwos6RF`51`rG{LP0WSeRP3l3XkZNO@kR( zVEYXav#F{PbAV7^Q!`8!1{Qc!*luC~Zksf;Fk)7m5$2Wg!#c*m-QQ^Rz}zn|jGHkK z^wYF9wP8y|YBnR>nL}kn5 zEWWb@x>-B3dYS5bTDxMAO?HS>Imt@gd(Udp3PcMjR=s$ zg{e}S)3w>!wUn=^1^ zLJ#*$rU+gk7U+Y=r`%wLA?9=8)ZLgPQ6RP%w#6?b`};S4+?%LXF*X>lX;NGe)f>Fc zg`gielAk|KYd$9lidCtu&TTnwaem@hw+BBT-(rF$*x9mH9Q^R0>t!E)3hWnyl`{1v zrTF%zCGLLGsC0phC~rLpd(h5dRXFEI&m^Kd(x}qGM)s&9GeK_DU(vwHm3Z$#j2YHE z=SCQ`o^?#NLm2SmDH`l&uCf_{y!2{!h#Og=OvufGkD&+_A$dTrH!&y>EQ^jk7}2xZ z$voP0%VZZnNx6G4Dc^>z`eptVaK52ei(~%1Qzpfn*nkMa_4#SO|G5R+Ehg%~&Cj87 ze+3oom$tgz*vEeZ{bSh6>t2Fq z$G85A|*wzh;_MT;KuowV4lUiiPN#u4kGqe-XC+5e;!Od#pt+dT0cG4y{ZqZ+ zD~qaZ8DCmuQm(>CrmhG6iK7c+Y<>-v0{y38{=}JNrRJ|iwP}GBA1#y`JFk8e(id8w z&YGcO&e^j=h-OSdF42IDsj5KCTwf_rn(5wDCpVUzvbE4e?eHCyZ+1Dz7Ntx(YgbRH zxb1F#Co=(}O@~6~axLW^4gBlGqhwOcF4~b2<$$Tb&s2YKdG*6^@5%P&e>gSvJ*P5> zq$E4mfvC>%qx_m8IL54* zYxIF{h(Z2u*(KKm3S`^vP^_u#%%zI2&37Zt>gzNm#|B3#KS^!QIv^bB< zL6qthdqN}4gEz&zg23IXyVaofsJ9W~lMNWY?~m_g^UkGmrX-}R7w>c#IqqDXLGaai z?!{fP?>96$-y)U0YKeDi4dLlbK+ZIX(Qn=du~%c^^8Ht1FpBK}2v(tX0;NIh0=<&* zEuvO-dK}H!*RQgMTwnc^aWd^t*by=p`IkVmxa=8$=M8ssd3`4{nx0Qv+sLuVQ!zym zuUdr(KiZD}SXFN9dZ-C}qn=Be z32OnU5Eq?80gR^~I!n)FxAK`G-$;Zl zrvC;ekJ?Si+1m{CVgR4A-)7703vk31rCJ}liZZ}ncu*ll2OPaym}VB#T2r+ zRP23GQ0%y$qRtioEdnFOh1#r_U-__Ui+8715&ZAoboG9{KFYJ5IsFgB{?Tq1)%7%j z>hNQ@lfrU!kEc4J zA>~|S(M+kjt6T73{{EUmthILDqs03xPGCs&9Ik-QM_%#i05*o|B9TZ9Tu{1*>L?nn z*G)t}sEY7`dlE1MUd%za0`}TnE_5mUo)#h%=iK8S@=)tLvMu!+Op_K^niAdWp(T!F z^F7=qzkgRfK2}U7Uw(V(`yy#T1Ef4$3Dnj7G#xKXMP?7L=gc=%x8m!~1i-R;90^C& zPpgP2-$k!c@Dhw*Z42$_q?a&JzBh%ji}1&7mOLQA1D*wW+}h}!D5o;d;s!iUi`C@4 z2N6iz7NA>$#DXUuAyPb`2y+gy1nT4eN<9}rA$U(sdV~cc_E#JV!#GHwmeddEgDML` zmA}xJv9luD+R4Bz?l5H;bi}TtVbM4n_KWW6@?8MzBAT(f@<5bvN{72^_sd;8D}cg? z%hLqkft_gn@rYy*3G3S>kOQ*jdF6cmG6dt(Q7;-0>D|-lW{>5E71ZvT5|-ia$SS_X zCN=sqP-N#&5%G+#kj1Da<$g+ZNA@~6mnO!s9K=2kwj$Q`(_n=Nv{AaYg5c+=J05P8 zsNk3cykHo^saCx>-5~!s+z=vu+oU~CFvbL2!Vhx~QHVEFYfb;ZtOraLNQ_}YS_~L@ z8`^`RKmLzB)_n@j8yBP&AKJaR#sHJ=RR9#&Z8xcG23NAGoQhFqiZAGSK0BM#EfKNE zxnBzvOAvR_;UP3X7%NfasPEwoC^N^uI~~5dUTW;ykId3xZbeGj!d1hvrvhxQ+%x) z)GT$>1kjtlPn@~pq3r*yDy3qx|@6`8UacM3c#&9Il&f|(=`O=R;?Pes5a%B ztf1E$4}_3#r&)K(=zZjdB;d5+>*?-|t)GWb{BYN-^)&z*u5QQN6Sx>odiiataraA> zdR}{)>dW#6L@1N~p+;-NL(){h9dn+`CDu?X6+tbDB`D;GLh5@XiyyRgpz=Zk=QAU@ zqJI)9P+s%RW)6ExEbMYwOh;O?%`?>b!X#08c~RQ=$_BF=RWo5IB6pX0wT%gSL`j+z6cz+m0Sih{pr4siBOVceEV^{7c)$=6UBZvr} zJ(CH;wyH%j4k3^DMwIKfy@?6GXHL6dnabs0N5P~+$Hr_D7?X-8j0%jDp@gWsry_1G z#TkryNi-8~*eX@Ux=!lxe%_h&p$FSV>ZD9rqlN&0k*fGbrc_129*OcNG)R*fvlGI6OhFE}1& zl0K9}tbosy`EA9S_Ko?-d38aFo(eg{XRRen4>j3nrY!~^56j78QHkf?2DQb!7sjK>e|pfvFf7MuG}=G%Xej^J|QQ1YM<2Tjo*@!nXH`_jwi)_p7eDps&T};ty6T=V9 zLAKcW{2%qcX7b`D-yI;#tUjtILy71Bt-eiX=qYs~odcKZC{=z3CBDg}Cdm|al0e*Z z*n|kb-(PVSyxm${_L z9|8@pgw(jU%@G#*dO{$VY4VUdQf&~cXVqY36L&9S-jVqGg_=6w;wS@NEk!kRK=_T` zMW(@$y8Fz}2~gsGQJXchutou+Bs-iM3;u&B%X;q5ca0+_ll0-?G{1MJ0&_UcZCn>e zh30n`l`;tCk-u7OWahhfuMQx3lyI|xY5u9Z<1;g1Z=JAq$QJ;Pe*&CbtBq~;=-cfb z7u5a2M7G=*0swEz$W)V*>La@ zMz{$Uw4Z!>=Rv@j^*@h*m*WOL|_n%eXa z9mRMxQI|}VBfP%^X(Fw^Cd`4BAHH~T!oez=>UIRBA9w(Uh>JV=%UHC zGq~`ues$|72LY~1V=H6)s7-K-`GkMlJmVmWNmc~36DS@WpIM7fkhM{paWKg>pJ*HA zQ;W6j$@Qsc141}4a|;&9HMa5H%f*25OM03-fM~3%$BFACw8wx<`|K>$tT|dO**M2( z#s1VOBB!(Xd}P9tDjPBUQ6l%Fw3-DoDpt~aAn58k5rK>T1`55#nkr_i5sLrE5d) zvYssv42@=bS#<(vdSsBTGb5>WV!{D6WWu47#@E-Uf-J^!Pq?&71Al46bS3O(J)nOb zppq7Q!vGmQZ77hfHyzYWVW>-!O0fOgG3`qp6{qC?bD1aE44#nmcfuvnCyn)M;Hk)) zk6~cpzB@cNyVY>+lr_&gXH4#dPFY?|WT#}fi7#ZL%Fm@haUQl3&#Nt;%}7M@BN60~~A*Ykh^zZEP{x03fyLSM0!NG!^yO-z!Ws!RE}0?gTeNKpCi$Q{I>BWV@Yj*ZtXCuE3E9UyZM#A%kTE=yRt5#MtyWOBWjcjkLWw= zciw_pEyQk;5E`c}>)3e}Pdhe6(2A@4N8Fc2b&%hmP7$|vBb;6{(d$F@x*UhO%!i4L zm?sjLR8Zhu0Oj*@}N?k5ITq_g?`s+9FawXn1 zQef(N4gX~kG;%jfXRT9WVG*{^EC8BT_xaBh%X3Y!FO{|1qd$BSBdS*Mh{PZg4%>U$ z59sDGp2pgXoSD=<%bb`geX=9}q4R+yfW)n9P}5v*7~E~_HPda=iN|$IKxfwOs{~Dv zJh6eqqy6U^8po~)z1d#T9DN=JB8n2J#!ox>=+&-)lEL%1K{a{tT^P>pMt>9x;+%xw zI^>$RD(w;gJ31_gDeoxE!K)#DeGVayI`QLV2+W26mxanS@l8KHoE=C-{2GT4!jMwP zNNSI_84&hcTH70q(**)khUWg!ictmKyiDLLg`Y~~`d}WS?*us^jbCR>vH7?2YBC61 zF6GGn676jvv)ziL{`SPOtbB#NI6bk>s;-M95@k&-+E0F#qtJRVPhP>tg>z}rhVixS zS8H|Oga0b3kUbVDDP52yj_x@$x+!Xf%(B?Fr})&9&=&Uy2{GL1>T%p5%7)AIrD9@< z^~u)~?o!EcRt?RpfylAXlTfnFL&&WvV+R6v(j?6vM$yYhO|<@nIKM-3jlIRWQ36aM z;;Gi2=t(wNCy3M-Oo6NqgRvj=<1ya99hGtj1;Jc4m2BCfAXUeayI~;Ei_^GO$bx%T zu;s}Xo@Ay!C?>cKl>kFyX+V89i|)U~@R}?6UcpAf!95|x^k5r4u;E zB|7_tA`NGW!CrI!KIy&iTS9-SEA!gD4WbqfL$L+K**#N&K~4yf9fP{$44ZEZ+s{J1 z{#oE9^lLkWGFN_ z5KBbY#V4XGz<+uDQeoT^vhL3D9M#E9Cp>lf?+~^5(uU(+XYri# zwd-$O>oTvBxr$A}f&0vnu-d072uVISRQk-IW(Vubr@63h=_0rBTH{|GfI$X>*`v?* zszNlMEN%jZhJD8gb^acXzY0%aPr+iug(WV@#W5>P&4LG$gm3E@d^e_$RS5Y!SD49M zruW*fQJ-l(IP{E*zqOOf@u;?r?GJ^%hxWHMyW7vIv>w*FDJNejx1>Z)@gUV8KrGgx z#*+-N8{o;od7GR@7}geKW!s#4;YE?E*Fh8DgaXf(`$7p}P%JxST#GxFK$Pl1*D`aU za&c?6bm2i2tPM7v@L&TSL$6CDV64F{>Xmcel+O7`Y^?IrT#^uyILuKHF{7-mER}-> zAjz+Am1j`U#N)vP8pcbz|8WeAHu#!MD67cH{)N4u7}trs8zdahB+wI`@d;JSNp|Kn zyAfhYH2{E#KB22+MHc&aVEfhBLrDvnDD@>gQ3N>;8ZcbgyW0t_7FYZzT_YNKnuf7F z*L|e~C`n7j72)AJH@;t2;j{pmFqkCUugoVg%D+f=ldPjX75>q| zoHBA+-TfzvX|=wv)B~f~JvlaM3VsRWnPJlNe+u7la@tkp@u_uncfZ?ksZ*Y? z78A}Qp{_U$MJx$%J5sAOiYj7!7SdAzKXZqe#*-*QCl!n=d-z6$pi)I|3zi0&GWl^X zH2dnt9^+mV6^Rs_Vslp0;1;$%`P~wvlNoqw?j+}P{vGWXWG2LidFLK=wQ4#8P+*M` z@-05!mtwss$C|-nqDY4@Rj{hOx*krt(E6Gym5nsfq%x*ZtGE5uU_Z32E+r!X$7(9Y zckSwV^IJz#Qk(%rT8=n*+ZP0KT%({w!bG0ZB&YKJGuI$)3K6lmiP<&D{?)y(l_(>RoF{((xgzAC za7Eh&#(RTYh^z`1?f{q@xEPA&PX8aL$)QYgMp`>)8PV__*mRt;9~mm&#RbeW+7XJr zS}!(m1B~m%1XAw8{AlZ>^AhIpJ1jH89YFh7{1CFt?RKlG?iad$6ycqNIq1dBp%b%n zX+c~4v>MASA}(q)1N+i?gX}NmT%+17D6zB$ut6H{Fizls{tZ^~qkJgZ5BGemUo#+` zphV$h3ld(b;ax8ScoD<{ynHg$uz>W2an1o(^BP|09rQXf@ge$!fUx zf;hj(QAoC*L>PP9g8MUO1A*7R35D;6J>!ZbQtI4L;pXeIm?8?>ZgE>`3={U0o5dRC zCx>9!`F-^OEiRe#I32F5KlgXwp%yKC^(>N5K=Ooqu)egt$S6{zO0;L*RD1u#@y+MY zlP(w|7T8WS={A{h-C-ZC3#c==kw)|gsjq0@%Bb?>&QedDfdpV>?PGk0WJ-HCdquTU z#Xn|+>`;WJB;a2gJOC+Md8HWy=LRVV(SpUDc?iQ5dC@I)$Y83XLP3aZavC>k{b8Pe zDt=RIVW%$QX5;!xTA>5=(ueDM=@-9HO;2>2oD*zqc8SB*m~~n#JTGF}Uh~VA;EIZ8 zRiyZ%&H2oRr0SS)SiEr(F*Q8-CsdNjK>#igSIqZX_Uz9?pOroUrl9s{iNDUF6vp*5 zC(&8j(L3Y(<`CF~jH3LpC2C-MoRgwh2v-^pJ*{;PHPbxIaHiH^53gVmpgFKEpk4J7~#Dw9UQ3MI{L zG12Njz*DSFlW2Zd(1R}=<0TJ-(JP}_B44YV;$A-Jn5ZrGkri(+>b)iG7+Tg0%t8m1{kFPDyb3X-i&4xY+&t-eD?;*%T*Dg?Y!|{hjy7;! zQ~rh7qa89U4ryAB-+lOb`tj}jIVa0*)$>`}+n7$3RuDBQXnAh%Cbs}rqZDviAqq{S ze@Dwvr?;@frMvVByoU_Ef?uGQT58ULRwJ$1lV{3w?5Is zU_4^|T#ZOW;s^fU=v*!+m>$stmA^sW$BG^3!2jyrmWOK8k~rNL(3v;hI zB1`?3EL~F43kal%so_5@=qlwjspzCpQ?C0^{mhRjD&q9J2)oL~U6;_-Qh|041u-r= zubt4O$E1DAtKOq2cI|Uetr8RlAjRz&4_*Wp4;Ru=71q)m+|xXVq}##L0=cfkpEh3^ zw{T{Ljpz=koJGV8;gc$nK_x9Z;LOpMkd#m}*B0|lJ^m3qeS4^xQb$69YDSHuIZtk5 zP49Hj942I(7vg=U+#+>2Crr1jEfZQKdFq9=Z@TVK&dpr+ACXqE<&OQDY5T|t$b=z< zW8_`q&!I)c1t$tKqFGJ=53*(9tV|GFRjW0l*r|OAU*z#2GwU2f2V(Vl5UL4hiv zE8EQkrEi8~A~VT#YoJOV)J&*=imiTtpSDQ8>2mg@+`1xKv&Q~>io6{3UC`k>nK$ng zo=+S@Ig=v$OYoBnOUb_IR(?6hN+{+4>%cBZt8T3HFrSsN9e({$X5+}n;o$G0O9fFz zze8Zj5DtYNpnPqMqZ@R>*NN&kPX++}T^jtGrd?s09#lNI)Ls7|bWsAg&Ruhu9675G zu5V8+j;4UFjjVp7f9*ZLP}V=guN&F<)oL<8AGG!+F{={mDm~v`)@2Nu|7zbCF6rXD5>-1y>gbbu&R@8; z^7KwsT2ilCdq2Mn;H?wdXYd7e)ry&%F`W7*aa(*z?uEP2@YWZ3Ol@u(9Ts=+#!=ufW)3H}(A5wz&FiT%fgzU@ff0smItIy+%2@Q)u`Q);jlZYA z>5$9mAeMA@;Y@Jra(_Sp!z<{cejn>Ed3rp(c&^hU05%05ehcI+`uQQ}ypFSM(r(+~ zri3`0x?S3vUwW4xS14{?7B3%7!?}0MifSRRB8~+Za#NUFwdrj;t`mmw?Y_U1QtQ1T z--Z8Gzvw}Kycce5^L=w7np&V2XC8d$B>c=p>g9?*sW+ij7zsTP0gv=|y?n?JEIypv z#rWP`s57RmpRAf%aeqT%z3{6))k#{v4-#UWZg1;Z-!B;BHA!67tp-p($kRhOu2Zt71Y{|B~80$hrx~6sdWz86bQ(?0#lwJ8b(@ok{B!Ss9`OF`X%=Qj)ZZFs4D> zkV(#g822Q{0Z}#cf)GMZZQNkR?9cFRpSi}UkBcu=-B6%@s;h}Pzc-AT7Huoy^%V}T zOW!Zr$lKtI5-kvfZSz6z$Qv{*rR1pHW$*62cT)gV9woXtQ_~bNn$VSoLLfC{Qn3iurJo4G+uqeN z>a~b1fb8gGvL=&GKMGn*&pzG?UCVj)oq5mxju~_Dw9Wj~Te;Y>4 zJn|OcV=Y8cw$?D)lZ=j_xB^Yk)z*YUUSql>CAI~Zf9xQcgj2f2k(zY=;6pZvM0Rj;$=TnneesXbw zHrl?tu8d2fI|{ctEbuJh?p&-rbijd0O;l}~YR-r7fDy;2O2BhmkZljrVoUDRge!SW z-fv2a3a1h5B$0$31WDilC2}V+*7xjGa&bOgrMPtgq89S!3Z69)EB0igy@MlNwWf(Qz2eT@Kj#*fXCy}l94^-S_>tpfTHDC9G~bxPc@PG2EEiLWInr7 z!oQ{WDsN^!z@xht$LOo8b~es(Zgh$UBP_0~H*4(=RGTKuD*U_FAnG%jWDdV904a~T zHFYEX4+_b?uZ3~tB!CiXP~Nt-_xJy)s*Yk{||;N(h;P=lPY*Tv_rCJvJXIj$-`&`HA9hJ0hNaU|5|tU_9;boW{q@`{jkw?L@2DMx&Ru-_O7b zykVP?a5`p6i`)kCO@k^h18u7LKv;hE|99I=z-@o&uHv>ho8A<0WL}B&(YN3EvGzmF zhvi~|R8wO5TZpCNeCAAaFQ010A+l`?Y*@I%f#V90?wJll~@dS7kXk1ej$(LwUQGv*-U^Lajj`B*Jy_53IJ{}!tOAQ*iV!#6hxx;Y!Vb78>VdESDGAZ+eXRsT<@s}|~P^xvQ@SCge_(O|X0l8kmmdGt3=;})DESqkkdh}%|A|2YWwt-G=; z4JvxDbm$^)Vy8RiIN~E-ngJ&Wyq03s$4_;DV<$mrY3rbpRkz=`>}hZ=@S44EG` z(J=~#r`B-y52zkA6CZbC;%YRi=7fpFh%uhz(T8_)L|zvCsb-}tQwb6oWM5#= z8B#UdEge8G(Ep4iFVzREEtM+mao7e|g+bliOETiwUhC2%yzA$VJ}}78ZJf=EZhNNKB~c?ZeYxeY>Sh@_UsVHC8P>tvgF3M zy2jJ}883@lRjk5uQYyq8mJJ0hUL(z4t}(hBra{AZb73ZM4b^}@OzL1StCb`E+p*J*`AK~s|F#cx550AkrJOv$r>8%0Sn>GGM`?8!p0$VhPR`7+gmhEJE>vbD?tN z`+4H`^PV0&75ZE&v}T8NwPb^mHHplz*L)x?JN+{Ca;wMl|2pZXEu6G0My<%OUb$Ov z=}#3;K`UPhwvsLsk^BCy)z^9j4m{$tWYtymBFnN*VNhs?2b`YGW|4md!VOzUxOpHw zz;+gygHj4+@5{K2w@Tel4kb~rk+PUu`b~QCSg!E1ts5OVMVT0bKBSvvIn0@$#hQ-+ z8vob4>AmNa6j|$PVf#q|Qt&pL?+2#5%%|xS2u|44dx)3`8jOqybJG^fENWmR{cYi$ z@)!fp-5;=yZezBGlDQWYo|CS9QHNvlGYXR$!;9!^eKFi(-Win?5Bx0+wAl0WEsM*+ zM2E0|*yJJ4z!PvVOuCb&zEJDl*&P0o*R7?6el4J3;^9NTe%ro4dP_PK>cGE^Ck6_Z zE1ZFx1@AG>Pd2w%g#D^E0|VbU*niBN!^RKXa1$2T%Ls(5uB2JU zWDSkg7Tg^g@o^~dk32;wPD!)7m&@A__adz`a-wiJx!5Xl2_=1%k{TT0kPhN|WRkhv zkFUkY7}8w`AX7M&*tgx`pxdxm__)H$VmXXwz#NfQEFagR`|VzDTs<9Z1h%tE?5coy+6&x$uq0)90j| z`9|-g`Osj##rGrvvEweK!m z?%_@k_;yu-CLT;cxtZj@(}uM2@%a+$U(56BY)_hdM>zJ~9#Jmy+J;(1`o|EdD$A=4 zStdGATCjFzfT{D!ClaEuFVzTgx3P*lesYM?80?Y*HeK1u|Gl6K=rB7T3Cx!Yh;{g& zw`lzn+dn(SqTw8F@0gSH^Y3%uSbUiS&_yKrI{Yp!~F9*O;1Uwxxv!5a>H=8-?>eMuzAJyK3_TGLp(>~N{Mga zp3Ij~pPG`iyVTlJN<#xFx8uzt+>XtOl4n;LT8ADodf|vF^bzgbmN-^&@o4ls_q;BF zrJSrXOTPLx1KeXy>vwpQO|rgf$q-C-3NXY$R|BTq514klp%B#C5mD@YN7R055NIA3 zr|w8}?ayJH`3iZxncUD?N3QMs%!FYbSdT~Y-lE9Qq9R=_Ogat6lt}zWiC_;S-nN1b z_eLJaMt}S__I;8ujE|v|zJ&P&-&Q?>8zyD%0fhktp{EX&VYk(Vxm6)Qiwo1)?3FgIiNOGJir47#m`PZ=MHM04~OwGrQI+^O#yCL7Jcj?!R*_#f-gyl@jz^)&~`Py9ZBy7C@&yj6D}np#kMD03Gr< zk4q+Wc_){T%ur8G>l01!^B)&q8;v0Vnezl6m6LxWYDkJ3TG}#Nkgao48Qd<^*P95( ztJzu<78ruMl7?^n`r(Ni7tfDDVtfeN&)i;jvbr^Gz~Y+I@0daYCXU+m{$xcQF-5<@ zCM}QN-;3!)Zb-27fzoBNlXpc{5APB23$M*-3UuJb9@n8g<>x*ocn81cGT+~ zmQ7GTz#~CZaOyw{`we3B`~3Y;=cH-YL@&9fh1+qRkyV~$SB1iZ%%3c#0ShV;aK|+f zL@+~U^$eA#`y3UKgT;VBtCCvDnbhu@YmgU@Mw0aTw25RL~ z<=TOC9)wP2a@M^bhi}D?j$%(yl2lP`EErMs~(7vw)D1R{pohw6s>S1=;9)o-Zh!tL%OgBIhsJs(QaG`Ik|T3n#>1eg?nC1f+GC~;6p zpoIEOs3MlVrhW>?fjJ6TWp3SO+K%PX1z}lIoLF=|a5l=_cQ<>Il!8M}p8iiVq#}R@ zo(Vx8%!^82Wpr6uHzQ0ml5)af`;(lRDP_pN`p%Y6T{X6)nn9r=i`l?>D4K)Rc&(tG zHSXdP)N^HMZdp^G8!DJy6~w_;C@vcY=2(ZCz%GIE8CUF$Ia`))CX7(rlBGT0(4H9A z47Op*YV+xWh_%MN=qwoO4->(+hkS$*S*nae5cv`)gf}wy^dn+u8>s+i+8RYeX3w$T z|L)ist~OKjXVV?6qj7s6jM)1EOF?{`gkDFa0EO3JPIKqec_SMvV~6@$@}xKgE{zK+ z_)!4qwmi+%9lHd%59PMQBejq4M}cCM4$2ct$-O9K=YvB+eEk-Ivf5BWYn{lk;TwHX z9sQI%^11t4iBGr{@yDyf3FZF}n@3keH$DbBk1)hNWq=#aDLVM62G#x;PH-?F>9%}foc+#Qk~aF7tdX>F zaSDN7so;(q%)R!b1^cZk`#PnfLcM6WAxH zsy%u^R;9XHN`D`DnOlQ5wn&HqQscmB6hv|<=m2=z+4xdv=5*1{t8cfa9uMJn!s}^j zGi)GpYKG)?#jurY_!R1JNTBDO#7B`VX5DO2FOn+%&t6u09axD{s>0G_bR^E^s_E-dV;i0j zPo1xB883;%?`-jOQXL;sQK<`cV$Qou;;vnNT_OG+bY;_#QYHdHC<2Dzpf#CSB{{K= z=PwiER1U2wjx?>lUQh$!%IUW>5HREx^$pf=a9(7F45Hr*?l>hR-T@d-~^*AHU0ykl-PAXN0g2r|vj;F9o^~|gB zj}RSR{4@oJomgmLt*2X|Q?s0Mtx28r1s(P~4Fmd(X0GCkneTb#4JwzRC+9o|VbNbi zy&!R@;bscq#el(MuhutA-Kk$@uq{(th@lg(Eyf5@@ZY4TOMv90L8N;N0m_K=6HVn; z=qAc{2)Mk}n#ydvjFelF7TQ)yzKvvXY-<-)M>zQSagVa`@6AYVC&tNwFj%!y-;bSB z&kYBdr*bCn@p=t9{AemA&C2-;tgYU+gM&pUgiLI-p<-?6oeBY-65SRgS6I1$d)dJR zRCrM7Ttxq!u7G+KqXrS#knzzXcgr$U+tbWnS;+tlrrUMh_qMJ16qRYas0#;~`i&$A zSO{2l@2e_E)|hgea?|P~0=6zXDW4mISCc(Ro0hx}3GW z8GR2M-QL+ZF8;DN^24&b%IA?GYM-G0=>`;g?&{%WgHvqKafq=F{T@O<0WSjemZ6k@ zi}>gzl^T~(mEwi8yyW$*hFpR;)0m$Ad>ndmqc1GzC0xjK;IudYJ~+N@g|1Wp-J`F# zX2Q5Z1Nkt&n)i+ocp6!LxF}uOgtLS(>&J|;gJ?z+>Oa7udw_4V3kO~csd?iOvTUME zsZxv)^gkb5Gv}ys9SKY|k_$?f{0PlI!2N2ZWo1rzU5Os2|9}dJnj@P?uQMo44P8r= zVZ2t^vTCNu=npnT(35BMg^h1o!z+%^tE;*BKuX;=Ch&jYA7)MNn}ejCFM_xoof zu|KOGj=fncY%{Gp9W#i~t8v6~YEGJ&qT^3oipX_TtJVqfnSo@r=PNzZw0S1U7x8Vb z<(oO(e3^ejhmbjLae{T+b5&cmZM9LiQgEkBohIm>Wvcn-*@wj%&DQl8=Qg$|JgqUZ zPmgGZ2Iv_5LCN8fos>+oZ(YBDe|~nw3v?kRlQ?AG+g;R-g!z@JxSMSf#JA{fe5p+4 zy-IFZTKK(lErG5drg1di0p{i|PLH*0&!vT5gjy~quQAzc$Yw%I4m^c$Bg?eQo3=B! zSr-})ktyR@z=On-UMs1gwWC1m$j#632vGOY{y;;O>)mKoa7O38BrE;=0_=J#VTKkk z-be%H2&1bp*DKta6kYv@;#1$(NpZzH^bs~`{M?wP$>}hq~Rjf_rdFNRj(ObqQ zx};92I$Cl4$|+<1%b?PD-f>Bdi#x0icgE<^-2%U-smufgNC``3aVd06yM$rqG5{#G zwqWQ{&?D=i=wkXurHb;6(V3>S0j0_CjG<3^i!GN_6pT8(fs`+Vws21-J|sY8tcGIL zgi6{Jz3#A(v4BJ08MB+S!O{wB9oM0yZ7hgbXycJJKNwXF$2y&V`afe5RVZyq+6^7$Y_FV>!@% zV+=ELb7-|zqZB2ShAJGRUE zVVON&W$E?MUV7$9G@sQiWIG_Etgj7R4l*^|W?JoP`5y;a8~`LPBxd@4Dp)*8xl|Y5 zn48PGSN1R+LjElRlW2B9czb>$XO%Wo3wgss(f|nCt|CJZ*CyDstMv~BXTf{JXw-s0 zbLs5K59`&(k||Q{Ka~gY@P}3|f=v}}O-ux+Z2FVE;-8;h>BOf0G5GQ{&P2CNI<@Xb z^xJrY00x!%EM^Y{9~-3TU!u7LqT!g4DqpMf-~n?NkPq@lX2${3`XMd5dCpJW#JydT z2$ZhG_3%3AR;quJ7+SBF1)5_0ZA3TTpX8~iLM?zJ`8fk^DMhQK2m~%&Yn3TY77q!~ zfMDll$z!~Dlin7jiGVoy8$lWNXn-m2&gd0(Z0wIt$o5Oi_~a4UA z&MT*>&{^^vFS%t&N7fv2W`bniFG*0iW++ni4e;{VCAI2NR)j(Fipz{^=jR|?b8eTw zzN8+gEcf0J?q(}bPRVWh6P~dgLRc;55$U4T9`w;6gK`V6od_A))#nL;w(((zDm+p3 zK{AE~FW0H5SU%%xZ6io%D!%+dPR|$1CXfg8l~+?6esx=>|FF;#^B-nNobXr7f+QP2 zk0iy^cC+T`03-*&EZ%OHGVXwIRqZ!HJoa@K`#q1+sMGVX7a)x{a$>Z`pe z-C+rZ{$90>1X_WgyQLO}u>Ht8r7lG8{M7oERV{wOskyuEPJs27irWRE!jS;H8_UQ} zj(lGG`aQ3jE!aUBP+X6?Rh^<1BT=2Q23e1iJl0YJbZNI(sIP)x)EF(5YhiF4F59%l zKVe^IE%MJ|L>{r;V^4>E)s;2_?_%=6wx#0xHJ0lY3vYVRf9))iY>+CmccQ2D_q<)R zbrSUj_+wu<*CA(s?xeNn!(ngxVpqf1v&h~u`D0NV=qCdEOZa((l)jVP4`A`F@E7X| zeJaaM$E<^Tl8fV3W_l6;{kfEmaJY65Yuk|tc_pN_xTe{pIQ{i+02l145$&Gc>|^6f{kV*UZq29+DPlF12Y4z6a3jXcT2lFMvC2(xKJ-4))+b1c;pZKJ)| zF9N9WmQTb$3bT_ztL!|n(Pnh#%~~sp_}amY@uapg1Xf$UC=kJVIc8(2xfvyD(#KJF%~XPkkR=@#vNrs94dtDxq!pxK!qA z)l(gp9~|Ufl$<`yOiGoaNfpZk=N*oo$6B`h>8X`4oV48~=x#7Kx)@KTY^Z9`0gzkD z>nBU95Wn0qxA=!7+rUl%cgNw@@nyejEr!V$ccqG{k<<3v*WlD0-#3I8vkwh;o}%`! zWFolIePbrzX6RMw5WzVsHEVZ44<1p`t5x)P;@$jZa+1}9%A}t2X<-foj;hxu?<(P* z(dSi6*rq9OU}Xs|z{rR6!dpG9=trTwnWJMV!r|SFAr|%<|1pOR1 zq?FJ^`WDv2dSwCA0pofQu%QM%NLqvNO9=Q-l!EXKz5v-AV^dJG+kHjg5o0B?Lp*sV zI4WY%W5pMeMZz9 z%gpH6Yri}-5?B1{<=kYVLTkXy!O5$w;YsD)T$Jo?q4gvrlbGkoms1lmcqpxt-`9aZ z`l{v;-YljR`7uI@Q+r_96{=2Iy~60$C?1y88PGKU`p%%!TNqZTYs~sY91qBbIHb&8 z2lA}EuIQV|0w);K)yhihbe3sW5Xz~9?B~IUp!AR4ko~8!z}g~{+iXzUWSSFIzJlvy z6|MZ?#7(ILG9cv24GG6enR90lLGOFpkLwCj+*@&IbxddpBgqr=lq$|pd3%1$m#uFr z2nIcEQz_i;a`HIvya3(vq+ws>z`r=KO9_<$YwEjDgY`>iL||}JfiA&ZXilIJ7!($l z;JwoQ=+VSoU(YKBhPYH^&u~)NDeW7A>!-V8@jXhMygPzw#=B8T)uSI4!xSF%st@A3 z-b&MMpjQYjdbeC&Q=wMMs5wnN(#F@)o;WCt0JfKt=Q@G>`NSx!WFTL621mTJyDDh3 zeOZp;u(yF8JuqC_U}Qhb7x=aSOx-4+?YG`Wz6;7}D;_xhlT#2=9rDI2HDmVWs!Zq@ zOlV0}Qp`0~9SO#(3)+T}LRW^RzEZb1xj$LjFGV~s#oJH9n<`fzyEdkef)WZH5JFa!OZKr;V~Yf1f~UMs~>SMLNLsG z&zn-b7prQ=nw*L3sl%W6T$uhA(=76DZS$?-k0a%z`&FxyL+f zm!CKaKgp2dWp}^4tUGu?*AFU{enfmQJ{6i^Ui-`X@c2M|Dwsobo2F#%4~+g*N~ip- zxB@1;easb_El&?1`Qn+k=PgyJ$z|LUoBRWjvu1{CWNjt5S@^dsHn=}3zfb0vVQR>^0c<{l(xySp0D2&{JQ zPV>zxIXXE7?5BI+Gb2=znu7<@LF;CFx&cx2c;T+DsCMH4E-#_r_0$%2PGdm@naB zp5t+USp0K(r;t397F<@Uqb8(HT)f$pe)bGetRaOa^iCKW9#QU=cpKvWVpC#>G*lGH zb#mRK1M?~G-b5e_mS1?Dvpb7Azprc~7=}-{zrgiLO*+H6!ongbk2f<#)scFA@f$sg z!angC>`-F0!oC{F=Kgg?Qg(+%gH9o7DGldQgh5L&d=C(;YuFckMo7cllI@S>4X&Dn=XSKFlD{nN<8pWH+YEAj*Ob+Xz^ zow+#H^djGn6UlZQX>l3HeJn1roIF*fP$bR@qo`w4MI)ov>g-!|pShDf|E2UXNqq!Q+{p@U(oF(A zatzfH-KaV@qVY-ZdW2e9WOh-~O~dXa?6BbKJLLPayTF^VzfVkJbNZD!TqS;(d%RM~ z)qXJGuZ_bfFz6K^`yQkoQ?!L~aS}UEFF{W(v)XpS6z{0lO}6^V{&cmwp40umr$HzZ z6;97zCPHKn;B%-0iN1i_n*FR;`lW9`_?vXO_X%S90+4Bd6Y;qtvnYqBq$nX7UUWdxqLS4bo`qq@vu|fYeHRI zdOi9IvP8OosWTUdh?z;Jhm-%ewy09s_D}n^RB#tUe0! zlT$5S;d8nE7@eb2jm`H_%TmAqSQvXA*?)RGIUis3q7LsXqo}m9g}j8U;?mk`5oSMv zSpW=M`cUc$nfn&V=4+jKAHY4jZfLFNEoD21c`NJ#zdE9eUt)S|$LYZhLh)^uxc&G| z2c-nF{xVF9*p@jy3N#9Oj=(JH!JaUKv@+;a(Z~_WNz45sVY- zZHoqXY@}sTv&i}Eu93CZB#gK`Jbq2(b=K09|Dj><5uW-8IEyVf z6%PJ*ht=WZv}af@{F5O-K+aEb2*Q}I643ZpBt?cuz{Y4hGBqNjr8|bJc!SK1yl+A% zyE5B&Gcb1Mtq3Dlp(=tx%8!6g4r(zR3YKQYyc&&^zfVZ^cn{9alf@A6n%nCy8pNN; z6?SkjMR?_s^!0Wdb>K&na|K8ju0c^Y-oLS`whI*50Vf=f4Z{>`ebrjk!`{$!!!k!v z!vDiBWTT##7T)}QTIFKg=mEh7hARJ!@DLbKVA3QlV_ZVY$HF@4HNJ#V>Ib|73DG!V zlts_mp%hHNtXLQwW~&kK?F*BtEZ;CjPg1^E^GU7rubbZc=ichqQwWHH%`W@Y-?zBn;S zvTYBzz}Q6dJR>0OmZtrDCLOG9uLTT_{Q}-*xb?@!c#8VQSZFlc-E-xZ{M}!Ee2fGt zB24S(oU39Lm+)lea~_O@@=j& z_uudUAb(%A75GOlM05&=<-^L37O;Y%v#0~oLtM0&lTq4JekH6a;9!`a&ssWVml*4c zG1mF;(Ss*>7*}W7RXthP;J}c}y{YGyu>=DdxgWmIVEv01IMLnP1k>sIBJ-{~{*BRN zy5tvE$ikT5)AjUzvu*0>KFYDlg1}q&PlKb?@0P-$m(z<+k7{E`bW01D>8nQC_|(CqDCvXsCX6 zjzaC7pr?!Xut1TNmnkd@+eE_X+W;#Y;sVs|_b@ZR_!YVV*HI1S!Yj1_E`Hx!<_v-F zeDl#6(8Km8ih=xj?WuLxA>?mHw3YLr$EbM1pSH65Z=;u^x3#l*9Du40mcp)HWp-{=O9^eTtog0s-TXN+^6 zMpss+yb4{JfPhZn$&L*iPFlv6J9m*K{TPR<4V~$Q-uH)z!M7Xz`D@l*8A<$2x($f( z{#@DV*MkQDEjQV8-7fGb5z0V*=j zK5b&ibpn`QO$>nS4Oqcm9sS1qFH_px6ATuN%f#iY)1&yb8`Vi|Xw>h-f?Nv=CARHQMn(b!MSWFj(HE_4ML#=b1I5Tfa<_u{Hn^&q zscq{2vNU|6N?eG*+?GqyO4)Wp^hSbXGN^w+*Xd^u2H=ZzYifngIoT>p)uJ`E^^p#U z$5~a{GkmGXhFMJ*hueNmV{>Gd(ccVEacp!SLSgu$@^Jc5Ia3HoRcl0^x1q{mm5O zV(AD!b71m#VK)o8Drt4nz@OZzD z7xDxYAYT{#Xq{J1^7G@S*b$UO&(WGuH3xN51?Vu3x#Bqmub#!OlxSVs&2*ZvSU}$} z<#Be+j_C2Gi?F}>d@|KZI{L`m{;ls4-1gJVhrn^`Z9nPC+QK5|?X z>v9H{X6=Ff^+|!=ysEH0Q)G3$_*C`rXteK1! z%>@b6XIga#5$Ar3(f4W5uRBOjh{GPXpVGz@96&<}dy=w{GMz zuoCFf5O77w_sNzW`S*?2LBRP#+gKcl5WId;Hj$9q!bvELzcJ|_VURo}Nuf+feH%^dbijIh!U!80~x8&DOT&|3V zy|(E(H(GgAxumg>L5Z&O5rVyAhg<3UOtdDSzuGe%%p~n|0w=C|pKhA_h*8%P}% z&zDN&ha9D8!C-wFc@8mL0rS)5Z0HGMt9x;>X@(3g^3BKMQ~!dEfh*oU2Mgswov~tR z63;>fK*Vrbv=+v@%a$(W=AkI-CariYUDGw0LOe~j-0263h7aX!%EA`=@3^%Y4{G6a z_H>KkR*Kfk5C&a2zy#JoSjaBP!p+|rl{mymKK$*JV2?vzCLdMfX6L2{l{!Rh3gz3H zf5P5{<7a<{d>&w{Zwe_fBaZ|W&0!Qh;ZzFUqr_twek*<#sgdw(d69@C`wx zAo@YJ?41QiZh$a&Bn(?NP%x&|GmRL3%ao{o6Kqo8sXqh9Vl>xbr$-I{o`yep77ia+ zL2iL2WJT>Dx{9bJxveF9x`kxoWj8EqW>f=pV#g6lH>wcIiRG8Mv5ajK`ClyPFh z13o|*6lTu#hgu+VLnnC@Cf5`_!K|leDPhNa>T5F*^;l$m447)LCvJG9ELCq2&w42N zX(8aK`~Xl#(Yk`Dnsw5Dt-@gVDi>Boj|PEhebLI9nbYygou{#< z;k^)|?X%w=z-X~INF>Cj@x;bofwcjqB`-FrL52%iNLaE)ihBAHcpIrTxoI59;> z?oJhb)=elU;ldy8a&=;BomL^Wlib6lZG26BrnfWO);J?4s6y@Mk=8qYQs$NKHYVwF zr4rJ15o651&m^EE;S=I+Niuech{&I`Kq32TCsqWj`6cmI?>FZvHLH_c;OF0R72~0( zL@G|htWsGcBF(Kiu3$~Ucteu^zxyRu1vb$_o!5boCy{jd!pXNW$$hT6P@lYEV zAa;ohg&@Cn@)K`8-QqMts9V*v$R625x=_@b2 z)Ec1gyf>TTVF!Ap0!yVN*}7F6J>=lol1|@!k@^~T5fY5tbo0G4;MGO2T5=ZVn9`pV zr>JFcSI+65{HF5SgFuY?&#b#D$92Asva2xHGNH6(QA=SZHN*SSriX^n*3kp#6cUE* zZ84-3nj*Q|v+ZqnFY1*BPeLEpgBdA9+%Lk*^_!hiQd^6j*X$=n1+57nHd8{QG}(!C zA=}<3QQF%@n~`R)A##$@F%CI*JFi_K(@WhTG%2YoL)0^Fd+m>t0*n?qHXLDdz(Uq0 zYdK*I)@Nsc%xaw;P6)xQc6~vrEc8XM8v}J0?V!2-zR?NpOi~T5u)($>umMXCG44$) z#ff5lRzH7;vT(CH%pc99!_E@vkm+$*1;r}4{o%p{z>7BllU!#!2sTp&2>%^Zj~fNW zt1Asn@=5~YJeGMjHtAub`Y&R$hi+Q(Is_v1QI+73Sa}g_+&xXOyr)F6n=!*zde9h) z8J14;p9p3V&x!N~hVCk3GR;IW@=r4-ybE^`jBphi+`Fc_=_^$iE0@IFUNU%Ys z#C<>rF@l@QGWqZ1cRZP_9Jw$kyRSVX5*U||lvjBd)h<03op-XM1C5bK&((h(fqNv4 zwyvJd)QxWKn#_b@V=`&18f>2&uprTV6LC@1^VR;RK-lFFl=hM9ZFZG2?dDZ^D>Xs~ zn@*Y?Ao9#Nx8&o&beN8uK&ipd+o;IbNc#Fzze2}5@YFr^KxhEs#YDg1153XZ+VBm- z8uk@?G|~-+93mQ15oo+a8XliOo>19MT(Efn(k3pOsLkg_lr=DJQ^Z4TvSO^6cGhuv z`^p{3n5`N2-CN#8@F(lO(ax5by5fNBjU%8xad!-&rtxXJtL9h6qGggXFiDjkuGQM{CNZ*c6gkksH>a zsEqsoq;3f4Xn@;rf>X(FxwDsd@b00Tr@9{(Lq6ZIQ%!=oLzgklN~@PjYx(%RLjMrg zR!JSr-W$x?H`-7EkCm-wW2GIi6Oe56FyR54uk~eU{!x}X0mw4b;qyP{)oJ`PPYF- z4Oq_dQaR{5Y!6p^s{$aCcBcnzL5t`Uns9TlE>CV?wVYc=H4k3YRvw8FS-{448m0#p zxbJ^(u7&HkU?`9g<3vA+9lfLh)dMP<=*@oVQR*ZUhxIg$+id6i$5EwasLwP&>{BZm zuf31^<~{2Ros5Be_k!1(D=X6WrhCn5Y~!>8dCf;TAi+8bU}HS!#<79jCpRyam{da( zK^94LU@uH71s1ed$|qo z7CO3q3>u5alDqQ8kQJ?Q{03)5QrORH_zCV~&ymkFn1G{eo_DAZue|4`t|MG?qmBg? z&BnI~mf_vX`{IpweOPX#u!o?{(C%d zF78Y)#QkJ}aJY;L(H&1X+)TSvFZd_n)#@iy*MK1>saxoqb-I^&nEeMfsRV6`$b1p=i8&k$3OZ5q2G}>$*vX1KTx+_q1WQM{5ZDi&+Gey)XL8CT?=$L zCdK`pdH{|h$uD?$Ck1|08GYb*^UsDSfa>|-qE?+XadIMVpQv2Jo2KjA2VksmOnp~y zPWwD-`|vz%_f>}DuTKp(0#+VflB3D z#qa{NZi9qW%)BTklVPl~gM#VsrEmMtTfrq4hvdZd#&bY)dq3oo$ik&s_-;+0H5=4G zLnfE7Zp}YvU>?#zK?=2YZf5QvjtoM&HU1~woh0WTU4cYoxfMNZiJf16l z{qGIcRAHy}7B=mi*7P^dx`;jTd;yX_9VC|P;u$*|T?f^})f%xWK6qGT{pQB&`q39U zsA^LZa#VB40Ak_z-7fV#?1$SQ4p6{zFysGkAa2vd79tz#e&QE4RHZF3mDUk6! zMf#7ZqwFnmWH>TjULpJ$!&R;pSZ|tJh|zw1;r&$$mjQM2L8ai1Jzj=}C!7bG;_K;v zpw&`U{IiB32>Q_vA&&9Pv$^UxrF?5#`3APxs17VASbd%H!ssmd*$;5yPG*P5EQBuk zXmA(O4JYL2F$9#)YM`wDyA=*l#We2AmBily!Lzr&wByZzPs};~GEIm$^AWJKDs&3! z%#wKoQwJz|5!NV?A?3>Qe^c14okkLffZ-=5=H-o`Wqmm?548t4Y;MVLbE>Xi?`_|P zEHw;)`8hHb^lt?E+CqskLmLy*^Yj@LQ2pK4X&A1+#atO&Y^S=-DOF8Ty{mG~F zM~f^~&;XH2J)&EpG!aay!v;u-+ShVb`4mMIG;L|7=~hQx;hO2a9uMCO$Ryg>K2(*5 z!l0-cnwH~}5q3R{m?wpdhAju(R?j_ID0y2)Yjv38bFL)#5JvG|k}fR0OnNOJn9Mn$ z2Xw?G5F<=_j;ZkDlmIj|O+U^*KkAQgI`Z+x_OeoDggLrt7P2TwczRk+hL6#&uAVzQ zKX}WPW6ltMLq3it_r@6kshBdA`>%Qb;G9D)&OgSnf)*1Z0luH+f31}Fv zvvePi^VP%RISxHhojbM;%)hVPMZ1mvZ#PljF6L_eN|hi;v45u*}5(>O+@v=?)S^5 z{^Ti6b9;V%i{Bu65Rz^C!BygQ$@)W{DIC%-cmEw6C+460z;z+p=riuJFYg^` zp+vyGid=MBtREq55i@GUY@T@ui=P==Nl`f`&yJino&ERDxTQ$@FHe_O|M_sP@sb|w zvCp{|)5slK8blSN+9?k02aQsLU$(yX#(KRf`w1768~I$Z_#&YAomtoI`4zoXP)xcs zB#6C*oc6B;OxP%!GgXI*e8G8_Y9QJSFjrc06+gtQ_qwtYVamFB&(}oOM$e~C<5E}$ zF4UT7ks6>m=OY$oF!d`37VBoy{-7p{!9GEZBSA=~%7xD%)qM%=rJgLaBd>HSLG2KK zij*~xdZ#qqzSrar)dlk7`Xg@*jAO}$@+TJ0wJ@ysL8-uS+zdr<0i{NFrLaI=UU<>B zPDWgIT0U)dAi9AN#LTqCDjRW*tj9PRlRWjKyziEX6LkZ7vkiN^|3fL7$;xBL-*HaY z<1tlB*@$GS>be5rT~Jo&r5$2zxs~-2 z^HU-S-UeqC?~+E;j5Ol;j@%n<4eVLHfC&0UlM_j!@(eoL<3gK&WB2&_x9g@p(q#NF zNt}VG{CMWVUDo!Zkgm1o1)>_XeI}0w-qz<1Mxc&0Tg7l}#;t%7$+&rumBOAyg_$X6 zfBQ4PU@cuIHSbiW|6O-*U~Hkh1!P8AnXIrK^e9)+y=_>K5_}-^y^v;oqMHTA zFS?HQh0MtYy;#%AX)RHO*_Ut*nV9mmr}A$nI1w%cOUk;uL@1OCRkV^q66Ihb7bt$u zi1b~T&6>ibG^Hl5ZlPqw^c*xjls5Au!gUkol=GdsApg3}NieguQjVh4rI3vX%U91X zI(VHR)kHT&l*zjxrqHOIEc8&8jT#FD(U*5}jAVc%H+W=Ey*^`rvMafmYX~t2`ds|w zqvg*Aw;_6OaD#uHq63Motn#eV9B~Ou7`oCT;0~0gb1u`8ZmB%gUR8l|Qbwo-AarYx z*neWW*s7#yBGY|>FS$PVtePDg#Sgx;O$9Y0B{pe{6bw43w&Q^GoG9tn7Y_6zit3a^ zH0X+AwZx=z)9))C3s{+`=fZQ&0W#`@iJ*ZjkHxTbgnBzz`~+BrPyPKZ8D)`h#LGz3(7(zcxTNRI-zFP zrhhDUhqre1Aa*`z4j>C>urexq^tUoh=c>W|+IRP>hcSq5nCwX}M46gIX0$IGn|K4m zg$EkC5EB1_jE{!ePL3YkE8_m-LG`&XhU=f3s3Qp_VQeQ)+n+Js2~9~-OMaqne0i*P zy(9gjhRTjH=1Fktvj5NDK6|!!=gfNfUPECv)+=GYsQ{rm!Fx`kpB-LB_GszAdd3%^ zIgSK#Lt1N&e#r&h>hy3N3*mi!Y@C|u2k zYM&X%l&NT;9m3QNi=gujYe^?u1N^hBtcuO!*jMU=oqGd$0u#jc_20kKu)yZeVdpP2uR+{9_4dI6X?IJ;TDpO##s^aonj6^@ z=cfcdBJp~;{1foCk8wY7N<)hs z_~(5?z3zaV1`3-u)=3uT@Yd+oTV!Re!#)r+CMYz8E0f%780JI@YV68aN=tltnfATe z^LfD7UMv7IUJ^ck5h`)ZkJ0Sf1tHCcqU#BO<_Px<>N=u2mP!B_s=j}jVF~_pjqV^p zxL(4J!i!31#=%QlgG={VYU1<2pR#z=Awpm-5gt>|Sds+=*(%mTf#Z$JN>!h?SCrc` z2{K;Sv-HIY)g?d<{`IJ+5qotf(QBgmT0{39Td+yq9M;R46I2+}@tc$hY3i+5>?rVv zW~q`KLd#R=KPjDr}r} zGYX&`ySh#41g6cLrk5#;)eCR-Z{!WKF_9P^nmVU7VUqX0{ zG6F^q*vxZ+}gjQosyTq2#sPsTn{NA zj`#zu(7SvO1OO@f-Em?1#`+cut(H7Jb`aSSVu~~>w$W*GP!mCZ*#}xFebD+3O70=r zx-~WWNtK|>wmpZsi(oWeX{t%I9HJFWdfoH?(ve5(S9&qll*B(*q+?E>fVJLh@Gw zwI|MOLh91njBMY+ZN3o2!e6~ri^Fj1A?t~croJ#+$7&$oe9r!}qF`;)=pySE!ZlYOUWc^o7I~*JwH9jn5TA0PK!oUbeze z<#lbiDepg>v3=J5heQubhXDIEvD7pqxGue6sA+r@HTCt?T|W<5L=Hpxz3@y0lY3mK z)Ot}R$`H4x<3w4Y<1ar31mvbN#&bv*4_cxON#9%riyoW8Gf1ZmIaWFW!>L-m`nm(= z2XANO1yJxDkyo`W<4S>KS>J20IgxJ*IA}5u*j1vS%tvZ^M|Hf5T+w$mHNhc>ZaL`r zV~z$j`i82q()2KIv={z4my4zd8g2DhaP3LhfinoT78 z4Z$&AY^@#J{(KO9qsTi;>0_~5wcXV?P)zMv#2|?RN~otD_bNlCK-s%D!=WCMYxQP& zoL=IRU&LraAhx(Xgyc`U78I#-7{lCOpDa(*v5l~tRpVyd^~L>`Fu{6et~(}ZkkMml zDG)m0VF;8m;zY~$OFYC(CkHqUXJMFzK~{m>Ci$%?FEtl|EM6V>ebR3hjr1tyk6$UR zL;1#LSt3s7@$Sd_nx_-OWi#!lxXJ8ltuL0ux{5dCTJUA%VU*cw@m_M#JP^Yo4a6yy zukgzn!5{PkJsT!yl&M>W{x2rkKeBbRf5~t5#bTu~>S>ZBgG_($F|g_Xr=c;XnHR^h zjhhd;G*@Ol|HR!}mop?5+0(-zj-jmitzE0+%8HWum^W>2EW69-NfZS$g<7{dA&7^Hz+ z3N;NgS5jQ3KJG7c1M#VUq?mtG>AAE)HWIYcvaAz8RUV zTiGkcl;}smiaChiU~5>zf01XRGl51{qFep}MIL`G6y=-f2*_0jjYf$bD=zKv+Ov^) z+)B-79iKNtO)s6KpVPJuucu{N6mb5V^={26_jA4~S;Wcyr5ztu#GNurwHiV zI1}cO-y&}pawNT0)A z*MXS0dI$3>yjeh)4P+M-T{Qt}iDknXM=e}cI+$(AIx0mE^f#1c67Ia?$tkjC{4iAr zr=V_2mxw*)E;>!dEc!f*ij=*`l4_@Lge&-5Z2Mje60C z;Vu$-7twiN4K8zI-w{F5mWeB(y%TtQe8f6PkXt!$` z(tj^*Y#H5GGRVQ%?J!-rJ{?jbpS^|xW8yfBRFBFVyH$XRAK@AWd1{DT%AIzJgzy~yU1hEN@rdpFY$}@ zfV%j0R0gCB@>%^NWdoOxQ%GAlxf(Vo1=IW#NagiPm3M1(nS|9dW<@3uFKc$a@2sI) zGwEH5axg3u=j|$WcLzK$Sk!{o`Yq!XIZf2_Qis4tAY%+2qG_E zpSnox+P;pVPNnNaIn@EY4_XSEe2SeF0EEJ#%uR4mYVJk&(h!qWuh9@V@%M`0&#@N< z5Hk6*FL%*FRcw0~S%k;=f#>rB4vv!zFqP#{6a3~vo=D(x07QBQr zl*PwORq+TarSF2`woO}o*-DrxwIrSH$O$O)y-1QTx5);*G`64^Nu`4I<2}%cRX$Ii z_x*N7`U@bA$=uVBFNpE%yS^rH(47!rKW7>r7BtUDnkhc<(k%1>9e>sBE%68&W&{Vh z^yqp20AXN|{>DQ0YJ>Xs3J4t@^hkqZI^9}d8h!4oseKjBO-6q*8f9nc{4$Rb#HN|=I=Y=F z0fZXvyP_~;qUwb~pALJP77uaH5c#8ZId^n_TT3~yk=#Ck^NXAj{(6!{`w$YvK z@Tu0#x`y>x;9<{QH~(Ub6qAreWO=88COGP+#_47+(u66HbrM>i9MU-gW(9!gri9-K zu)gk4XE#HZfEVPH=se2`H^$}ovgVctItoA%$Fn0B=|-wfX^7msf%vlX~&;;b8^CRr)#s7 zcXH{G_ub$t%5a&_iwXCPOi--$;?;OMdf$$|R|(>=j#oX1Dk^v}8_j(D`R%o@ob56e z`|tD)s{B>yC0uJ)M)MF#cI+x5J!^299_i`N+rYN}AzjRG7A9xW)HB6BUf|@(2sD;i zz(VO_%HIM9V?q#f6bap`wF;fc5qi)DYqyYiz;7Zmh5!zy9Bu~AeirH4t0qU z_4&|dbge3$c)F$s^W4$SYzqtU{_gnU2d2q|?+R@!pFs5Nikoh_y@+$u*Ri7%Y`J0s ztbOd#l}s3V8;9Br`NIC0zb)uJ+%L7Ye>C}QO#3>p>__9YZNHuD8bwr(3bV0X=5%QMTF%3Iv z_HeI65e4yc*47ww$fMF;fHhh*s%>(SOQslR@Tg!J@{@2CrR`m$~hS>Q-PT&G#{FPDMVC?Bk(;}eH@wC^k4~YwKdcM-W9;!R@43Z znC!$6so`0@g0VZXb*ai@{(&n2m+soN{{poJe%oWEc3(LK6FL~FR`d!a(WfP9`ja4G zE%W(&l7!%4xF#76O-$=fSExdQP`*uB$ zStI}KB=e9dntzja#aftp_BnwWg9iI)SXda?pCQAD*l#CYO1p1<_TycdqK`};veaI{ zWMEQ^mAc~?5LFrhStK(;ps7C_bA^_?I_(*H2DUx+<0HyZ?h(g`f=#5r-N8Zkul6F1 z@Vro7bak_E(%pI2SOu8tcB)r*Rux~C%GHWwI){Z%fvFANFPlRR*?U}iCXR2N8H(_` zh~mSyi^i$_0It>%h9ii2cMZ|`y4Qse->wW#hkshLkt-u|*wt~dYNE3j*&n#5rK@Ea zy)|O6I0sOhP*$@g;zo#AasP#tV!m18T6?<)|*^0u*f!jm=f128Wh zgSL2aNgu4(SfRKc88&HCQO~M;=4Nt`u+PU~$XFE{h-%b)Wxx}UJ0DD_+#zhO{Y0XR zPwXlZ1G&y9>+a`Eu9BdI!<{43M#DTUo;fIRCd1*U4xTTOd{q=uL;~^&hgLN}!5J0-v(?uc1ADibSJ%&mbUl z6f-4<5Y>WC_K_0t-a43CS@|<+hoXhmD~`#Va?)Fmr2;yUCi(9WqCF8_f#F2OSY4DD zfD~j{4ozls@Dagh9U>eeo^Smp*Mq{Dpr-)=17VDTp~8v3#$j_F=!PFnuv zxJ!%|owzV}mxR7Rckdc0%2{||{2Q9_kdtnK8w1AwdMd{t{~zh8Sg zrb^{9219L{&~HX=h5WKwK+>ZX(!<3M1 zO#Jg#%`=$ zX%ZV7m>57m0la5a9c)1liRkX7eTKOwMR0KRM~ko#k`Y8$*ehM=VtI}YY#_GfBDyLW zgJw63wKj`?mjQn+e+!7 z=DK}H8$B)jIGmd2056bFFj(#%?xcD0%lk9H``VS?&Y+gEKdM5=T!tUll)keH)rt%+ zpfi64-&Xd;`rK?mz-v$xc-Htv(#XQhU7~WGK_^+exf-#<2)c`_l;Z5QYWKY0u&a0k z_KC+$B2)`crvepuaf@0~H>iuxOEO;w=nh%6CVYo|nCUN0y-^wX{J;1br%+Bj5z=7k z$Vaa^SVa|^ryq+0Q8nW^v43fvB03563~miD*H?sjmBL+{B*&{}v=ZSj>||!P04cYd zYb*^}%<_N1kl@PRUga@Gtt=5PX z-T3=Nys|yux<1RfMHS=QpczrHI0&oa~pW64A7Vb|ZUZRXFk#^_uJOXX&Ah zSS)-ZtDEpRYG)H|w`}bjOUAWw_HV!`D6G!AH?KQrx;k-wNfr<|k)$Oc1JlKgyl+zN zEnojvA!D%fh&Mdz`v@iNJO*DOr8yQek(446fEzF$G@lqVCJumVkzs#vs0h!jFx# zoCN_9Fa?Nf*c0v!LT_48*n!sTIffXe6L08yEjc{>JZPstgU~3#Mf9uGlh!ypnqmGn zfni|vB{VF8L%GEr;mLb5z(-qs&$(+MY|$a(RhhfdD&K9vCvPL?XvHPg@VLO{OZWIj zP@}JJ(6L5XUiFvm`3oui4%$UxS2pTp1Ars{u?C;g>8d9B#VCYth(n3sA}%8wx~=Ti ziskyLxPFi)*GdkcRT^ggw)X3ls)R$T=8T;4-m1GD{W;pdY_5z(L19> zX^WlphBBVrr{E62$Pj_@Gq$ulXQ#YUv{cENuds8{LqRo_G1hG4MLOn|cMJ2$Alm z5xqM6ur_sA-mWrD)p2JF`-RR1wV_D> zTLJw2Z^=ZaecmtUv!**XSnE(fEzG&`Sfc$QFtWiZ@v{GG?MfoI$U{*=P%5GF1P5}l zs`_VIqZtEokWn&aaeJXpHR?sLuSk=i^irE8K+lhMj6_MH$x|&6Uyz)+wKf_fA;(wm zfvA&J?&ZW_;3wxUOo55wb$pgqg7-ANN4Rszs$bfR*iUau{pDaF(ZEQlSDspe!kmE} z%l{0j9hE$fqr7w9CP`5JQqPWj)k-pC)Yg?^2Vfn874lbf zy*FpjxB|1NhW|THdME+sGf>jIa$8n5@D%!Wd~!El&C-mg3YAj%O64m0d|F13&=SbXnRv(JA6)>uY}1cvno`GiDKj;zEwc^Sa;IuEd)+YcbPTMf)m zBewG77a=rW@yT{z5mFgvE;pn{`hejbGSfL0vN!2Seqt=Dvi_fGE@?E|A&^7s|_6VRn39c zJ2~I)?Dy#<*-xA!PJIuOsKSq7NO8;)iD6bzxhn&xM!;NN?NXsC=+~B0f|(YYnU8&g*n) z@aLPDE+j!&vSIjA3@Xc_4!mihr&TLp$?&9hxKggV8&9=FCR^*7$LZL?DO`Sc zjhbPRb@WdB&}2ET=)^TkYU|S3z^i66(F@QIR?h(x)|TSeD|bQVdYPaL?q`u|pDBMQ zCDZ3r*tRT&(NLL6Zov(gES%HIQ0AvC)EHSn^U3i13$W)8|DJmrIUVvRw%7tg5Q20G zH-hrxEbF5}W2(qS@heXRadtq*A6X`0?dS_1(SEr?%%$F+e?oW5CNgE>`V=r)AYkoRU^gT-`I2K^PMIJM-`eX~^d0>^Cbqt+*#>5l@_c>dw-Ec^7UlIu zT4x+nRRXaNi?A-Q>bG{WHmMfNg^doL{2tS)3OpQo_@(PzpI~VJwRInI%rrUlMZrxW z=mRd*;w~;?z9_PsRv4~lPlDzRCtwr~!8Hd}yr~#av$iZ2d<80R6>jno^AwWx`3fSW zGc8W9LB0OvA8A7saYMntR)7^P4S51ej`TcuhfGCr!yV=~g5Z2P08rRdSqup0Xi*CX^7z?U zBT%}!K!E8e$A!X+6K{>Mm76T-$4Cc8up2D8WpN3*wy zUN%obUgeG^LD+8D!@Xk1rfxmbwB`;<5{(n(#`F?)#F2*jnugdHNF)CP0SD%+-WAW8 zX=xUX3LR)pa_Y|p884Z^*s3TLSHv!3f!Sva$9)XV@tp>VH)jE!D=ZEsr$H0n5WKGg zF?+jgnV2Qie^4=*sA;k?aT?&70bwY*rd$RPttnybj#_E(u|?+fz4_-Utl!N9Zymz% zXK!-6a+)t0)ELhdkX44Hqe>-+^ekBf4CFy#zz-SX);!f z8gp=)s=zci-xi)=rUf=w34@&yJd#{4#Azt zp>w`WALO`Unmhoxed?1C3lu1!$wnpdE+aVTtAf78ln3|Zn|v$A%%Ev_3p2Vlqj!AO zy$OcriqckH!*mjKNh667^aau>>F5U5r*_~5${cX2@3ah$E4hJktuYy?;PZaa`?)Q7 zQrSdm^-dS)o#K3aS3~X+$L=DbPT@lgpP^|4dB3{Yf@i3jgclsI0=6-pa(Bp2p z3R#L^xj(Z?6zWx1^F>%PALSBTGTv0?)?HX>VvdOa?`A&XEGf*8e1xV`3_kJ(56nDXnemh zUF85k>IjeO*Oc#uK5ux;Y-r+FW?o(GhXjgh{^{RNr8D1VGpZ@D5h<|ausXEK3}Ph1 zYs3Xi$~vU$O`h+B;I3o`7@YEHaZKBg&0!ns3jXW5ZM|Dp^!xifjHoUoRis@qt3vK{M*{$zO|v}yZ|=DZ zTkkKQG4=#rhH~S70vcsfNGRFIjdngbNo@Ee`B zZ@SQ{xGy(9cqewak-&%tT~O&w3C9tYjnShIQrQx>Tc0*T*wB=yJMyuAIZ>qrDBuAi z$1$3L*1dFSqgyQuoJa-{Suzle5eG zINR-$z{2SgT9D{8bKYIUeR-CZdYkBFq`Zl!|6 zXPXWD=GY6pQm}(-*QZnA#>}GV+SwAk+rWm@@_E=%ZApLN5R92?&Y@)T8w9W^QUK&k`?Gf1*fU znnRFjMR7Z|9q9jHuXsefm}+KVw`%X0XA6&Zh;e`)jHbcT4TcjC;SGRgAqOdX$gi<( zYwPz*K719ScM^n4O!U_}7fK!l)kf;`Zomk(9#nX}jn37mvrHJ8_qRYShewV3-^MDg z`5&8%tTVNNG4pYTA zdGm-eTNyknWIMnqELKmZ?>$72IX$1MeSGYv(L6_&z``fj`Vt`BzwBrWP9AOVf5s4a z2>w5!SDTHegkGH2TrIZW;yD*?@{b{zaEbY@9d3LmjC7`a!mc*3c6A0fDeh?SF*pp8 zQ#b0I%y2*-;&TEDzQp%~TARX!J3ax)Vkzsb8a!^AJY4{XZe9^r4&&$Nn7EvVr$p&+ z|H58q{FBi0ehwhwGv6s9wS(usE5rb*XK%MFm~!UO9@g^F(aJ|~@M!_rsu;3#9WI{^ zn!bOt3Tmqe`kYDjW?g>MRH!R$x!Oa?e>L05GaV* z);*G-!gYoXoN?tBW5gz=EDGr&ReIO26JAOy4bGU>!Wut<-@yguPhzal>v+mg)B_<4RY##YTq@22 zn(2tpIuva|^_?`j4NTzt7W&2d<7rM0Oy6xbok3FvJl(7_yqb|S6v2%ZKec{SL8G*$ z;*aet!_WMN9WB6gu4)aj>?WCkTYp%G6b?&Gv2m9FZW?T^&kf( zuiGzV%HrM9F+9rA4+>1e&E1uKm(YW~ysu5K)LIp$(dP^S1tUkY{r-Ev9r@D1?Vpeh zO_;RiN3Z~0pnpGNg⁢3y}ivK(IGhN97)er^I=rrAO+2vwdrgxgSDj?FG$7BdGx% z;#ogY*~t8hu1M5WZ07;x@w$r#V;s?Zg;ViCix4tXnI!ay7~4y2!ItV>y(WS^(A5x2 z@1q#6x}c6B)}lzR$Q99$TQ|UJ@6VasFnT_d$P0XyG0i{l3oPAK4ivg`IU6cRuDDX0 zWpwt<1r8UL>-L)GDr2M>j(MjAmcc~TEYmsU2f0oe!;teXHf4PWYPr- zytT-1)Ebx-tRQc4ZID{Ra@w-RUaqM`J;Nqx=4ZrCPgiM&UPo>lvRx*y4zqkE3$aH! zCz4(CW&@$b!22#dyJ1DhVmFv7{S*~~9Onnux(hj`F}m88QtTFjjFn=bOhTwLlRe@tHLE8+_GnnH0Sq0U=ITVPi{_VyVlI#rEkC?DB=plTKjZTpOYkwPa%$ zpJr+c8a-SIw~K}3FrY8TQnNbFiYNO0TNOTkz~+Ktcx*b&dV$BVNgKlw}{QF%=P z%7O+hSG&p(Iq42Xpi2~zR0SJ)i)LBv|9p3ke7YmZZeodJHHtfGqU z6_3-C_zyxzkLf|0^y1ucac#|3J9hJSQLrZwlzZ*Q+9{Y@46FK4|2`bdV@#|%n;q%h zTnpA_xRtVo?K<|#9z1Wsk&nuxyNhA^<^f|?48&$n=;9|7cw`5H>TL*36!=aY5d8+n z=+ve;S1|2FCYsoY3{zrtSkvuX?hIK>m6v7rDn-_#rWj){$JFy%)Focq^jtNOa^4h* zDyNo_)zC0!yvEIe8{EX2PMbsef*N>E7kMZuwCXplAQqb0}uNOM%Ie5c1h5+`g8v z7`VT;6wA((rZOO=vWY(Pxxr<5mT@(Fg*wZ0`+Y_UF=-hBT?(vx1Z_y=|X=w>#77#7#4*tT&I!j`;vW7k44kx&=-rxT;~ z?_ajGK?yJNzf1U^{%!PZ3<RK~O-yb2D_#;!9r!-Tp2&F8Fbl~J^M zF`NmcEAX4d7!H{!50nUO8i2%l;b=6XA&qr~s5)avxp~?jmFiVyOB_@u2VR9X%efh~ z>lx5ht^Qbvp;#Kl*8jKu54l{(emQjTdXb(Ulmg}Hx~jCH#@qaXkSv*fV?w+qYxqv- z$DXM+qKAzo%krKjY89-}7ND=HIUTd!Me*;hh3n%!%QZBjPW%Eqqqc|XQ{7zm$el4K z;cDVgy6o$!*}^K);Vyf#UfdKw|5r!Hcv--WdyqK3eX={08eQGAtQW6Sp}}k}V<5s( zJ-(fKs&q69x@@-3&oBQOA*7vSU`T`9k$rAkb0s_G0m#g--E%R`+xEDnTlFO2fWJ6f zIi#R0DOjprNP*(&6v6n5jUmMqP~Cs^y&@csx@t?R7HwVCyS$CXL3#rz-N~xm8%|6R zD5kA(tyD~dvLBS7e?4$+!a5}u#-4bv>HesTe3;>EkZGKkPuy0LAm-_a^k3xfl%u{y zG|_lB+8m8$>`G*y*7idKxC5A@XbOj|EWPE3QZ>tI9h30Zzg{Fe;69}b|FnC-7O5gg zr1Q+UjZ0M66ovIb z1Gcj)k0@}Oa#<^EcfV^4BN;!F>T@uo!=v(I!!CklItS7SgCc+eB5{?y6MK2-T@Pqq zO&C_Om~5zi&Q5ydHwk!N-h9weO*$Pq1|iVM;3_ddet*HGm>4`o4L0!4=havRzxR%T;{Dmz&^rF z;^`QgT)M3LFYAa9k-8|8M}X-@@)-MKDqBhObez{c|L$I{vjb zPl(8x9t*n4+@&^Dmx|)I?~EAy#6~ycxL>MC!tBq$6Lz-vnfX8??cSus{4md;*(F2& zHumLsd8J1L40_7)V?cHBL?j`y6x@U%RA?_9Ix$){<_6u7yQS(xIk8Sg)cC|QV5-O7 z1HAX~_LzGd+Il{?!<^|^`JNKe(T?w)t#XY%9J4Cbq96c|i|>4gjkF}#s``)sCwg8V zn4UIRo*_M8V9Rx!H`NrWbP{Piaj@++NSuI`16K~4*u$m-%s$nOYCWeFs`_N(m_h2} z=*!ZREQjNTR6M!;zR9j!qZVi3_x~x|M=o~p@T7%ylvRm+mbrDT5Nf6-z=hCYR)XY` zQzxlyPz5UD6r{)Ha4mqqSpJ_4u_;~32@#GsIgyg`h>lts6wOG`IGEU<`n10z69K%z z5wH3qC4E(lzqNbEIts`6xrb2$si3YI`@u8;G{o!_`n0J`0&6=*<0EG&3xM`@Mwx$4 zEk0{pTo=d3#*JoNaPn`Jyo1xW$8OY^^xC-kd9fa1rUmuZKV9CQ`~AN7ducU|O_)ib z3fm>1V)x)aM;CHSh94y_=m!1h2}<{h^rT&7DM}W65J*kYejAI&RZv?axtRv7Q~pwDDff@@ea+fW8$rE8oCamUjbKt z{}1|c$m4cck61Ozx_mY15bgYXwCe05Mkj5^yO^%s+9FRm5L%}iiP1+oGmrJc)B4P2E`DBoWo0j98)-Sk#x~I|DUeg)hiusM-bJqlY`mwOcE2-RmMz zTO?7af>g{u4*a$Qly$ODb+(8CP1aEXpT^_@Jx>U`FJl&#p{#bsj?Wocvbfat4kU-b zU_PKOFawR4u$)~ius3-sfm}}9>6%vJ&z(@ZE^kwb@E*cZ_Py~trEj1pqIfAsqZl;c zFu9ciM)o+mLr$8Q{q6q(%8*k#K_vRNeH~nI>B5tEng* zsqXzwA#@0JomHHV?t05Q6 zo^*c-U^{=~ECqUME=+}E;C5(gRsH35W_{PsJ2RilW((E2T@~ZdqI=J%@kjpDGetfY z|KY}M+fc|(p_oC_5aPz5oLt;sL7#|Vj#I@lA$hou{_q$Is*+qZQ=bCz#5`=Mmv;n~ z*$A8vbf;JV3+wuV=EkH{zvZkWlD)q~0sO(veem!5$!{a{&l*O_SSWOSb!>yaC#U>P z*lblK&n8*Of*!E)ni80p8m0}v7U%rJ)NJi_8V#qTh$NEHw#mUG;IfxfrGH~cw&Wap zOsGUZZX?S~bE-8C<$IV}1$*>Rb88>?-}z15e^oVq z*@BQR%!IvHXM)hQ$RPWN7YGw%dROSF=tW}+`jtb)es(TlY8*0nInw1h$u;p%H7(hs z^>nHZbz6gZTe4S4;_|z|DJ^hOuT=Q(o&L18QmU5sdGrmMaUSMIYRT1pYY_6QAEH~bk;{kYbB z)kIy91+R99tyRp$GEHBS3cpC;u35jMBk(3%r*~oV{=U=7d1u7pn9D73Yg`*O`g+9- zM9XdvpcWo|U?-|kH!AO?a{QON!F#bzaMtoKb!2N_^I8v6^Um=AJ&WyeWkNl?#%Xv)vV`5Lq1FMo4Qq1`sU$SCXPC{ z&ArzsG4q%*8q;zXTSQ1gKpZQEgG!%p7qVCW22*?`k{Z^%Gi%ms5Vp-n$7gs`1RzJ zt}4T<{nB(1CwV!}hbVDF_g7fQ`$MNN~(QRe2cv)oK;Tl*|2UIr9^4Q z*bWYg4@cpFD=(l#o&1mNk_Cf9$zJ$t6&+N+SVD4&4IE{jAc7HzrTtMQm@|$dYB!cT} z?P!G!{cympyDi5&gC(a3%(`8&9x?CX>=T;EfXqCg-k-UD^4aaUL)~h^dqN+AEF_@-{)|~^gHx`Xu3jOrId}b}<&Q;N9=tG&*)1tvOla1mlci>vgj&6oQ}Fx)2uhFa z!CPau;hdea6Zk%@Vs6?8ZH`y?d%jBnFXrxc^rJH5wzS!Asn-!jGU=A1EYvU&`W?-| z^<4oq1wQ8KYpWc#QI`$`x;+Q_1))`>O#7J*q=T6KS6ANkT2O#es+u^&x`2Y~`}|v9 zUSY(a{hPG~1eeqe(~_bPpsY$fbCDqUs(_#-AxJTo#{er4VLyTsXf zB@I+ha?x7yv)qin0tskrag@OpxAs^xSyb{mvVDA}AK&XOmC;vIMkN*)g~Ko})E+dA zwDPS$ggTk`7(#~?EPUnNM)ef-^gjBbgxKxS%pP-A_)o9G4X-*CVR{Km=uV7}bows4;d;C~EF^Lq|)&7hQ2W%#@aaij=@Lgo8eUOTv#?V4;D9|Qj;DEUkv z&z*uVmj%xl8>c#qp&P;0{(oungvbEU0$W_+$l&7)S6hf-;d=#NZk}<_u@}pl&FL4V z7*U&{#=Lo0Q6MrR<#91yQElZ7m)GxOsx=`r1x%1F183TOd+}7^+UuvdX6k`vPRt!g ziLv)L1(3#;dg_<&&n8@~fY~#M08Fw-sD3mnuVIKu60OTS+V`i4R&rXw>N|t{d0AXIY z#A$7d(9aaJ!R7j}w$8i=lMea7wTEcM%cp~$iHuQlQVlyKxej<5N?>rOneD6yN zrFg$%PO^wkK&u&Bo|O5H{*0{9BbYAWsRVwREh{7Pm^_ck24$V^)9--SBTYS*N7a?WR9#yMy?{ddKY9K1 z{=lzwaMvig_fN44ona3TTvPsFUXI|l_DI_f_HyWbJ+rL8KZw>XEf``1MKeY#zcz0Q zZD*aS=nx8~tzE4CiP!Wy5fkH&(LKKhp)=Nb`VDC*d;r zTs#+X4*sAXx|gj3PZ~H=0r6&=xE6EjZiqk8U?{1^AqBX%*4K2$Zu=#7>EOhTlH~}C zZSy=jGX+YSItW1}JMd%hZ4WrZ>}P=AQi%c3g9g2nwXr@747|lUkdqOW7CmB{HjV>~ z=GUxmv#fYF9dqG%0~|Y|T3BL))*BpC>O#EXU)9rxK~r9({5-{F&JhtW(nOTxofzoS z{8o67;8$nO-Ez$pOSyiu+GkmKs!o3Ck7M_{cF9>z?%jgxlhuyoA$q#5I){!#mD?x|6#(L3v=YcAYe9%B z>#C_PAHYr2gSyWyPgmZT3I%lPh1f4u3_d`!36&s=2%=!Q2Y69{8qd9hlk`u~iNs^A zk>esl3R`EZ{l6sE|M9U5PaYXy{M3GhOI889E6gWU*Pd{XVkQ-#|Mx5k&Kg;-pqVlA zWc;;nBN{v+4a{I%;sN6F9nRtoO!+ZR6Oq8$Pzin$zZN>u+oc30IvmdG;DIQjxs4p; z!=iX|MALql<=!Wvc$DK_7q8~b^DWAE1*hoF)$EG)94HY)? zG0!ty%^b?Rs1$c;C^60kYFFvpYReqfh`c;kQ-i!iOnFbvDXad|x}NH^9T4a;D@Fwc9!-Hfg7 z_^hsMa!-DLxAgVgU!xTy`sMD0x$C#udmjF}s(>OB)zBjTQP6rUx*v zX$EN*-M-7%ne&8NCeS3uAAj-f(HBlOxwid_V=ldNxt4@H>^epg)crjiUyykI(Doy@ z!E5t2b&FP?`z3S4>>t1LuDRBAj+T_vq2;+OlVn#x>)(DAN%

`DOp@1y65>037XI z5=)rJBrrgMM7IZ=95>J47MU->(p!NDC=lL&97qj{4gEb%=N6mPGUrHDLX;dBaTd0% z)uywYVj1VzQ2Xt|1N5HJ7&ZLOEW~U^{$C#m$Gu+4!+ZqGzQ9#gC2*-#4J{AgQS_&+ zaJi$0;FC<>lF_4?_aZ)#qXE$WaFq(KlnkR)V=ULa@QM_Tcz3CIfH{Y2L`YAQDy&TF z;3^!36Vw6`QFMmv_}4k0cn$iZ1F!G5XUNQ{yOQYXi~oDp-Wn*f-m1~kqdoA?Tg%=? z#6pcJvU<+}1$3~WlHz09jS&|YoCv1>)T0o^;4hWjlpqQWmCP7aM76c49)8=0rn<=n zst7W_&X^XFC6DyoWx|Wa>@*?3LYO*Z+Efk(vrEqW*Xxa@LHtCnbb*G3kF6wi#Gup zBl{MJpTr~4Kw{gna#(t^eQDDUpE%1|^f@o>+J`$rVnrPV$`JVw6an5(u+a1-p5DsW zNmCo&^kn>>(v;Halp6AFv{&Q`iecBqAUcGg$d74 zk^gLGVY*+!eJ=OkI*LqHhIl9c>jYx0DMnQl-=-UmLu+L4T-l0=19|WCp4@GK)s9K8 zR@b$ddi>IxF{h*m)2CO>bwbG9%F;oC%5$_JW^32z*v;;>Q)8ZQ3|nK>8}lzw!``j| zu}6Hq^TUcvaQ>lMdPO;dU1Ka>Xb`lVpdKzwe0Kh2st?iqP+xpoi3y0iCq6xuh@U6L zh(A0<(>7HIGsEbxU0W0~oE4cmHynJ9^8pZS6Fgv&6cvsG{b2sO9C&9m1=|Z&ob{C0 z-N8uAI#q}Zddv#&H(;&~SH<=1WmERbxosMges=6G;OU&jH=mWuAsh5^;%;dAvdp0{ zE&hWSN!2?>8=JdZFhBT`Sb<7w%Luc0M*&p-L?CRqp%X{*dI0RWedvlfm%&jV;G|g5 zeq3)7JMfE!Y3Q5^hXvpYLh`L!+QZGB&mL^$NE`tq1-&S;@B-v&B@=D!G_&X(u-k1* z6*ET6g8V%ScG|6xoBpSB=hDx+y?{YNkT#h&~uggm`y&)l8dPJ?2aiZTGAP74X zo`F0T9t0o9XvP!*+Z&&GfsGOyweZNGlw~?PM`cwxmRVfFtDdxTP`1m7)uRu>3J}yD zPoT1RolHE*&roCgu+3rL2vkiWtOvV}oRX$Nu?p%WB^tSpsDnIXzt=m*fJy#w{zofA zsC9xDuGk56K4UQodxt?j0Ki6q93S2V4G!afAv3PWa*QUy*4y*+l+U>OA&T>US@bDs$8kD?=-eeB}wm*GQ{PkHc;v*QV@LO~1k)^0=Bg4s>K z+?cA`!01Ts8r2uK1ca@FFQT>XM|IoTvS8L;z{PQ!$M)1;)I3;5%U>O@+|5 z*@APlOet_l5h5V78sUpvE?gApskh~UUl3VYl!MP`O2^8ud%+L|IXla^jS9Q{OjBD# zxC#M4lXnL+V|1~W*_Ku7aD(#W1wA$mT^c$by$FQ1d4|?S#La5~zZp<;*i2Ks_L)o= za-Mu;?kvw~CNs>}cFLJ)s2}D6YrZQUVAbzx)NoO`6`Y(bZ2&bu%D;KU(bQT2F5j>s z%uN6ebgI}?TfqmE>vM<##zkQ(bU38*%~>pePjz?>10?Uh0zXr39<95rS|Hm@?9PS% z=E$oSKD00s0A@__tZreel7+}X@y2K+!?~G;wug0QL%CitI-UeJUMls3RMJIWe*3XJ zXey6MKE#P!-reTk5+cM8FL`7TigbYRWsUEK@M3=x{dy+McQ$7R3Fqg84^xBd7w3mv z{(QHToZM!|QU;T&>>PQXh_DBebeLPCYt|}9vgZ|cN^)(lO}D|>=5AJCXzy_lAL zHLFG|?guDFrEuC7+^dc+EZVF=oedMpZ04Jp^?ltlB){mz7od#!1qYgP(Z%2>zEj)c zYaf&7Yk8>RYMcfQ`BYNM-!02XV=ytb<$0&n_^Mkq-;U6~rgIgl6s8Ad(kSAFO)30{ zl1cJ4B>M$>>sPbqaNRr!HB1cdbCUZdB$r;WSI!;t3tjqj?r2P_G+F&?6A>}k;8|bz z>apZ&2jW+DO@3Os96qWMfXKL;n7o(+(0vK-vL~`EE5iF(VZA^FE6)X z@s3lajSck7S_g9&EGD=DZGRRwoJ}VW2KfmBgFz$yx`7+i6{07#k~ShG5}CCQ(Nx5l z+=mnUv*>14p`R-2nc~j*pgnf6%=u(Bg8#YV-`e27aAu_Hjjts6VyA!YMmiw)x|If} z{bf?5d(hssI{b=6MzsgVex}gP?UCL<&xW7HXJljID{6k|-5K{= zlh8C#H-xt_7S+@|QRX_2mRVk@O6`sY4%2CJVlBchEyEI28l0FFJ5u**`Ai zhKsmrD7x#3-$HVvG?xDRM5pfrJ!piGMw7QMxP6QAwqVr&?Yz&Y_zE~C5JC)AwihDbn1S1K+EA^vnCe4VK4vWx;irtp*aD6^bE?0yQ z8*Wn0THUi(!wuT6Czr6R_cBJd=R)KdrYKxJH37qYu)uezEC_D@Xy@)i_u?=Du>hZb zSLT4ZSC2E*IHf*B3X=C*Wudwf!Nv<=+hk61?@$nPI>!q<0sUBUY4lsJZF>Oo6h$$G z){kX!KMm1}Veb)8mf3hEa8N>5*R$M+8+dJe+w7^fA}xFk)dLvCnhx=%^$t&`1QXdc zcw=b3P00G7r@K7ZhG^vC#}d9z%X48I?=gORkd|m?>)F@;;>CvgrLlT+M*{d>3RP<{ zafc`!a>1^~=5KZ(Kh$omWp3RaegoT-`p;!CWJ+0+lvyupfpEr#>+2g!3w4een*J#1 zn6RfTye7kR4;(Oj1MnvyfPRKBGxRQpQSWz|s1MkXQQ2EBmI&TNL?0=q`dQ|`p~*R^ zio2Jn?joMaKovdb=|=E&DIEuioIyqPz#7lvC*f@f)3>b9@8eTDo3}EE&u&G0lo-%u zBD_R43OOW~b0H1ww4waur}vm`j zw$NK9F;ULioZvhrr54X!?WtmsJ2)DL0RnzIrrSj{1;mD6C8?i9t1vl!g6E5l=gNs zclhh?JQz1TI>8|=0uwp2vijp9;K;k=JT(t+|I|WG>_oOuabIq2x?*$jxMW#2|k5c(w6~hI!RC1g7>Bv!9 z*)-`i8Wmu-;&7)7QA>iI!bJ5Lf9TZZ-yfljkl$ zl^vs{5Z|zq>-+^)j0LPU@3LRL@CAonOpfgNZpC3OIIH!$l?>In`|TX?8`EkKY=J$p z7(XtdvFWMSVFK@P=M{xg@}#KkuvLg?EJMSfvy1p({3ay;(PE0(ANHn}mF}uDr7Q+H zCmFh|c_YbzB1yZeg)TSbc;&__!W z$6K5T@tSgki1yFj1hNDcOH|*~CqS@#Wg3W?<%c-j^ttY@&sEE&Q26kOI5ip!Q_~!W z7a=*s#_2^=9xB<%chwnpvjqf5A52h=vttulexq89m}`*gp7SPwNxHd2xK2RK|VhxzOlOeGiHU5jwwaG#NRZpkd_}Lw!@#>h0!oJokzkz^lIfg zmNyguY&LmMRH6DP(;kd!Zv5gAYC*6MbT)heCgLM60OG?(GYKL~lR8ZN^7-``A6%bX z&c;?Z`lgHwmGe?LVkg^zE)~qw!%n(_PL-V(JqHF8;4A?yUa$DXwYpdrplUX=(HZhhKeMyw>OLRTrMl7^^h3F2Svd-d4(du`# zIf^Obs>x7c!0ZxwBnb?<_a^H?NS6V$N&J&k`S%1szcZL20awmF>gJ6!x9O5k1yH`rJ`KmukHT8sMj=hi>51NA+p+Nd$!cYE^Z7gp;!H{`e+&lm)dJEIHra->HGvP__rVaX zw+7iZIRvRfe4UmhF74`YTXv;mST;_-4-Lg{B&y50QS%vPgKN9|f6o~0hfjRF|KaNC z(PK{z*jcTfgvihZS5LIKft}$3GVm$g4nay2cz41zck|`)@SR??iKKe>h_kh#mPR8> zC8*kR(7g5SV{)a}td~ks(Vx$>+pqHK!r8N)zY#d+AX7tB&0TOEwv%3?zcNEQ7ywMX zKz>tB{%*E#AXj@NpdufpiWM$y{E49_J1#f0Qbo0ip#T}ctbOZrxlgGNppjv4MO+5F zHy!R4b6!unMGf}J#V3bsOap@RxT;5kJJP=Td{wr}Eqa*un_RRaS6!Q;t7L?vUn_nf z4#bf4eZf99>cQ1ZmkG=%TSz)l^~rE+lUl3=H4lKeb6np*1SNQ&J!@@t4tZQ02bmWK zUi=xd%-lnoatqkmiaxGx9GY5m5}_3MEvIX$vKV5iqwY;t@go4OnfZ=|lM&L2eaIfu zA*8Sl{0K;01X5xE6N>m|G&crJU`Kf7;L&l0#GV=3G@g$MhCo16&k0c0*53-Ci@=V8&H~@9t6vI!T>{o|v4t4wT)Mvhe1IFhx zRzg^6?w4G*f)?m~lrdn$_;d5&3G+GxV5!hY2Wj0gJs1@#Nx(>-NXn3}o$Hm=P1V*3 zEk`%Q0%98TME7!Dz0L}zi?kCcar+KKA#T8g-PLWw#oo&lseUv_YMp36SjFzNWg>ZAs10dPSJArAey)?#R*ywT^;V-3P*$)TtxG(@8G= zU=l#pD?r62;GodLnq)p6BU;Ol6R}Q3pOyJVTzXJ8#QeewQxpHs$EM9I{ z8!ZYHV=3Lx=5mrI3^Q~XJ;hITbYff|HB_K$`O zO~VK8-RhlaQwx>~uE~&l2ynVpDs$CATKCTj5xO54RB9tI=m4rSU7sx<3^C|>-o%&K zgU2P-QB57shO*?2HSbl#y_Ho~z+CBtGdH8_36W`sh7>@;7TnqCXzJ@@| z%B2hOt^MM#ZbqDDVY|+R9$2;r2M=B|IKN>5%l`b&mSo}Uh9-0eY!JUCh(w~#7~OhM zjiNsQO?1S|ZJ4EOS@EG7Pw$_uIA`{3lgj=xkyo*GV^3Rm$Z8jNxU{Jn1UG4%mM0e_ zR-y3D25$1d&&M8S-YW}E#z}^q0Em?E8-=8DENM#}4f|N2ywBS+@qdiWuNu6+pce~j zOqt)!3+NyejUV{0jSUe}!A&2|?mv%A6?Qml- zMTVuZ1KxoTw*{f7N1?0o++n4-1Tj(x1TX$qC1Y~##eSla_4K&I!E)aL=HJspg z>h&a77|iBLpp|Q;Bz&I=?o|1sn(glu2~CvUAd`b+7F2l`0Sx)hU#xuC$;6Siz~yTqsfSU0;xAnM|zJH*1H#YeC+Ao+8-ZAUjt$A>zP zx7Ycx>10yfFK&5!Ro}vCXsrOc0`TvUdNSt#K&mnQPz%-XQhKkWE>KcOl_D9v{O1;Io2PpywlAXtOABAv~@~=V4j_dlT3x})z7iZv%}RWO zh47>=rok@F+e+PmB3=tdEWUVplvXvc(%R52yksTA0TR;-XYEC@rF4J7PRBI8^Vvm( z5WHE3XhGu^UgiVSfznWYsOyIDp4UA1gxr~5tHp3nOMmXt4N^B4X=E!clY>!Az$K9< zezIPCMMfCoVkj)ahTUa$y1|g7#Cnw>g<)SbvAx<%1)`n0^WW#D6*-9hb1gcCdM;G>sy=kRzGv=we(iH35t820iQ8y} zR*oKH-q(iyteu!Qr4n(2rWl-AWoG}L)=EDN z1b?=*%q21Z}2Z>ye#9eHy4KMKK;^T9W_}sI_S2K^xA25mp@-$2D?2781XA z2lsj`apaLZ%T!9-QX@sy;Do191Vs!@AlT0+LW|EPcb4HZ?w=(CYVS#|u;_f)AP5@K zMJv2rw4tgaA>SP#^k{3R&!i21m?QYChFPaX*E@p&Bki^J4rX9`sbLD5lvSPl+lo@C z*$ZRpXtKU_& zF{%;ZbXK{Ir}SGf)cC|zr)kb(A{vLyO%z&G4G zrxvB-7&On^3Fv~7DT5gwQ=aS%kvqA}?1oe@hnYOH2?~&?mZ~@2V)X%u841C-m6Po2 z1mrt>^ilT`fZLJ@R8mrMQd;~`120cRfqMLfuPzul+Lp!D%OufVC9+~?IKUddqLbB2Pd^37*_`d?W5F`SA%l)cDiN|lu#4vk zC~KC;Im050aO-8D8ySS9CmqD`!aJ*IBKlA4>gw0kcox3Bxh>+wyyFfHtILJU3h{c0 znF|Mq8z#da^${!0P+z#;pE;FcHH&MP@Q9Q!&UV=&{0YyfY#cGzKQ4R*uD1dx@k>a| z|DD>!{%{c;F_9W2w9&HBs17n$R(Q654HXJxrgxB7lh^)q@Ubvj>tg~a(HA)#G8*3B z3VAVV&iCp5sAM}r%^7gV(mgp}F{(X1!kNQ`a9?#(yI=FqAR;}$WL+54@X%&jSM7>u zB+W;FJ0H4;6cTNwNat(MmZT;{szg$MFf8$-1;mXkw68QRHKr?P(`XxCqczQsW@6;Q zNfaNpjP*O1Qdir#A_C7oXQib zKODwR!=ft7%N^3!G>Ptifbr|kKMK)U+_v%uR#)~Re7uGf23=X5JS?{?D*))edbND5nb;Tc%^(ccqFs?n8s)#-Ke-I> zE#C!8k8SPm!ksg6len}F4J-NMS%`Onv>d;7^T})x()r4)I>tV&jjJ6Uxk^p9fQBJk zf^SKl^0Lx}WVo8{%rrD*Upe^bVjJ`!T6#vebou{I)w;f2RJt0g^rL9wIU>X>T<~|8 zm8K#D=(I(D(e;(G#_tBhlrAay1tO!F35Nk9=YU(}3I8CXFq-_a96?+Y^xHKOY;w}L zQ{*UGRaMxP8OxTvTpSHsMm8$F$TiAxm@1w0!3m8z4hD_Z$i;V7QS-X)c!y9mSd8^m z$LZ@B9d?;;C)h!OK(Bwy!RyZ5V1||~qz^$qsqESP=~~>UWnC>6T1SsFfPmUtT*ru$ zrOn;qDYep58XuFRycoDH@|it2NBg1*eaXC8FQsFp1*5oOdyQA^Q*ZsJvg*X^T}Fas znH18MmLtxV%PtGo8;?F3f)u>w^46-n0tyLotXKjP=Q48qFdc~vbK6?8FfsX<(ND1{ z6}uGqpgdG5m&2ly`_slMo69I7P^b}39F50H z3NI`J>sozSt6wl4bGlI@zJ+jYi%N8rw)AKk+cn`P^_|gN`*w{XcT4EbWH-6DTj<5@ zaKUCg+cScnNnJD+ITg*ICRnxY?)>izNZ0(jNBM=88O-9Ux`8?1Y+h>cKsz5@7GiT{ ze-b0L-MQR{)j5@&@A^)z6{_q3$3-(A{Zr~h<2NHlp0+DRhy{mC%C%T|9Kc=)JbVJ zg_cxoVNTb!Mh_KHs*p|{@ny-H^X0&H#6K;nij%NMQtRff5nwsqkbn{-ZMIQO4kPnk zJnJ!irGe(o#Mo0)FWH7$VJkr)JPw+?$ z7#*WWcK@_XQ-Gv-@%SExB%TGN?iSwP{g7j!Z=p!2J_5~JSzf()&_ezrCLdN*g4&JL z4~}9*yUWA}n?=eKQgnWSG6WbLTOrXBEHp&^PIfTtIYdrQ;xY?7e#&)#zHr~;}{(!mE~$t-JcNIYmtR3{OuwM86i3ou%}XsjxPrB zEt>Ma=+MYhK}I&=8p8RKE4M{Wf#_3hW=@<(mW&h^IL+WRqCMw^AxPmxAbcW3uVrne zC_R3};Zl@Sn1qc|tg!^BIPrC`P5d8aO+G*mIUbf3-?`7p#$vm7q=O;rfqi*kWU9^> z@P*@?$uqdniFq0$M2}oEq}(R3!v>40mTXiX%0=0?*_EHdmo=Syp=2&!%+3PhdLwz6 zaLf`U$|q0F6%5$H^GcS*gp>nIe*vCCWD_d?i>0en&0gz&SDJgg2Ylvf=Ye~xCL+F zPUiAV5JtLMbEB0XV)NYwPb8GBDHb+bZ_1`^klt-ojo(dd+)?j3hzh_TAvgUnY#jQ_ ziL(|odgO)qI{rTstS87hJuz%e$w_u(1wdH|ETo-bYldGnw)vB+EggTy1xO}Wan)VJ z-&-J42?ER*pY`$)f&Uba;~DXk=8g0?&`R%ZYb9&n4@c#vL%&7FyQz;R{0&m{uLX@w zFq�UuxhEyleX#&QtC?Asnka@n@l7rwj)U0V@cwB?dglPaW}Jd~;5InSiWxDlY9& zWCEUDLk;|pLbCP<*E&IX#ZiBCsZau)aYzMNiggqfGJ@G3{gpDcDV8cw1@}YjwdBH| zoVYi*&Mr)&RQTQjbM5_YB<2^1GBxzk6`-@aBbJ?H*X~2?87N~^9HK%2vpcM+PTW!d zZGL&#q0{l9oV4bNmL|FLZ%W<~brax4mnPu^(L`Rv34qwJrCNRuLiOw5J^Wk4#0|cK z&an~hbkvPpCY>L`Gy<6=qn6({O0aCk>S_g}P3BDxYCWL03f5O5&gPbD4Ng7z?UZMT z@zwm5q;IY#TWs>OaR|Ncj>#BfZ7X}YR2&z+n6@hP_-Tkp4ii&C5wkd1;zOovMdQi% znMJnS{3JMQI9oZ595Et%G%1c}8qU;vhMLQkPaVm7zv!OuQG;zeeN42}}-6&Bi zSL0_w#?+QVNQFu#5nW^*9gvz(9?Q`*9Ig6g5a4PkJ zY%-J&Fks5JT9#q;V*al_sNmnKlT+h5PNg1sTZ5_^F135JyL9`hJFAZSU$>kCF=}9m zoZsO$w$9;ihhJ$rHjzseIxdgYBsbvF{)rM+6f&w32?L$DMivXUxA4%2(2DP?`nc;^ zXI{tjhic%v5M{V9v(2&n(XFBR)K&jSful~ne`Ig{Kl+f=j%fl{yPOfP2C|q*TaX-< z#wjbq+I_pLwwF2vmcv*?Bz!nShkr0~#v5%+czqg$hbyRjRLgZ zezIyPkR9a8maDP=L3uBc*LBA@uwo%LNp5ecYu&@2G&e;Q}kI_>)cHt{t7s3nAO1KJ<09i2~4kV%2HfgH3mw_iPsIw6XO8%nA*mo+Y+#` zZ6!wDm9N>1E+CW7RRtQZp#OpYr|B}-oA6v^X1c#zW_(4rrQ&xeUK2URsNiUAoNH)l zJC~hmOS;Iz7C>Wu!qSux-7G&%W?222jzKM0yT{p&gTH4-tnm-rykd@GkrLRBYU0NE z^R#_kSd9cZ-2Rg5<1}b|^?qL!u?p&A@+8%9_u<<;5>e7pxxLfRHl++%b5Tk=uvn@9#4u zagebQF8uN20+qiJ`>`2$UbR7)iS4N8Ue+3Aj&rW)W~nb3yYhU^Q-s- z41sYu>?i6&ZlJIr{6bd1eT^dUMQ{R!%iDcEMTF*}fRu<2GwWb5Y2;0dU#`!T-iDmg zp+&&oOeHcvrCiFtqax#AxZ+C({(ww(%b@+o$;l?%zVC0j-AZ07I@@}ro7O$; z+p_ME)n(iJrCAK~XKiJmXp3H~ULOLS*zKj74)QGb59<84SgMdjQ(EYOMKIYpr-XG(p ztmn+{3`$`t-wzg>vVY)mI)7*=r+E@_LBy@%HZi3p5HO`dtY&0p9=4k#n|%o@(E{0; zRz23D{3c4N7SUmN1Al@G2;n7iV?VRZ7R#~E{}YocY7XI3gdJenb*T>nb)@L zf^9TMXw2BQ9<-z)#J76m*u}!`u1x+l0BhO&AW_gcf&^t4;ei~J;a_oN!qw_lgPJ`S z%S_gla<}0g{w`DvdQwHx@_^jOrBkKR;FKkqt%ecTalC>zgjJ~KJ_f_kSyZHtlLov7 z^=+AWG2SN)lpG=t9H-*ZL2SS3kq6!o=j8q8mi9w1Yh^0!dE8R+K}4gM7RV^(%o-< zXGh0F?O9vmm_qZ7M>tkM9N%h=|k-BkDOZ?wt%X--$QJF_JQpSnlvmUMuEaIaFG0K;2nj0{v{e^R{f zSuzM@uZq*bsm;KxAqUPFHDAe2Rm~c+DBt#6#W&Ri2|v8-cU&WS5u)p=fOWP%y)(JS z<}1&1_38Ks5|M1oT-0|zPN=tH<-s+3i^YBmgu4#+IripeXC0HsJ>vu*HC#?u5ZCgK zr7_kn67(4*eV`8rJ3iR?yblWJC3>NTPW%RIsg|5bOI&`**v-nxi%F(#H|O{I;c($W z#_RRx0y6H0l;qky@%`O8UxV4)8Bv1{lZ7>H3aT-#Q6;-sMRy4?Bij{yk;Fy` zx^{9JSzT2Gdfal!-Sl2VYnEv%i>xDn^6C4J4vyyPOVy0Ufh5t`o%kA;Q9Yv!SrT(d zWzC64n8*bdIF7*r35bE^#k98^Y-y78wj?v+)K$cfxWLJB@sOrO4a%qHf>t2$I=_#- zOP=t*8|3$3jXe>5y4t!u-3!yuyi7TjC~&^5<`wOq;}+G4aQ|hp5!r|ZA+nl z#@xzxq{4a5w`@G+93LvZ5zGGjXKwxn%n{^9A>Vt0+UQr>6W%mf;VVaPYN#1M{D~X# z$3#(KlMAK+6)T}1D)KcOa{LI_YzuN2H?06$sQOIMFVe@zsw(&c75=(TY*aJ*ns_aS z;uCd${I=-Z5CQ;Q*`>nu)<1^q&9dhKMG@L2`2GAHjjBR+ou%%&zCk{yqtE%OLxK(OJtUn}5DU&Q^Cl?!0R`tsOGF%J-G4=#*xovw4P#SN2N01}sRx%yuUS+lf2GFIg=S7Y=!7l-3Ls>XDkD6| ztg0jLb!stvQo?{_0QU^uz?rX5KWvoECJ-R@?O)iyF!4F+jjj$}25wm08fYudKQ(*h z4fXukr5Bch<_fGPjELO| zA|D2)Tl6u8dB?9|$>HzhAPiH(T6E|4GE<#ub9vZbt=;@QP*omd)U;uC|KPLiP&o`T zd0*JSCy50s@6@BHJWl6%&{cCea7z?|5J@OPLz^&Bb^vqe=XlGz zp$4Gx3>r6$|ce!kYiv8gm{I8d|;2WkK_j}0b z?mFr24Bxi}w)746Mv)p8A9M>jDXu_QbH~K#*{7Zl!9_XE-t)e|^5q?x^hme85>`ta zz+Ct17mAi%o0-7|;0;#AB=V9nP!l(7yjQHnBRn7SYFi#nU5@~P0|NwwbLAw8FoT<1 zH&yMO4E}y+v$tR<_c%5n!mabn+&k__;i+6Puhyim&6r{+vm7ISgm*y;YQN|7tqM!j z-UY)|K=*&@KO*jBA+*fEu7!4C<;B#<)0E39T>YaSk6)8|s%bO1B}CS~HTwSZ-6fE8 z4X*#1Qlv?~D2h3$tDmr5pEF+^@=Grxq>oIfFT<}EKNS7# zcE`DX22FF`N!~+ zG6$6pHTnj`opD8~(_~=|S55yNP8y2~B+~KUGWQ_x03)ykE*DJ4eKnOK6bE*{dW6y= zS;~|q(v`I;Wtj~h^nk9*rPSKT<5ETR{9Ar*6qO@8@AEfVh zK$YX&F06cVZ^4&V1i7yS3Y^r#dQa;do91xT>g0|XjHd7;Icp^&N=d01KO#o!d>kMD z9#x3AT8xAS*9y)5p%q6#4hTn+U5K{DY_70E$RuPq0T4CkoDJH?Zu54U(CkI3F~z*F z8$zjs+S)m)%!Iqq{sZe>g$araUxJx)z1D2EgCZXRLow|FJ&3l0>PB6~Fr=2d8j?x_ zWr)Rw2wod(tY=CuA6F2}=emtv(Ks zp{K-UrOtCZZLuI!QzrcN5N{dmBiyNSL1M(EMoxRdb%o*VHER#a4rNu{&e&yN7jEAdAN8O@2Ys| zSMa=M8Jc>cbV~l*;(E!d;UgdM%t~Yvh6cI;Abvx6XfiXl7q` zN)dhc@x7uXVge*8)v3K#t~bKXYV+1T zq3%~kj$#VTA|CMzbJ8Tqz1i6|o8!0F|D2DnD_6iqWlPLI07%G3?-C}vxEQJGu6Fbf?zEAe>9?`6y>lb&l4|wSZH$$N&U))&Yq!#=E9n0 zLI#yYvp1kR8q-KX=Wc|1%B#7Fta<~i%Vxz=Z^5rl+m|6u@4V2Aw=P?Q)2}^n0SPXO zCJAM&_)Sbo5kXHB%|NT3uTK6MzK0f!R3v>4RGR zTe`|m?55+6DaX;7sM4H7b3OV*H=)jw6iN(Qx`|FBb(%ufyY3Az;Csy|}E>mzfuiP_2V5On^nJKu8up5r0{#XPhY zC*PQ2ZFueNz-2+19bt?SoteIWV}fcY3yWTIj}0tAb*kUSjWB#fa)?W*KW*r(=5U$y zl>|HsNkQFU$%iDu7;^?-T616Z4Bxz!m$eA30=cVqm}{*Gxc78c^=R5(bI+Ljg*^&} zh>ckutOhHKw)DCs0f(qvhoKw(V+-^t1VA?mhL^#uf0|m>xqO;VJ*e92kK8OSkK8iI zSYF9UQ!CGfWY+^}LegDk`Q1UwRzyG`Rn8rE?nMxx&XH7R%wE3jUy z0)=HqBdTtEr6{tX4VN$K)eau_Yn++~97wVx9{JZ`AbbU1;Te$w3DyMGC?U6de;-A- z8Z!M)SR5NuaVJ{RogqXfsaQvMD1Cw9xJ0#;vcTtwt^uNZqGm9DGX1#}Ihhvi@u^AD zTpYK> z;KM*4S6?#%(5A?+K2_sO;@}Toyv*^P8xia^jo#W-G+A)vC>L3cDb~vN`6Li~q%?Qz zEZ+e)x;Sc-6HCxCiwbCM7nR%{X3+S%S*}dwxa@~ZWT_Bggc9y)i*t4fnp?HrzKmYu zk~Qlz=(o)je6hNsc=2+LK^j!#AOTG%IWOEmp#2rfHX`)n%_IrW)Q%G00ZfJu=Wuph zmQp^B{W?y1U-P}ta!8TBJp}bn{QyxnTIJ)4WXU4jTQ&ITIPa|Tl;PRCaj)xO>YnHZofj9RB&bZ=zV$uT%^ulrHhE&u_U2O^Jp!i<;(Q9f%l+(@$53&Y z^JYFwojMhjvM#eF8w)H^(*pEH+XdKRZxB9xKHfZ{!bk;l<8Co;dv~bqT3lfCBd=rI zZ`;sTkI%k$QTMmcV?U@YZcZAD%G+7 z!||WN3Gz>lE)DIaAb>O)jvF);GVzL3DW}TPAM>@Rw~nM$O{KMb7)gdq7us?*ZI7jx zT&w-OA>mtj{g^hW?TtY8N+`q29(`;UCeE?#Qa`X0A4(rkr;@xo1^2y#e@!w#WDH;z z4idJd^MQ@4KP?ggZ6m7h*WQxr)`~}DGfYYZJgPX9#TczJ<47A=3Bjq_UHr4>PNxo6 z@OqkQ*mQ!1>n6iZT0mCu*u}FRFFe||QVgqm@_GDXBEl6CG=wJ1;4^d`1XVx^%)pO< zLO5phJ-PiD?hZkCQ_HWIfwxiOB54l&Y<(L*6ROfY6J-rH3mi!sF%F&wW4eB$eJP@5 z$IlY4Xo|)r))a2&bU}Zx@>>0LcJ%OB)>@k7xY7xV7o~pb_z4Z+6dl~qVNV>fGOi75 zpRWC0rp7z80bz0W5beD8*F*>{8Kyr2L`W9vYEV1Fafg-f83MPEPKn2m+@mEl#bSC# zh`vORbX;x!UHBHpC}rKH!e7bH)Q zbYB5sY_UA3H4VPaf!70?kab7;pJoph-PPUG?Ai6rmG?oHUWn*06;BO>hgVYArp{P2 zkH*ANq{yN57Ts9HJl<-EAjCx26mbEw&OMl#)3ZfbP|PfZRvX|jKvYg9Rs)TW^~b?l zbnVq!e$|lgUbp(-Mj<^?O=j_RC?;Or2nj&wL&4^V;I6a;ZL9(eWn~_t&!gn;PbK>? z87O(?vd8#_K{&0SQ4pLY&!>V}`eMZi+&b)KEJnV$Y7X9jjB{(}9QZXx&#w;LeFw)J z>LAHVRVRp}uUg|~{_o(XZ5|_`l{5W|YDYD_U0!)daan+Xao(51fy>%gupeOj@DJ#d zg|WWU5P4T4aoXRuw^Z`;k{{9DAxAXk0+=YHkMUv>JQ^sUD7WtyRe2c_vG4RRqQNJ^ z^LTX@_9Vr4CkhM$o6TRFx41A#g)f`-ws4=-(?`D`jevYSKJ{S~Z}3#MEt?tB(LG-gy6 zg2Q6(e&Q@!Rbl((8(9XLGZ$d2bo4^?;T-*~`H%RxA4r!zT?yN>5srEw^`fn=VydJC z*+Yava9FgzYZnI(Vgr>vq~X5@sGX-Tser^~n}S9r^j?{_B!{UG_o@Qi82=70fdLpksheOQYBzVl2?)q+5TXW=>lZ~=o!oxjP znPeLP|6I3-1zyQyQL`5lJmA(Usj4SsOd45yfZ{NNvxlYQY2-9ta=Jm*EZ3%+soE%c z5hzPj(9xw056W;Yg~F$4@w@B)Gz@ca=+b)nQ)t1v=zMyXKYv5y2HKdP4o?l7iZ`dr zCDG?bLM851Vt>i=VZli*6%K1X^0Wj!&x-W($x+4-v6PZPPjpko3iAHXWq(fA9Pwfn zR)O}PJ9p<`tVRWT_&oOa6Fjc?e#2-uExgAx_gbM%$AmUMtN<*6CmoPY_cl7fRJJ@J z%e4Ihalth1A(ztu-qbp{L!(t6XyehWp@*{f=TzKVpMKM*hlfbE`a8uclCw-H&UUj_ z5ef-wGAV8uNBJSWxFa&K?7C)O>}JI6Wc2FxtXjMeA{Y8;ho&~{U{HZ)B*5H{$wBg~ zjtuEMVJ;l57$>xf&68dZ?$46#xBu&Sugr2|ul3afzC^FACqNN$(R6$^0-RFs`> zjNAr#dW3E*DJh-ZUd~l#*!sT4s;mCkoV|MOq$oI{r&>{s{OG54mOwY?d)9trM&mzpb6;hVtOSeUblj0K1%SM6b_Jek@`qiew+JrOdFv%yq}9Dw+frPk*ic zi+x5U1tZJ85?h`@b;`f&tCF8>`1Z&Y+fuh+5&hq7or2**KmvcgTmr`!{hR=4mxso= z)XB0?Lija@rd1-P27Mj7+eFb<7{|F^W0 zX7t`#@4h+?A-C@o`+TkE{rsY^#|XK0l=<0#HA5`yxSG0`n;t1KcD;*WFS+N3iuSFd z4@0>b)-5hb|32uIiaz};9#gCvRd_N`@$!y;)jUE&;nDjp2IiW~=GvD!_w&c+Nt3WI zoc=s|+fvJDtHX`U0g=k@fysrjtn0w9PEFOaLm#OeB|uhCKAv@*{!3ZrY#Bt`_Pn zyl#?ylyEMbo@bP&nABFgLbnC0Bf0Yk-soX7gR39A^oXZKQClK^ro1sX|EkkWTsqfl zcT4H!F395Q$c_V&Se%aGfp^W&mVIhp3AR!@sx+3I76Z#@cOWgfE6{$k-{Lu0|I?m* z9?N1(kL#Y%DUG5Z=1thu{i2u9(I6CGY_XWwV8@gqyLUhR`v*Lg76Xzue*PlfFKa#? zp*)E13kMDi6LIM-uN!4fA>IWw5_1iGzNUmh(08YH6ug?Tz4@y7|L@5)PJxBASdKz@ zqA=2h;LYx|VjqvvBXi!7kEMnb1MQG5nFV9yi9O?5ma1QWbK8%p_^b4@?mST?PB8H% zsDfgq#_|v&yZM;SWnsx+lbc`jzgHjfq{@Xqs)8l_jQu*H?U`N2w`HDw$B|I3BRrU7 zggTO0h-la^ulc#&zd!42D-xDe|tu70RG-{9sg zZG&#%vMk@QkKN@>j`458-{SF+c=L(iSn`6_fKzHEhJTdlxm!LRszGG38Cd9WyL}}& zlVpH+38vs59+1*;%hOQJ9`L7FHnmXtc z*b}>hXUVvU*0tF7da(ZcP&!QzL1q`%u{Rhom(3&Zx5mEFYa-dFJs-00Ukc1EsWbC;T`0UxdtQUu+32*bcP*Qg}*MR~HgXBGM!gU%55*JrDI z7ffmWMJ7e)W*yddBMOPPsK?Jdi~m>^1xPPw`!#xonZ$~Z#$EBd z)ReG=Qe)qCm9)dP$*5$63(Gpchy*sxOB&CPgq7WFq%Cw9y7DG07rQQ4 z{5voJ4j}Kaq)}0-M7@ue3M0d!OvC@H^PF@4SzFW)NL^DvnbB;&l-3(fy&g++ILLNV z{VciGTN}MfD5UbjYV?=-5ttBa${U9QFFX=0_GgvAuo3e@>CdpMW*Z-B750|2E2|)i z?CsmQ4P7ZQ-Q)(C@>b}TNtsWZ-ZX~jf+gILMrzyKCo~6O1=B}n@}?2#uT7U@?jFV@*t|uWs`s zPAy1BREnoW^e7pJDv08xXP0dr%ix?@6mPInl}fIeXt%hf?DCECQP>f7f4z9zd;hYe zs`ppu7-WOnUSuF5$-fk`qLKlgRM5)BOE{?>p8K7RzmLK$euKl(d4N~Z?y2Cw|6ajw z&dQQ1rYaN0)o0t6wynFs^R+rfV?%3uWif1{uNyL{))Znmz6WaGoayRsUJ5Cp~q7U-}wv@5p3|r#6V8|_ap>o#mBkh|clZD$Q<7EX<*LHM3 zbDcSHDr8pKCy5H?xO#Pl)QBzH-zutgCjMCDA9USvvh1K0&nw44derZ~^;QAFaN4nV zM#UJ4JTqD~SM2=alk}!9Cp+1jI6t$5J0*lbDs_y`*XM+cJ%GXMcNfes&GWW&`q&@& z%6OMuaH#J+rT4-btx7@=(a5FDK?GWjlboP zmD;G&#m~Qy;bVS29x5fCH1YDsT_$Ok^G8I>Jjy!mjm6>LVckLprF69y zjriqUjp7b@Sg9BV$GJ&t7%<_(qC9bP1(=erXEURmDyvqa1QXTHP0_8X=D)a4Xa-Po zk+e~_gmovwS>J?;#+D~C&IkIQ?2W%~FO}z=XB)NUad#n@+mr|ntDKwW1JFXEO%=O_9dLZS-3C;y!zA9mmIY)-cR5#WFcbo^%Gu!58V7sI}7u5ij?% zfiTNwm2a&uHDv)s(4CimpwVU6H{MGE4sX|~a_)s*;+BNV(zArD*mO@u;r z?sitW2xxxU2uTpt0kCko>+CrDe;zSLkXN;^GPnTz*pm-}3d2TTy7i)_gCy!pz&@va z0)MUC-tE7zu)Bi@82A9j9&d0IA;oqE6E=4MUl-@{yrmyKi0SKAf>5jH`fYAxKG)Y*Y-K_KrhogBCRr4|V(@o>|OG&nW*l9Wz6>n?s4Yqeg?uk+b%u^7`wLdl#LVr!G~d)#I*oY1QhfJz38tyf_e;`3B0 z)IA^cG8u*N@VH;oc&qBw*Iv8fP(;(f3HU<4&Scyel8YA&$5mv*F+lTIE$j z(x5yf;Ab$5?lPKNb7!H?UtY|Z{64KAFcbWc2Sn8o-DGecs9kGz2&gKWI89Ubg{d^J zt=T=VVa_9?+5l9tmy1Sl{4>G*63*>>TXBuz#xjCq!gQ9a$E3?BQ*-p|=LyiTsWVL+ z4J+VYH`F3|bwtecdGH>A@it3^q>1jm@Omo&0#B+lO7zUl1OE#H+o-D^krRJ)tVpER zh3or+l)=XA!2^|yNgXp-|Mq(SL-I&tYW;S5F@lOg84k(&C|N>4a;n~Rm#v4*C{p3%&^c|Fh;QGl-7WgE^MmpGzSgZ+?~Me;b}rO?rwswEAr z5=-9Ak^#w8*~M{Wf(X`_NfAV3Wu~Kxc>(W0m=KdRUe=Iv$b{vcEU9+- z7s)EgsBF635&+ZpTu>g%m3p*m)m3u(+j>N#O8E9n@I`||1Vc$I`E$}^=q(OZY2XK| zp4YJfyUTh;Fwm5j87D&s6YTo*8$y2E=fSUwO8ucQkuXxSt}NE&1>L6kzI}2a`!IPA z7$5M+hpk?Oi4g&zk(=(e(oI90xYk^g`3dgpC!JhB#5PK76ZgK#i zWE{{E4Brf2-3~OF+J{S{wq3BUwl7%ljM{w5X1i91=0mG1y2Yen6lLNMIs*Zx#({lA z=kTpi-vJFtw|WNkGrtSPw8^kEhL%=)216-FZ{D^)DXy%9T#hnY zVZCCpl1kfB%6!F}*G?qR>kG}4!nZl^F?0(oLpas2nlxr4CFoB2BsP))ppJ84by%EV z5xJ_KqheqFtxbMfGWewJe_oxqe*W}-OpV||8_XsvV7UE9c=+gw3VnZZ>Y035Nf#-7 zj$-$(p?LS6mdk2BIHyE_vGh(D5S68u&=n|-P*U>I-`k{nBH-REet85ur( zC7TF6KsD0r*sJHdDClO4TMacJ$hX>7BPLnQA)8mO5O(V<$vWP5upz9+b;pyCIpCFb z&()9PH3g%H%@x|FXw=Jz&49u{U^{h|7!BIl>+U2-{o{T>lhD~B=nu>MM8!Ii)o?va zM{i6z4NwhbZMqCAuCMd2hw^}>f)<%A*3GTaAw!6s$Vfle1%?3lK?Ky|D9ml-Dkq4o ztca;2#lFI%x~b)ePAi)6z}kzzhiv@S1)tc&%3vpzBlP$H516R~;aAS0+;XXMrBC7| z6|QYEi?{0#EUi3otvVYb&&*db8~SzZt43CQFjp*fOF^}fp!_5j7?Xl9ZG$?ezOFd{ zHCsFDCqgK)RuO-mTMpXj2vBSdkW(t07tj*kaW@@ECHR_)xX=P-EUlPqYEg$?ur*wI zkCzJ!fhXG-(7=QjLxU{BlCGJguPjLTV_4l{d-(>Tp;KWYSxEUVZjW#D0_-0yzc+Gl z;T6+W%3MVlfn2WFW9AF43(QX9h^8>v#eM>s%76}c42XmKf#@z&u_lo|OMGY<{iGf3 zJ(9~B-o?aNqyUaqRDnlDVHYZ?yr&i#sIcK+pvCHZu#AuX8c=;@< z2?o||02d0(m;wvv)y4l62H=Fg%c3-)$jX4H5q^AjK-}o&VzGOKmE7O1?!J3;EA_U8!U<^4bFUUeq=Q+ z=VKR++dk!=mdM(2tm{Wyp-KvaoY~)o%iSY(1x{J{l71(x;3#w72}pz&wB_WJAo$FTl&AKljbHk;vz6EB3_2 zK>h~+&UB+13GFa{PXPCh?Ad@_6cHZ%ynCD9(GgHRB}ny5wD#NSBfMaEi1=UCuPn?& z&(1R)nj)RCFPFq>MwbBxTIZ$tp61<%9V9bM;7SU;Gi#*Asx`Z_tst&3TNaw>@FszP zR%N3OOr-61=@M_x(!`|5_sB3gVqM9w0H{_LFd)FKmN@3_N25C3_i{`<=a9zzeu&6j z?o4Sj;!((q?x zP{~dv`{^f+J;=zV$5HvlLt>F2QLdCo{ERdF4U!I|Wl8u?Du350YG>B=Yec7zRClw7 z*vDcA%7`Ff!U(sA@aRm&03;gJ_mS}wcI)rCo=~WNM@B9EX!sffnus8M%ZtxUsUpq4 z!oQ5ca#NbRoVaWq?0k#~5hz{t%BARQ`wUM1~}+k2usTW=x69IC4wc-C_*BizAd1l zDMvK09Jm2N_Y?oisBvdI8)jy=yJ=yluw?)lUb+c5<x ze6cvj3LJq}3AV;0|N7(QC0y0tH~Iljv1!{Oxf){g*=GL0YKVL7a5}B@g-&)T&LPy6 zWzG0Oc*5;iRU(1&hN{GVZzzJ-AI+*R?TYIkNPdCi!cvg$R6i^3XWcl=CRpv+ zk>U|fa_owffQzPDUn-%#7d38bHfVn&YaB6w*Z0oJD(4mAGgU zyh6uYJaDe(w(R}WI3BoD+42Px0;Dq?hqPlupTh6oRGPokK|TPBAga6Lf8wB}=7t1{ z>W!XO8royyzFSyr;U2buhn2N{IkE|e3#7Y6+GU?io7V(z?ZT0#*`svL<0HlKaL%#^ z&@S{xCbur-QGqf{2g}G9q15~|PxURMPRC?s23{}Ssp~jEs=bhf9pb82;2!W~_Rz`wWdkXkQ! z9q=>O8)~f?IGG+Z1%5xtq>9a=ZWEqtvy4YOn@~tX(FHvCeQGK7<>2VMvV_icq0kPU zp2i2(ei}9(fe-Ryhx{rN;SybcvJ9fY!p4?jS7ck)$%MS4=&m-2ZV;V&-#qMZMWwPe zA0LLaO4r|{#v#to@ZJUFb5(!strRwqc51R0fYNAqQL|))Kh?&F-%CR zwo^IZaSODS7ECh`mlD~H1Sip6EQkk4JS)MW_Q_2N4DlvST#W<{TKj)~3WuiC(aUIq zf7?^Fe6j5H-9%Ayo(sWoD?Ts@{dM7>*JdMe1b8~)(Y+zndy>_*+#3B`d(R}-oxzF> zKmdlAU2r*WG#xUAG{SGbI-?S#J;^7@A^O1h59Gr2kv&vE6SS5 zqb&0aD}To#obqi93p6@zP?vRI2tG0P^>}C1F}Ng8SG+~lYks>kirHIqX`W2RW^~>D zUfc=Sk0$NAJ|}E_59Wv$K&n~5ybH1-PI-$;_LgNJsi*g-HOe!(w@ZQL(Uls%;d<|d zlt7xLDVsG^Ydac03%UG&49HmyMe$FHlUhli)Xjs)kQrB5j!}goKJ=f4KoQYmbu`7m zr@w{zsgCK@hZ`jD%v*KqDDbPygU2+K?Fcc6;J^q$2Uh=QKunUa&A-Boi9dp{6C2hX zDDl6GJSm*wJiCc7q!yMzG2#Nmy7Q7MQ6Le$N3QpyzQS)n`z-b~K@plpngf(_YHYq9 z=ZRP5o($v>Y)mU4dJwT!1-w9#&lMxzbM6a#S%cCh-EVHX_df|}#P^;ivE1Eb#N0vC{ zL_vD%7|ok2;L6Qr7FI7eZQp)3Hq6^1_?C9)!hmM{B*j7ZSR?GzBnNtV{>0ZCSW@&_ zoIIrfX)gQO2b<~r*(i#vIMc7siT2+TE1- z#GOuC8Bu3F-a>FoH2WVQ84p87QApH=#y3Lvcr!XYjy^uLu^k^`$o9$HH4L61c@HEv zu1$(u6&cI+i^|?bQ-d?!!6NlkgOijG)P#>m6CgJ4TL^&yG@wT-V#}F1$WlJGt{_S# z4GSV9$%y3F3oDAjaxN64fynGnW``M`J$!Rs-&b9+F=iOUb|q@278A$%_0;mmo8+;L z-QY)cfcXD>83JPUWuxaC6Gs4t9h>mRVTP5m-*ypC*6i!1;qACJ=kf)~uSUstS12hJ zA|RinKuen=oQ~Rc|TdAIy7bRv$@!k@=o&Cg*L4F!gdj;FTsTsRV2~|MeX5=9ad$++L%Z=D_O^CNV12%mPQ6 z4}uQ}clc1+=62p+OAG0H&w6Oa+~t{dm#YvK714P{CGPnY5(zF+jdV9#wzId6;GK78 zlF3FxnM8Uj&kV^Qvb=yJ*crdt;xw>^ss*>W4PseZ6^kSph1!Bg_`$yOW~VjJS;K&k z%5A=F<%gLCfA9s7b8?&5E6_O)Ygp`YA5;-lhwT?`n466n`4Hp`d{CH~HZr;A`QzHu zCm56Q?UR$WcLzLq?tda2drJ6Uque-`WUn2wtMs#X-@zW%?J{Od6o5#m9VOo)>SP`7 z5Y|~YBE}LD$ys?wLVjxd78o>iZqEEb?}Y%jB^rf7)d5voI%~gb&hsK})Qhx8*U{@e zR+os`sFaJHE4tirlct*8Ru$O}wQG9=MkG zGk6PLC-bga=8Cq}*72K6sQbUHl_>Hq*O6xZ%>};gzdy=wOcsBfcwOtQzN{OBNgK%B zrW`_a4@CouXFWf{0Lwd4%ueyQ`P zY9y9C!-bh9`eLRqN zdePAee9LCv{jU(T{0dX{jUIPtnm~+{`5|Da5DV=gVL^dP6aGtne1Utf3+9R(hwTyf zw7b_a@YHo2oW27c!lKYA>)g_{s@L;JbyiuOspt#edm!+@i8_|_w9W?3P;q9xKXv)s zDTI=Qw&Bv5DE32R-Vtc3H$cF_7tr{&t?$K{va!XB#x5BdlM6lBNJ(GU>4AQ-1`Oy| zEj@iGaV3&|53W4{4=gk-YZ45^0ZdLWn_hF_(e6W*#RW%s0n>xs&j`#WJYpJeG5 z;}!JLoN;D?5rYVNE(DIXKxllq4bQEjK?_GX7Ct(uoR!eo!$3@`v`;uQ>u1^Js#{>( zizo~WZ`Z@8@W0n5f)4k7e zmhp6}C7dqz$zubQQ-9H|cfQHcLz1vztE<tnQgieOdBSD+P|aPYGWDwfPo&=fjc$@jo#N&GH6(wU12lROI;A8llC9tuq6^DA4{A4$+MFvMCSog72xQ zfe;-WeQ5&VoC63m4nM#IjMf=K1CT*}x=$wv1!|=S2dPjFH&5d*ZiD#Eg_Dj`IH>98 z@@Pw+34+3&(2G=-IfrW$G~#6QDk~=vCE5Dn0z-Sjhc>&T&qFhZ${_Q9d-mP?j%9O1 zd)?5}gWXZf5o(k1{}cHEKZnREO3QV}`}#cCrJF|^y=>lJ2&0=Wx&@s&AyLaE3VFFC zPTy5N2@uw(5~-^~9ojK^xm9LBQlcxRpkQYqooB%!wo~@zgX{&smFOnHFXQq%*?7oAWGIV7{#LiS7c3DvEIsPd>dJWa=pt{B?sbo zFJcJpj;a+Nq2hQ~w@W+ZsK4!hB-c^J;^Ix}9%ZN{*L!Au1}wFR7?!nXSE$Q%=9RU5 zn`7&B)x$tP8o|5=ft?Pl6-U!3j{PU?gof2d;~r#?AX(AL?b$L=m@K`D4e=<^&ilKH zzKaHhXw&F3_&i!TFaUw62Cpwlhb-`K+>?5wAsMW}0}*vLr>dKiWD)2U`>wd#0bRg* zEKXPjZ8Q=%X{yGdN8Rh>3L-R_h^g(yP!-^AQ$zB&AgR4a^4PODH z$9=v>1~2lPjs>gVU*-zFYKeN5srs9}yL^T&&ZS;_icCfp5b-2QjMkqMOFY(f+QA@g z!phzJ+|1Xp0Rj0eF5Lw?ZaNo6(gS_nwfw5-Vg$mu`p*mbuDl~5M!(F~F?A{Yu7>`9 zBkRatDp!yTyem5$h(aj`t}H7~y7Tw0vODTSJbk$Yun5>Lm`AyRk+T$K)_K*yx@1s4 zdCEgEL51j~)b&!4X-_TeTr-U%2u;rY36E0Dvdg-tT3g4&S4#x7i2@Yucx*EF#e`%> z@~6y93`;r}V!d%Poc4RVWfl-|mj{Yv?kIN5bVNRMy2Eo!@#P8?s50orKwmiQcs}9hsd&Z>5f}PLg zowtPu=;q*D;w~uWYn-EkhWEi(eA1vVveX-w|99*#Vh$mWpI)9S-vMvr}EsGuI=1+N_bSgDf{v0 z#*-fpnNNF>gDX0C#u2k6s)9JfFopa7A{)C(?z3D;(@M3>%itWnvW!XVOum(_bj*1y zj!(x~;IZLP*P@Re@u=y1^#;%_To!r|i_2qZ3fAh@9!ONrRvRMcvxAof6aJMY!1UJP zfK0r{aetda%QEv0il97rngqNR0EuTYJn8=P>J3iaD2O#w%j{F_HLy0)Uc5-Sp7xuo z-|Y`bx;ydJo905J?3=%7nbPTCHu9iq=4b4A<<5St5#12@b5^Kg+fFs4n6F*5P!whr z;1KAm4?hUf($ODOaeKU^8^vy2hY9&|TCnxn$=Rg_;DvO320Y=f2z?KBp|1}Zr~!z{ z>b!oYP9h3V{F?(F9Irzn6X=(2iZrfnec5lsHqIa`7-Seq(b3sGzsh{^_g@w5y|9(O za?B_Iu&ffBJ`Dia{m%yfzwJ*Wem)g9!lBikmB{6;;+JEyq6JQMGYoAu`yk7m0@HuH zOIheDNKeJJDj`Cb9$64c9s^W-bK>-VGz8r+!z9dA3PSoVIY+l=ZXv(N)!;R$Gf060 zg}OV&wNaAlF1&>ZP~q?QJ}tlg+#F)1O61jH0aMEhvSYLUBH@Vj$?n02<-q-ppb%TQ zxIa41M#|No-OZpxJj1U<3jO>}J_!7>YUfX`T1OdO^!xLEJW~ZN5JkQJ%wnky3}WY; z5XI8~ib7KQ0*F>kC4{H}=HpCtcs_2Q5^PV_o78zJCki+nNba|d1OK0-+G2! z%Hx19J%~cN2v^<22LH3jxqBz+h8~Zq^q%n+_L}_-8(rAXBcqp89n&xcA$x|SB8RWK z%B@JOa0=isp6un>X==Y5{bUJzch6@cr8(TJE9Loy_*}AZ`rl$C_kD|lfol)Nm2{sd z+v4ZHcdncf(I47FF$z5Y@ zTc4rr-~6rj!zyz|&U=MJe&W>P{Ak?d+Imi;gjag)Cx|_1T-=(~xX09E+BM!b0qwPG zwGNzGNV}D+nz&JyVsj#$?6ntY_ImM$Hd7QKknNu<_*DoJvr*NdA^LP4XFbY{ppUH< zOrceW`Znv96R7<>NCzkf!gd}0sHNjlm2^a{c$p=$6;h2?RDUuxg#VV58mg*v^hLk^ zK7wt>O|z|5TT_w&$yL_|0USP~sQAm?Q+fAchoUy%Pu6)LxGBfLUF07bCKY$2gc^9H z9GPypCl0SgS&+iJJlR<72#7cza%EGZai1nzr$$t~aRz5hiFpVIxPl&M;!rL4dnrS_ z&pzESbe)8>_b~E5?tt6PPL?D)l{I9-1FQ>2D=Kv+I|P!h>pbSwnrf}BFQ}5wEBY7& zHfxm7_&EXJuNQIj8`w&^lWZiKS=EQStx(=1((_yzquvg`^O6|%m1PDFqkjs4SvZhv z@yTr+mCbww?|@3xM8griFxml$3f~wt;mh~h`s94%R8nnUe5^Ok3T1AZc)u3p$Ur}1 z0FP8j#o%;NrA8M!{x1wXwLrNcUg$o5vwXLJofeZaLN&Cs&a?VSp3GH9>9m%ZB0eIG zoV3fX)p{{97KCTtm1L}lz*s{36N~SkL8~biw*yYkWI5r?0sgV^VH366`D?5?4(Kzi z_tU$N4tG6}9Cm!6$(lByEW)HqRYNiuLE44?@EUZ_W5S*INC~O#a`8YvIzy*m0kwW)M(jeq9I1sYtqBHR(LLjsw+G5nUtM5Awj2`{V@`hp?n1OxN#kZgVD_$2l4>bX%SA-QT4t1uC~Rj6jQ-}w^I$--Mz*upvpZe1vz0c%ct zW>IJyzgfAK9!R#u6Pu+@f}tc*yci#@hyFMo=qgdDHMt+e?aG6HxDX3cT@u(5=yT9{ zA6QX;)*;4rGDwPlhgHn3&7z6xwm?krRR+Sgg%fR)ZBJ>~8#aY+2X!CyE1^YU`yW=w z*WO6C@?T4<-LbV?4EIMi2cZd-aU=j2hSgdww*N21^dA+bvc(2z zc>hT-l!}Mg1#ub#g{q1OC04h#?>b{K_yD3iGzs6#spjMdS$=T{3^yyM)I;}nT=7KU zTJqRCwSkLmqIX2;n+_PM){JX5kt;K+1j=*f6#z{4{liN;@w1MvYlMDe9Vs8QFw8pdeD`8<|iZp22x1=YejF`RqxY z=ug#llANXx+_h;sC6g&(?8pPTdqFUDQ|cdAh!LjkpK+{>r_Qu4ZL6J1Ra zb#{9qde#nLPoloqlj56>5p&M(F7-m2x19eKrU-Hg5PX&%{?txyPXzAYF@wX+eX{85HPYGE@?o@ez}wS=GRI9_TrLpfTOE zPd`Toxa{lE3;osJUq2Se*l38+j0MX9i4uEhf=ES8JHX(+PYDBuLi!8%_-qz_pS=nW z-$(a?RgQ3WM#Q!U!(K?Vw+AU%rPnR`&569wD~9xA0-f6Wu)x18yPe zZ@;oB?3Uo2sc8uxMUFV)fcG~YH>0kcf56s~>VG7;X#>D@)b7JIvay|revEteDvgl@ zO(P0wUy}GwWs?UIiZ?T$^g@KilShN}dKjmU(+VdT8aBCEdRkhfed3(6&T?V?LYslP zl;*ir?l-XN+@H^X8@%9X{BvZLC0X)`U zpea0w`_ImB#!!4?u3$h-xq_e=ip(kZ_b`!Z_xMR*hlRN0Tq1PXH)s7LT-Mri>FxNx zHEBRsvKd0DIA;U92+0RuUNETKp5!sv1^icmGm^?K}+5Xb+{GXE>+g2u9TaO+<^HP(s81?n8005-k462QEsXCFG3ln z>7ajsNP7+h`MbkQK|%mH<V}m}B$kz{a_w@-ZBCtd|G=&<>}eMl5u`4t z309Jt4wYeFZkQPtv%(^Qhu;QZ+Z{peV+YWt$IR*dkMY@XodC89ZcZ8E(^zQ04RCZg zXI+!~k8Zm#)oF6X;ncoy?$ZQ<$AX>f?fQubF9b=^=479e+0gO_V6NwbEpp6{*6g!1 z>VcWJNwc+R1C+pJ2O9z3Z-v}8(1W2^#4uGy{!xZ+9nfpSuaQ!ALyi_sD->~3<)L5o z50q;b6;@6=l>G?M?Y!4AX~~N$!JKw--_HmvgevkVe9JkswsvNHxJ`)E^AV3f(~hbQ z4)PtaE_wRgdqjsrewM)f(8GG!DML{C1E7m2?mR$)5P6dQ@B@Z4Bd0-*cI{hU1tAWRJX6v4)Pl3qx1WcVtD!8Q*Xbj5J_@#uz~ zfG^;j1NF;+rIJokr|wgrW2x;(rQr>cjIlfa|6t6h<6ODhVErJC&OBT({b|W(iI^J= z;bG7xW)EsQC3k0rb$D48^17x&+Anis8dg}AY`cTXqz9hn#SD4Cj7WqVZqohHB{--i z%Up;lM<*Ioq0)lh?%bB}HNEkp_Bt!W+W>1z?3}%E=FSTLC3fbLUKqfK;ZcJ>(rT98 zbartBydTe`6R$#;c4wKPIwk{+J|#lZ=6o8Q*d`|s8N|-eR48aJ<%@sYl8ggUQx*OO ze}38>;QyU8UyR@_ab~3xqX%y&Sbik-^5!&@Y7aPjVH@L~CGdW6ul_^^Sh=$x;xKG? zy^i7t_XMjZI!$BuzzCwU)cqafd2#A8qrA(8xxb$SM5KnS8iY{>$0nx~6j;Dqa(K{V z8L`6Y<4~ZbWKzMfFw{%~5L&wr?TCY8vwP2)$#c`e^;@3xUBjYl7k5FMsRE|X_6rCq z4?3PH1%yUj5IQ1HfS!25R-0ZMBo@7SzkN+YX=mpN++3#d_8fH1!5mpbXPm?w8zCJG zUaqX10BaTYW=Mm?^iKz!yMBt(6exjI z`ctjYlfEU*%@H8Yrt3$CIAuZ7Kcq1TKYck>p&;ONHfCOKnsPDrML7m<{eO++VyniG zyVz2pmyK~wLJZsGV)hfeSD}oz=!L@VN{eocrMWj}P-SKM1>~Zy*fvj))}nlQVH)vb zy(DAW%S0w8zM%Um&qP((gs$}r!4OC$P{b5JDz2U3TH+4b zYy+d`-X@m-5L^fj@mW)_>R?`x2eJ$R)s$d^SAEjYzS z^oy-jTeH@Si|wXWH?wfE*!ZCGh!JFce!a}y!2@J!&akYqtoT-HJ!}M)hNHq0}}$?$731$GOd8&li~mERaa9dBI96pdAsKhL@*(k#yGRSztJM$-&SBfyLj@+=X7`Vwsd3Z6s54q$el{DJ z7kFVTLaU|YZPF8CPy3c;a2*rA4L{O0mgF-ou#XCyEQk|8uD5ftQj~>P8b?l-WWw#) zW&H0^hXN5B=GX9`;7L!WY%WVSzUXu3|~{Sz6SUoYHan2Eeskwn@LA_<2Ya zD^h~6zgDt%RYmJP5fKt0DtQ#iU9g1y?J~Z$XVywuN`XKAW(s7WjPsXH7V&5Q=>|tj zj?>!UMKn{u1Tr(h@dEh2exzP^{qt7$`kUVXK|sF0@S(=s7*kKPeL%V74M1hI<=5h9 z(L=&b1C!`8)8n(o4y*qY&22@y8f&)j3gQRjQ8y@{D(oaQ!#14C-tk>2k9D>#Bq)-& ze+T}597JB*Q73FKgN}}HCY-Q@Y6Wq@)R)6=9}4Tm_rS^A*O8yYX;rpXx63ujNU!@Z z?79Tn8Xsr}g71DNn>FhRN z2cgg`9+dtA2#qT$P!s7QGghKzh5uRsrQ+MaJ+t zO)fAhdIt-U0W($@3cdOS<&Ki@QUF7_vZ(%w>+ z=Y*!ydAie|UTrFU)9KmcI?av5W}Sd=_{5!0J^8M@0jJNP!vH5oHdS59e=~0WxBqZA zKeE%tTJ%;p*q9mh+{}pn*w%+fghhX^8hG%IRvVKzUP)b}wMz_+e#F8HzZKXbj@^G; z>+V+LvH;4{*Yy<8Vx<463;**QEUMQ_#C%|t+f<#xl;YTJ|EL3~c4}X(hr(;l8f$)7 z#%Ilku+R%h>fXb@p!~~rE6akJrj>ojXhZ_qg%`XE;JM=<&7Q1yAT$TX_6-qGpu(nq zE~EoPubBgF^>cib$Abam=qJ$ck;APRusH|0u5V5Hg>3?)`0~%2{nsCmC-!xI0d2Bu zt-K|djx;qNs9psoPf=|M8bbGU9h|%_wuD2keoBe6mhqErlJHE% zGok5lboYr+xQ(IUoeJpc>K{5cwaE+S57b=$^-zlH4augWugWK@t=;pv;fBP%{Is&f zWYblEx)G0RNv(xZK3{F9r2Rt|Z!2Xy$sf@S1U|$<6|?p>$r7aT`!wGlzs-q(O~=A^ zJLjfgq3_$n(G^L|i6a@EdbKF9q#z}U>WQZ7xIh6yWN4M)wcv29T!j;0Ip(i{-0tQ4 z@nkC&?0(-UwDqEs1T0Q37h$WzI>(u3>w)_N*J`;NREVdaG=1#=tMjw!|Vl++z?pyDmdyDlV$sxqda;aTHgmvxHzt4< z0=2uo480mcHm{e^Pe7jdtKMx|F96qR!AqFyeMb2<&?`ROw4z$rX)vStB3lXjYl8!r zY5l>DbV8bpb~CE&-@QqqF6-=oWr3kE0%s!kau$(JX)1So1ARV!92w0*;Z#f}CG*i` zovE(-b69a1TS&1k{GudoxfGbIGx%`GR#yU(fD)@_=1W?RP?m2XZ`n*0^daG{xGHM& zTa;0FySD3xCP4gp+=HT*cbT6iyS66|43?k)j z!>b*nJ$M^p-+o_Zf!`&YCd_7NRB$z&=b|bel%rO`-Jbl9u4^R{uu8)Cky`g+{ltg3 zm2+r6L!YS?zN*#mLD5jwu)I|)SOS$$2}xiQ|NI<#@?CY@kXmbbhwKr%I*dgAfw0N~ zW0V!PYR?ni?ch>2eI(0lVHu6K3fHUD_~$El9s#DLE0ktqL|N%0Yg6q_=i!TwM7W&Y z&I+Xp<)<<=Yj)J2d!LURtzU~AfR~{t!M%w!=>mtSB&D*qktHEu%wqR=EmVTddOsjJ zr(iDe_z4(Z00miL>R-7yuY;MO`JWBESg|5d=SG+{u;4Uq$2Vv>A^N0nGruWUd)OmB7?1IqJSOZeut zC^o*PWL=|h9CWK&<=|Ae8Zrol% zzLzUS9|B6fY$T;mr(n>EKnu>$)!hk-1SHzQ$J~`Pg^Td1O5g|z8%ZQ9zpVJIk>y!g zB@tS*{t+QDxq4hV`uuVpad!n71Of*GaI6uV2u-!0^|l_#w?-D1F$P?ou4VgA-$$HB zPGAOY9s>8A?d8f=SNL$~%O0?CN;*sc>siBX_$7y|@#A1sOH18AQ2FEL9ElbE(h|Wm zzZua@Vn2XPJ?M2=Jld+y(rE;Q>!t#cLb)g8x!9>|r!c<|rV2oB^c1R}MHja0INe+~ zaVOA11Z56c5yeOR``=g^e`1Q~558Vf8fMk6X^~isUUYP-4_MVCrVQYtX$^AoVY`&j z01j?vTYk9Heby2Kv2(4>0YI!d*RMf>VRkUHssX8QuFxGNMFAx65gUkI4j4~+d2W;? zXr(6e4lfLa^`eJL>N(tjU6z<`ZN(-x#!DKXmC(Ripba`jz0y;%9J%gmMWrMrjC1(u z?^=!2Ei357KrSk7mK({c^Q+p9X!s$(HV|GyZAgT_QT;N{ONz&O)tnCH>!GD_Y(ByW zZ^l?_hMMJIVV)ddO(-cJ>gEH8D_k1asNVg!xK!(pODqMFt7mprHfGnasYkzlHm*q}}XG4u9Lrjux@ zG%(=^bq%A=sK4PNw_3pFU8rR-MW5x3EKvi>nSvUV73&FCaG*D8LGS+9f#cBw&#hv><`11n(O+7OBIqEsOg{vpBmkT7|u5iNg$ThuY)BkQs?(X?Hr*?9qb4+DMk;2iX~2SgVj2{-4;7I2)w(TC;=J~UeU{1<=sm!*Gq zkHx5SNgFc=mmfbm{iotZCp=oE89a zmg`jD`MDg4`qQ-*dWsZbe4H%!l0k52rL!i(564v8hT1{_ZNC)??Eiw#0CEf(KUsQ( zqX_4)&l^w74Dn^AjeB*@Uar17a?F9e%?a@Qjn5IIl*{6EgY0T90lp|rtAzYUp6_r5 ztzEGdQH`ae(QMOLL?(>+SWbXM@9+EJZsmNMN7g7ekR@iT)T}{^Y7$kjb^Fp##=T{2 zQs*5~FROHXoyit~eS7Dr&Z<0w^i^I@F-!W5)b25Qa5j3))>rKJeJ;mA!Y~ON%A@w= z?IBcu>jD5&H_M1%u{2ZSW&*5t-`n;O6MmpzW1jeFc$nO9t}Q9av>UnNMJjV)K8=gWS^gW``{D69R4 zQO^q$02U`5lgenh%1CadHRMTLE z{X0pnu$KFjvQ!ZNNrl{2h|_dSctgEps=zhrae;ip9ri@{+t~sLzvZ-D8yEHX$;LsZ z4m@+(6u_+way#4vEQ@6s$EDpEcBG1oztz*f<*?JRawpe=B4G2H&apa^3g;tt(qnzL zi@xiPq1&WS#PBD$!r(i|p_&?1%Y^mTn?8Ia_Z{5HSJpj*F-cb)sf^J)?N}5mCHx;D zZJ9oQzE9q9Mk0`b#R}LBA$fHPej#>LaTik#H;~U)^KH_XQyW{*a2xZziRxVyf0&r^RVY zssfE80Q+0ga%J!hHLNU8C9+?@reJs3n2p5+T5Ky`>g|CuxNlCbzasdhzPneT`MTyd zHoks6a11von3imDYUCgD($+vgB2? zs|bzHwJ`IaUS~jo><6}_Gf9Z2(CyUGyYgLn;d*`xnL)tQPe?z9ZG^Q9K~Me)!{+D` zU(i~0e401x+6U*+F!>qUoEq;2&R*_?Wr`0RW!y9*qEYJkUy@P&+y?``az6%Y#Q!T5 zCxyGPknghqM|DIx8i*JuYnkJVA=htTk~i$Ph095?3I;Mhx9e&5T0cJg zOxTW!a;Es@qUmo1GZvEWA%epzQCdlhVOanye1TO69cE~3vKa^haPUxE^DU_LQ-hhW z;5#Z4cQubV3@ezA@=xo*WAlxPugl!hiAid$NKke^`vm z)l-gd^ruxd0Wxk;vbVJL)&r-S)r1{XjRP60uA=PbbwL!dn)gCnaR(Z^s z&z6Kz>PtKi$yXof4qL_9aLRM6&@Mw^B}kKT zFXh5}M0M9w$41?oN;OeAKLfGIc+_p>FtG2+MU{M+sBufjB{>4%6sSi_aUL=*n7YY| zgwk*ra?-c+%HN>|{&d9oha(o`(Q)cef!r-*zY2}A5Yv)_au-=@{Y?P`#qXbJK?U4z zGlrUNYFXVY`G&?6+8O86rHDY`M#i4Wf|<9%;NAW$YNq+Pd&^_5+O;nYz+$8~>j#l{ zzFzY$YOO?(sGI+DzceI07f(fWkw*L`4T+u6#CwBZb;282;D;cZ_KzjZ;QVyjzZ(=F z2j^UmjN^6@PnooddMN$Lvi*M%W42YuP#NLd_XhHSNz*{s;`Mef_!RO24+l}sFiid0 zJ3K#rcA>9xY@x9A%sTwgBLczK1`9=PAnlX4w~okEmCvd?SQL%7H8pMQNhX=;5CSV_ z+sDFo4zkxlR|>v^5lJp9k*8ilGS3m#JgWosjOkuG2B+S^OTE^|)}BN&_qhduDnTRN zwG$X;S!b(%``Y<_M&lz-C4@QKCd_-CzQr1A4yR^ z1?C=I7 zSje`7Mv@_GQU2JF0@JZBVN5jgpd4;sjXy}t#<(D{<^IW$bUj7;MH8}zo5HBJ)6NmR z#V)|#N}pX8DfWy6HhM3ESMD1)d2Q<7hj%E&pxei7Y!eZqbVs!E5 ze8m&>dF474);}w3A_Z2|xU>adCHF1&TD*&mglgWII2unX8a^F&eW&#fz4#dhW=dp} zV6}rNr#N%lpZDIDPv%nQq{3GqUM>xM=+1BA*@?(iS;k*YgBykHwfnkju4qX2toqZ$ zf7VZzOXbH^xL1<%V3dYQY6+ zMiaANlEFQR`%A)ps^Ygc2>u7m*SMs7s3B?GUF-CD+{Ck)27~h zl_*CXnIVEUq9Ets_qsP1)}BsrF%(JMY`FREUHb@PdDb(yIfpjUL}eH>ZE`P0bo*DO z35{K|vFz^A%)P0m@onqi`!qFRa-@l?g#?FN!!Xikv!LY&50gU2vWMD)K6`2GKVSba z({L{F7D2yWoim~>7r`7 z(BtV@euma%m~@GBLJkhxW?X21OP^9!q{>Z`TI7jD3XMInLdw@4k<7rMH4&RaC1*RQ z21dG_FyK8MSHINUn0mJYg$t%}{KZuSGF_@}zgv?`0_7 zHWGS_N0rSw8~F4Q!8m@_LN$6HLJ@AYu)5xsj?5RY3o=)K1$wQ-{`;N$(+(V_qghq^ zXBioJe+S6*eX&kP-iDb1!HCs4bv}jP)HR|7XZykLK3&`c*8a14v6B_dcpSfcr6NQ= z7|VCB1GcRN$6*$!N=Hd!h55OLx9TIh2Dn?-Lhhj2ZI!;u$FKe*b%9Wtq{j-uAhz3> z4<_@}NvlZsH?I~^e%}8W#TxtPPT>XQkRuj*!kQZ zuaFalz>v|r3HeKVw~rnCJMxq`y(JpbpwT~$W-r&*2i^c&u^#%~ z%*aMrg#ss(O&?-)oA!_w!g!0- zaU7R~`Cgp84g-bI)z6ZtC>m84W~}W$)=Ca)`L*Rl*j_*1sJDYy+Wx5FR3c<9DB-fhHeL zHVb}NO~)p5yjY?(*;Hv>1R5@Hgw;$@VJUab3P8y8()_eim+x#;A_k#8V5dlTj99IJ!luAnvHd_#E|AnXxS@vXhqnH;OixVG!RMl0MCmbt z$zy!?Qqve3f~eUIyCi&;I|p3@1D4Iz&vIkotz{HU9x}&o{ZCM>Zh*4d!eIsi)GEt# z>)Ob;RT%ep6EELRRV{&vh0nWGw{O27Q#e$qFh&(VEV$^l{AZUY1aZN;FCo;msA+kS zGEEzl+@p@Sw#Vj})&9IV_D(rypEqjup$3G%UMj*NoM6us?RByqBm%i*p^$K7Wn@Bg zb?_}zA8Jg7;j$7tThv^~@9wCNY2VFkPAJWZtwNXD&?k+Rg_hV!r<(yV*vr$%FAQ~& z|7&XqSzf*qJlq|rIo`G=$)ixjZsbmengg)6<)FMAp)Vpl!8=Y*;{nt@R@r34#E(6{ zBv3O_p`t?Z71)Q1gk|3SJc$BX_R~;gWD)41qS`pqD1{YYd8J;+KG~v=h`QIr&W~@9BfShpn z)T8V_9apduF@X{kYtNPfskg#`C|N%$7rwpVbt8u&iwq#)2n zcegG@&qFtmmA?imoFF?td(mX^ zO3`7Bxeyg1;4uYioiRHW&{1aN2`U1b=oyJl)|iScKkRI{*`HCat+n>b@(V>LN;tfHl>wwM94DmqO97ofqtH^M_qq zQb~3};Y1)5Y4nhm@#K_Y)KR%9=bKtvA;5$&W$aqb-x1l;rFCrl$+o2ard?aW+HmJlI? z{)s*mU}f{V2q^#dRW>rf=m79m6DBJ6Fl6u&s|4J+n#iRI=NlsImxyAHMa*BCk8}ep zK*P+V&vokne`{6lYH0VXdgzJ^cC?~$q~3s3TvF}C+0RSwwy26OoFdhI3>pm^t(a?8 zE`;>@wUEw5zA-vD6`L^~jD`oE(O9bF)Owx2VY}yl4w~oFKWo$_&h#or29S>^?V|{) zMAC3u}Lb28}3U*;5DC$~hn`3;-0DMP}lO(V)~J>pL3rRuIl2>bv^Q zm4?!(fPGz7|*e%t4GAlx#Yu)?#hUzQ_}@EjvEx=~;pEowM)mtNtjw(SC_ z>syTS)FXTb^>IK*2_H)AV$1R)2D`4a-Mfg`gw88|7!oTEra{T%cI_;K*EP4O9+#(& zz_3r>=iR!Gyh?>_J3Ch~)7Rt1&cC;TN%M@naL0mxl$mz*u9(37b`U} zA%mcYmbE3a_*10=V>isFFpf({V8feYg;^CJinyvN=ciQob2x~9JoVWj#Ff9Hhg~aW zhmf&vSF6*j-aRn6Dgo8Z1@sF8jUglANXXA`j;Wx73b10r-l1(YXWR<}?n6YOnyP~| z>YP;=YG&jkg~jYgljpcofDTtKmSjjD}THBHwdkXu)rH|+$9|I_q z9Zme)%Q_siAy^y{gvA@gC#cn_xVY`4xPb&!UdQI|X(ptYn znX7y~%aMHTJ;{R|mrE(33YFyi;F1QaOe9d}ck6p!6|$V0c`6p7mQjZ|xx}xH;3m;b zUGYSC0ZAm%Tz-nZ=O3E`S5NrqXWdIzVkMzhBYED|z-JxkRUa<_0aX@Tlc%rVl?uJf}TJ?Q!19hFfh2=fY zmr!1cv|Zk&!i38ge;<6qa1DTD4O>CiZ96C>vdFzxE1JC~`e>C?MI8E*sQU zc!jxbkZGJh<|^AmlxH3|BiRqi-#ZI zyrDa4Hh1AOtTc(Z;D+Kh&VjDh|9X zkbr>9tVc&M?xU>j>DWXuxgeWG##X;C~*itn^swewB35zmS6ay*<9T4Nnm4v3F7pZ64jkZJzp0)HG=LeO9 z@_<%{<2=8ZuSH1LdsS!?=+dB+IWg!9c{J?ZrcW0#{ROh_{fINCY7oTRFLxJF{%-Eq}7un!MtuLqf$b z#T^1hJsNL{7@zT0-@m5pQ?A`LVAI&hs>I~uZK;MJlT)wI8>A<^a><`;I$C>l&1XFl zf1uZrS%-Qj4Ckg%-yP@_xVe0Zg8aSooM0E31%!iyoZN!=A2@RDoS3I>7}u3&5iyYd zM~58pKVB4DZ5n_!vCHL~cZL|S&>|p&%R~eG=-?h$h#jUaYZ=Tqu#6bYiJ;suj4}08 zjW`5%;H*n~xPkxKbya>wy!4d>UdSi)7 zF*+@F&>tyE5o*m>FN6D&k{(W~Kc0sX9SMhN=i9;C?);8OIM&EYwgxL<3{O`?w@z;b7EX@b~)Xn2Zql(veV?2No{o} zIpH(ISB1$X%(fZ?lsJ~VoO=wW9YW~4KwbZRV0$`8!NXiH72s0(T|mzvqp3fV5!&0~ zMc0iWIL>mIO+ovAP2}H&totN@rDxuUnaIB0OnER+-YBTAup%S9VOY;5-qm&()0yVZ z{q|xK21T)6)##2j?5^Z%q7wDM>kJTV>kzMmasJqnf(QFe+M+<;Ks)4K@O%^E)zn2g zRM>LTm##~{mpdbs>>7zI1bh9&!XTSMrV7||THt?-5zS+K*)z~Z;jpSIaANs2pX>l%Dq&LNat z=MlSoAqu@1>zBiw2W*5-FKFx$t%LxQQ8+Q?Dv<~jjA0=Fz_kOE;rE{7USA*t=mVEO z$k8JEk~L)2_e!SQXi%y#yW}&Zh2wj?oz2DTne;v#kl#}VvGF@x?K4schbHwreiOe9 z90&?|mAmbH0BZuc=|c8i1y1e@LMEm|%lm%A#RYqb3O}xC)gF7_KaVhIY~mv8FHlZD zQG$Q&>C&BrUvtbG40CV4wH2@rocQFjBH}^%5}dl27XXC~#{*u4Y`@rp`fI1Asxf#@ zP@J$O`a#!IL$>m#;{g!YZc z?hpF!z~vZOav*1Jg&@kU`8W+Pc?HgvI|CdzstAG9FC?_H($G6T(+)(#btMN$FZF?O zfBINzobREW6i8H)DA`}g+g-o7!>>a)@~TQIr3;ky$p%q(%h7n7QT`#sXS9@b^!8sT z^Q_b_w|0JWE#iWu6Z7LLdY~3T9j;}&?L{9+pR~#@;S_iSa*_#K!OwP(AJ%6Hq1*gF zkIUF6iK**yBqog7x?^6RbsKH)YpvGFtNx$3?8~i#=VOyEN*c zA_kzcOh74=3rwG_>VIL&zksjEjqKxue&I9103z(cw(!C;ErHdeo~}lf^an*L5Ri16 z!&)$O)^j+4;M#=Ssb5^`?;QK^gAkuVjmIrIuwM7Pwz5;_*BrwSfOrk&~PJOqn{t#C>LtpPyk9%sIF0%MkMdF z6;HGg%f1$^e=;h1ehoA($M^aKDXB-hNPYK+l#Go!M?zN&4rOOuv8ZF#Q4XIY6B;?a zxT~ZKR0u1D!ee#N8a>pbU0Fu2OV)$ULnXJ;mQDS9s%EPM-9zez?pe#8hQ;zf@=Uzo zd10~)mjD@ch$x74ko(Qc+xl@{2hF0Vtxl$QLr@sIV}aU%i-7@rA3>)_NVkd!ur4hA zB=v1|Miox2PucUu2sb27Y=jU2V_x{6UXlD7$II8t3eFho!WmA~uOTI%2OCCW* z5s=tN$}8pw8U)#>a*!L4M1fwgx4{@~QW{1M`6SZF2Q|V5Mi~nMXh~f^xD?)DC>Q*k zWHw5QNH^hO{2Da)v^SKOKl@ELO#R4%z} zm@Gxo7+M}|>YF}J(}c;^25Q-358TRpPymKB+u$JfoRi9q(!OBYD& zrn`rzisKI%L_mTyJBCp{-3S$}#210!<~})VOVN{cQ$+i#b6DF>V6@+V zdk&;i)qCnEN&t!C@;~RA|B+D2wG^1C0?fedvj~VY)21=~oTDL+hf>lrAO?uex^=tm zYOb*b8fEJ2G|u)9aYKD7#nrrJC||6Co;|FItc4_Mb#W^;lG7{U-Y4-TESK(g+5we7 z!$j@_-+782>LuOmlyPx=NjFqG6cpbLj;6GKOSI1&WF!K4#W4p;Vg#l!mdD5#huADw zu?Xy}&xCx%W3LUMmQfK+ zjlwhPnQ(j^eP?rKGLz#A&iIvoDe;yJQCH zr*-MTJ7dO~fHo~`Sor-SswE4Q27kI5-5(0_qkNKjmsAJh(_0VizZFlajXlV!koQC= zr`^^21hT>!S~d;#QGF9pV(mpqWs8zbS3LO)?%#A{cOuo4Tt?npqaE+MFn)umwNmhJ7PT0pj zi{%Wsm@&v4@_T82Vt)j&Vb)=*b!lc8MeR6Mc*^*a%nEUO=h&ZeOqdpKr*A@oh&h)VGvDpmC(G7P78wlt~O2DTZ2MPl$+l z4+)e`fc?xl{SdB5yV#PImh9B`F9h}4)J)@>(l2%Y1vQSi5j_o5BrPRv_cj7a0h7od zQu{!ulmj#^U5ktrm{7sA5;DQ4ScgD?HD~A`GLX}2H8TSH4#op&?q|3v2C=Ps@4uT^ z``K7_uoj%jg^o}eJ5j6vd0FPpCvp~G4@~jTE=tuM;`Lw}fMZqEW&D_zveZC|W#IS5 zgJ+TB^tLe1dk(&Zs8D4Dsjy3Dq8v$kJcR~H=Tukot3^L8bpG<$2|yByI*lZR_WlZ* zG{Tt1$I&g333ZWQHp}OqSJIpv*OWGoZLJ*qP1b;(yAk~+wQ*zPI?&Wj;;SNBql5U} za~1>v`MedT9z>bws+!0oTfQd)L&0o7dUrzQ{Nsn#0X-AxH6gDt`zLU-o;)T0fqP62 z!4$YCF`TKJm6kM7IIY+!22{7psF2$~v>UwBkU_4Gi8xhuZ-C~h()~gD*`839!mvOI z&}4%M%L8dPjIPHhqDqd%&MsGO3&~Spy}hZgFBc9R*0+1kC}oqiGm#f;f_?DqW1l^A zH$|B8!bz3-og@_tcMfw>R`-H9-+q#F$tM#E1&3@iNFpOJnQ|XGROGag_J`EAj@dMs zQRIfaLum^hVQTY7_A2-L&Bb13pYUyei>SE(4%Q(k$^^LB99;|V`fyF~utqgHiS^gf z!T2YuRP7Esp!e=de%h0Zv{v%`oIe^yJt?ZWEV#wA7Qi>Hi@6Pme=P->!^~@sO!Z5Q z2~m?6)cU_KLT6P6)QA&QcGo>q)?yri5)zMCCl!R0q{?e@w7wCt^O@R@rIcn1X z(WOnHn$M@XrckxontI`SyM`n{3(#**T6kX@k#lsyys2L5YL;P+`uX!H%a@nfBBua` z8iTmcy8L#W6!ev2>{JF65}AR8y{a9-Pmgt)`}BuKOQha4gK7`Wvbj|tw%3amn3&-M zcGrLevhpJqj`D8)$Ddn+xg&$$DP8vaWxUUmcgP`|imx;hYlz9zK_d&*G$}e zibSc5hA+)2#p!o@Z&WajbyrFkAJ#%yJ$3mjYN8?0><~+6c{&fO4BWeJx9rtKE zvLrx>u$SP>`~BJG@Kk_=B(9ZVZ@@`tL)xRK$9)&p@sgi|=%Dn6s<7mt8tsZ54NbiMh8Wlby+` zOUgVlG6#!P6v5?baxUhaQ$Et0mHBr{57W-+`0a+6OGM+K2r2g-voV)^ea0KxnhwkS zS~C+!N>;*XInuks7ToJ5@sSlsv%1dJfD81jFKD%TAEI*$Vv+lziZwkcqK^oZim2Z zQw8m*4M8HT!+nHsCmV^li6|y}Ds3j=yVWF`PfZYaB@&--R;`Zj3|Wa>1los#lH83_ zudNZ}Yh8ohrv-l$Tu+VStE96ovuWu7*)%NvH6bd=^`wcHE^;_n+o^<SG5U;`2QKwR8{ACl6Dm;~aB{c9boOU}tGIe!bY{^jl z3hMzt3JLJ2u36PcP-@_yN(G$9WF+m2$Pv_gm3-$b=P6fqg?GC*!lxM_dYmO_9g^Xt zB7xkB&xOKqwsVq#0@A=7YvKpe79f2J`*>qBuAwX#%@=GwTV!)0OKbf-$TnVYDL-%Q z9ebNhP8%~efu$QJ6xW|bDt>ZocKS94N>_=Hp5CDz<2a7o;__IgX5&xPh0Cb)j0r^; zcz)2w;Gu$GDXu~cjF5^sQ433)rD&-vV#K*aa0JV+F?h zG0m)Bq9$4h*!h|q3QO6fKEP^%#|#f)E1^d$^y?DmuwHk_+(cjLr9N_|aWj4NF2FVP#|TEGfM@Cwx;BpVFvK-tZ^4Vumw zm1Mn)I#C2VZgkAVk3^UP2o|8R0+rt_8zE!l zI@07M{Qh0T5Pia@%ht<+qOW)%3m(6cL_#XWU$$z%3RFbkP(RfoO`44IHd0B%BE5`b zC$te)qrNB7LAP2e_olJs8bZFLIBS)|g|jd8p|ECkor!2b{dJ|?gC2%C|?ppDD-% zao1U|n)LIDvNzU-ZzL54VB82`y{2DOR*-k3E;KjcV#ab*=Fm=?4QX!qe~iykf*5&X zl-aMQ7{)>BfQ;%uS`z_d4BC3CN!{}Zjs>N2t#I9QQp0T$898}ND;RTFChDv zhL4qap5jLS{@(2?(Hu!ZE`CEVjbnbUcm=2?V0??fS;S&Vxl*VIPyNzo*=nG5s^=#1 z*P2DGfo*A8!OdH)qS#R2J!Nf7##`$tL_3z20!%HR0YMJVCwI!|`46C%>=8Qz`UIx@ z9{oib_;*RAN5UkmgYc9IVKPr$n1k$ih?PRwsXOvb2kvS=Ge(k+B28zf0yt>(`}XLZ z@h{pvF(UN+&)&T1SZnLnXf7 z%PCyMtGH19jiJfpsAPZKywig;gT;f$x%GTcdW&mhw*_ajY|j%M7-|c=TiqM}Wt()? z0I|Oq{TDqtIO6vQRIu`cc}PY**GtBxh$g8;4*6AchEwZlX{M>wv5~-W{8=zPZO7iT zN84u&QB|pFnVZB5EDN6|)chugJnxIZLm~{k5~x*b!sK%8{7$>d!be6^goLC|R72}LRF)d{%DxclWMm0JU=X1B?T5wLds1PfDx4I_3Ym) zR<9)$$49T@cQx@G@EuVM0Vo|n@iIU}m}(474szs@EyBG7J{ZfFW;|3YtriXr0A@CA zR)-_nK&P7W6w6Zpn|UVh7rB3*^YNjkj7-kD4f9`f5it+b8C~_;m|5S5ndMCA3d1-F zi0|Mz!b7_Pk6-DLA-ScK=y05v7D^c4+&|)U6K?t_Eh4~n+L7#~@U&e$zbNQol+1Je z(h}j9o=fU^V8vd_=3lJc83cs}!4`5SqdXS`AvG8Rgad$$k8hjtyZ{#9C#~Yc;k@Vh zScMsA1F#|^N$zy`U^=VP(1~HykTq0j;@HqUK^b@Q`ush+yCSZG~5?{+0Fnyu?!)$k9QJKQT2zTtC#9R=4AwnMRI@8a)ne*qA}EXwX0 z0xV^Hfdgd6)eD2jDzvS8zoCa!9=w$@B!~yiWe6qX+Q1HL8DK9kZYD^<<@)au!Tm6f0~HHn{B#_5^dK zer*#GU`)I|UL}X%V*Z-G!{B>$Ns4V~J(Q2&mYco#hp3G6b^;Epb0UFsm`%ip4^f%u zr4Zs@ehT3Ga>bSakApoD9dOlZ9{UkLdS%F`ZN&!|;mEzHhWz5u**scYJrHq%ZLRQ> zM7KhNpTb3Y6E<3d<$3CVvM-3Z(9zLi7vD9x+~SL!FPI880^#*2)A& z9-YOQV_m20Z9DtoRhmb~rgfS~6CF-b@Z=cp#7nJ-OSP8UGI_)Dv17X2kBAoU5lCok zO)kYVF5@X3a33jqu>%$KoYqXJvgm|aMbURpT2V6>JsAc4gJnC^&Ze3zZ|o~~1yAXo zlE$TMVV|LMMF{G!7T`)nseIBz>UtwNKN&x7!nTc+%Pw{4smMg|J328)(5bk?nz~K1-_| z)*CK${}2;b#NlgO)JP$FzF;GpVH;}C|Bo>u2jP9_w_jC(Kv3DA31fBA{ml64430I5 zOuZ>686cwpi~%O-0^aY?2)UK0$4VvySJVw?qsU9R?tE2gCb3F*sDS zvMIY=E`a7eP_1z#pO=tqCN7>c!L7tM4i=#0T_Kp&a&?5c6k>lkm zLC$G+;q~#F2T;|A?gWf{j|x%_(r%ZUeQ^DI3CP54T8Y`N+@eBG66;w7$fM)so3|I+T|p2BR;Q^x4Wx>|30MS7pkYRJ zYvz9qnFlGVJvk64#z1#vgZs~*gX)*uxasLH?zqa>61CQSL~SMfpu7f{8$v?@5Cq;M z?kGKi?el@1Ov9sJ(o}rtLWCd0owt7bzYY?Yy{--J>Z&v4It`pkpRot$zTn#ykA+dH zURxxl&eEA4dj+J3)%5IF!IlXYvo|$5`om#;;bO9o!UT~hztm%kmXH|innfZEMJ}+H zJ_5}orx(5hcR&!A#ApxhW?(CXExL(6F+%Qdmn!mno~|g;G$(Zei6tWTi0y`l%5@#0 zp?RHLCt+9z>P@G=%;$Cdep8-#hU~&_%amz~B~ivWt@*--eR21y+uGzM)nfD`-!)t` z2w5LSb18n^aDkeV9`J#?I%m6K5(k(c78kN=^dH-=GATe$-q4IM?=-(wPDyLr6F$av z#2)qhFWV|H@AROW{pIX~UBb@0wcZ%~Bdpe5?QFxa)NjrZ=DYU9 z?Osb3ORZmC`bjHRqJrgvUZV^IXndm2M~n2fX5JVoG>&U7K*z4j3kze9QbwlUCk(_3 zY9ANnK=XpxOEUP5K5EA{^!bytkiXZcT|E=wqdZZ?=w%&vgE>5M&a*wN9H?2#*J3aDF2N~$tIvf9*{ye&uca#iH*Sr-BqId2wLKk6?%RbY`YgTM7@3mx; z6pl>#27P#_*Ee~p{Uo#|b5GJbF-$2lhQgLxg(4rITjZ1+t-Cs*ahE~kka3tLliNlb zdW-aFb_pfU2?FJ@GNK~wx^}OoLEhFgFTQ`XJDn(%B!;IaonE=kINaIMpc4v_QQgT? z=MeArF;X1G8%$Jqf=hK?dUoQsGYlrF>%l+$>p2sm>$Lz2CJaASRPscz=Urb~<8@KY zW{GgUSy#cB#lLRBhiLPA$TXo}{PY5PrkZ9z{au)nGks~X6j5&!mh-IeO;sK+)71)~ z>v-}>2YA6qNr={F)0`s)YW({iLR^EIQ?c_NwecJ??M_RU1x-@Ldw`8p(L>YmT`a0o z4VFyC>1{^;BXUmS+l&2;jPoY&NM0EFY@NAXUNwGms)w$@-#8?NbrNl5MHpTe89-JG zp-$KGQGGQ!``t5+h%z%TR%{k|eRl}Lak6m>_MnW~oL?FUBTWM$BRLw*iUq_BQV?TM z8?Xm#(GCze=D_tH;bIuESd^V54~+xv*!rM#$k<>q5?AVmTDej)Bovc*0qqgU1NxP? z1GU9zVkjW#ab0lzy?=j%!6?O?O~MRfCuU%!c0$GND^iD*?;E)!yYFsA`iCp(2#iL8|Mw6l%#h^8U}Ka?q0r<-E}5)RzDDm3YFe7ppDy@=2US6~a=lA~w5)E8&-Xls#I9Mkli#u>*D z&%N?fnJ={Sd47<+QpZAY_vMJg?uD|ns#_My)~^8nSa28;oJFZlX8d>p!(~l&Jf>RI z7&!N6in8RQzjt#F3aJEb6DBi_U^K~(>Fow#-!BztI4dZ9k?sz(cL44Z^a2k(+h82u z=AJ1cs^nV8=wwL9Sl`-1Km#gMEB49#AZBxh^2H$j(^pazjuO>J<(e~!sEfq`Ln*GQ z*`(BH9zcxn!Mn8-z-9|CMVjSH4bvixuHb+5mJ_HXj#fmG95Q1_$-5}tEy0H( zmu60_ie4?VJzZCU^bcip&USIdSZP9`pJ!mwA`fohedc0b=$M+oM#YXK8*N9n`g~UDe9O*5*ag$HP?1!D`=StGjV70bkLl%eR-lA%176abM)P~h zf-OAw?j@nHKtoNF)ZVrW6z&+Wzg2DREfIVYzO3>@ZMWk5+w|XN6R9O!#K<419W$!4 z(q8Y9MSHR~K;H6!vs`_J(qtK^92Ed5qPqCT%&STU1C88bZUw>{%O~2matcuRJ7iuL z;iF^bIm$a<`@alw4pV7wY*N?-jvhS2!LHpdh;sP(q6jE^LgMOv-m*=9pfrH63W`zX zICfGh0eKt|@nK)+_)-Iv6-SJdMC|G@2eV{y_)hIMd6?q&9=Ag8l#6qsLM{YN`nG?m z7j9voeGmIc2C*ZKbH4ziBw4iCrchx~{ZcgAx7QW8< zmuTAMSv42$?l)8YH{T@G-P}ENvf(<1!B45k!zgbT2RC7B(Yjc!Ad|Ayw zJyRw4OS&-*X9}vHUVXP&A@dXW!T*IY!TwJ)>F=^(Gm67+TeJPC%`Io;pDKBN#s+^F z*9g3l`6l`>TXXQ_=Xn>VO}fvP)L5__u3z{RCouJ2Q+2^%uK;5@DAD=N+h1;DQhJ>^+4WL>f)YK`Qv zmYP>!Q75sOCNtS~Pbf8QFN5aR|M%{J9K5>mMs9XHD{^Tg4i~WC>ZwrUDI$pPdA$s^ zTw&2$%3Z-461fe(DsMiw-PZ7Cx!htK;9*sEO(lJ)-V-AQWG`hfR3s~@g%QuspT+e& z_UAsGu9!I%cGd_d^8f2Uluw3kWzkrs-Pm<8})Jy z-)KzS6yVN&U85-ABp3ct!@1%;uLPcofbEdH*+$N>L?hXxQ0zF z$X7JNVppvXc>b~q(hyzh$*K3P5%_;PCFbP{)g4Y1|!>6Fa*>`>>kGK&W5wo7zN;W7J-=}7at zcSY7^bO6;cDOpkoXG*H`U+F`^vcNaRZt0J6@>QwnLO9o;)?JZO4kJ-YYfsK2|3&OM zT0@Nrc&VFoRaGG1y%SF#uy0y+_-w=K5*7^77Z5X`_m;v*^Kez)NxP=ydvA{ZVlrvd zcuMMfwh9oVyEYpFntLU7Gk~Yn^b+kcy71xp-CIn*z%fr^^MQCwIodLdo-`Nc?dwvx z;s!_SxQnd4Uaj2?in*FL3Yj}H`29(ZGEMz4K3rV0W$!5;2Ic?WOuHg(D;bndG6BEU zA`Y$xkoQLA=r=6g{;a$zn&BlFXWm9hX1MD@u9nKD@CXi$4)b{fqxGv1+Z9*w6BELp2QFa8l?yxo)3&sPp0KKq`BGiWwnC$61LM<&GabO zJt_)>m!UecH6`goX!JVJ3yivWo^-@VSlggQ&!hynbQF9E9Pwy9lEIZHV~THH)!3J2ec8- zH>x6IzkNAhQiGOFY6kPr8fU2?KL5vYLuF8%dNIH<@<-uRUpTLYWpIcCgrtf7D+zpa z?+$>*<|=j2-=cGxbK{*aH`Kn}Q;)F(9GqDv z_b6H-_VDHylx>loNphxOvi*x z0@pq8*~5bK&12G9u9*b|*Hb9+&}?u+@55QD%0T3UmbZDl0nZ=GNbNuORUV{ZKh!(a z9xml}O+BHP8vWR3+*6?>?9qi=Hvm2?6Tb)%Zf2C3)IrqCCHkfL;k(%kODbYA-rGX{R$v*CC&0U& z0jUAn2{`n{)P9%tdre@L(Ts8AW)7~=_Nm}ae(^G&z_3erE{=G_n5-_9Qvkb3I2;tQ z^1=nye*AynQAwP5Br)dirSwQXYaQc`&n|Ox;BCp$e%9e0m`;|UM6UzGSnGb%LBp1p ze4OpN!s0xbo%tTU);@0cu<>zMvWjLb50%B`c(FP~z&r#7PBI+@k9DcM+)}^ zY+24Mg_VhuQ+3@LfCa^HAx)$2wSwK9V(N8pIrOf5*#pO>5<|Il^pnDbfK-M>E+vkv zGq<4;zPEo6Vp_z($8w@v+RjOyX#*{E5X!2nxEU090`j}Q3l^+*k~~fe!fU^du^9^Y z(3Akft}N-pW_Y(}Y1~_wJyl%yA;xrcva%x;@8y_24^mR@Mr0(5=+lv*2s%A{RaI^Dm=ZSh`^7-3&AbUVJJsr7HOG*^#` zn_|jfP7%I(efqMQpzDMo;_vYdkE!|`%uFe5Bv4ny-F}AV?OFxU4l4da{KE>*y(of9 zQXk>XuE?kKqXBe3wa7yUw@g+glxVCBqjR45@R0%jt_kO6))1lH=MeKT1NDb55_nfZ zOSI#N|8gl4?W$goe65}}n|0tp3>~oZPDBDQZP5Jm&5>q@7C^D(Dp&c8%USMd7y)Iw z)pSJ{}i}>1Jr>5Zn8a4;&fgjbzJ2GCCOCPHOme&jT@icQBv; z5xxCGCXwiKen;S$(Z2estbOEM;$`Xd4*kZ>J#kexnG!I1ZEKNc3jdYv{(q5Z9kZ{v zdry~hhZZH9#nc{BhibS?p!oIi56gBWWsF>}+zcPpJ-y)3%9zB%sOg2w`X(7#c4&g& z(IxsypU|UoerdQpPn>{USrqEz4-;W>9g^__)%}+g+UQLm?obcS6kX#rG(B`_>Nk+$ z?0Je#1XJ zisvUr^X2wyjb`sI`z`0%YHRf<-07?6ho-uc1uE<5F!AW%EqF*fI95RQ{0=ZTU$W}EFQ9`kkjhkZPV%oEMVM$d)1aOMLSti(BMYc)WNFnq6T zuGas5svsfs-9arb27q{vSWf&STjfzvT(VMTxHyc7$b;yXUTQ3A<8nGFljr9+S1B4; z+4JNL*v0}wr%#9!$_K8p7VpA3dByxoKx=AdG1$RMz%e2irO2;t#YMeGm$6)kM_u$u zVpZrY>=$0^?rray^^d1z>6~{v%fVSKLJ1P71X=LOajfqPumA5eWC%Bq?bN7s;a+w& zf%(7)!o=R|Do=VC=t)h>+A{{V+Y*a;;=&a{uy~5=SA~Jz&xeAwgsmV2L39VV#6R#M z1}GSR7bhVV^rnTa_!!2tWI-+Qba$8vH(^1w=81kb(kH=!P}5V%r9F+f%Xq=%6NU@! zoAKDKQ8Bv;Zh}4cz|e|Vx1yZ;Y6V9G&@-P{au5*ck~P$TO>`^ClqAXnjAZ3wwgjjW z!5s8;gM(+YxB|J5WK1J zDz|A>E;yl+M$;U(!R-Ro4>2MJxwVJMer36h@-NBfJ~NGwIg5X-z+xYwrx`27cH&ER zO=K9hFG^WRjgoTD;|Dmudie*>7|~u=aNWtqrl%2`nPqw{Qa2hYJ;to<;<)M+yf~9a zzUr)=41PaMv#wv?aB*5KKHX^1yQi1WdWZ0wFmXQtSHzM>3Yj*%MELQ8^tHddXxTyb z!S1(}^rL0E_K)pU45b&3G|j>bssTJ1$zKjz+E(w|^hQLnaS9wYiLL71FLJX#95Meu+3@;YaUYXOtoxAiv^z+Z{@o=0!zRgk#F(K^(qxM(v|k zMlkuDAEu|Zjg{T!ssJg4y&;0MT~-U&u$)z$l9{A4)i@TQlwCmULJH>c8iwttjTIg8 zNW*Ri9yU@ebAt-oBb_rTwyCaV?FpWErFhOpyJdHl@ zssKnBnY{%yxtwAhw{UHNI9ri*^E*ye)zpb>F{q1^@X{njV;1)oD+2$+ zb6hN8Vhca@McNq#2gdW2k<=`vmNLXc#4M}~TRA`o>rP9A=w1~r2Qx&+*x$9N64^X^ zh>m~Ls_k#y1MAm>`J1UeFBWs>9Yuv>4ob#5+U8U#wODeUb)4)F&>va9sHujlVfNAcU| zqs=-I2DM>tz9p&E8?tf;%GszRhHZU)B_qEF?0|!#{BA9p$cIuW4OpBw`d8=4b*wI% z2Lkg7*6lZ6hFAGTFMN^3oxL^Io=$WA4W<7>Tq-4FSi>5z+{;!HA{e43In`OfERfJ= z0EJ6=QTZFOxKyvHvlCWF8CRJV3&yH#XE-K2*y~DDhDr$IS6dU6x-i3B@Z1F8L*~VE z^J-9lN^%5EAt5DgSX()k4MFK3Wvza%ViQVg!4x(QY;<1e{8$-OhRY zfoq(sgac+{$53YG>}+vT`KWzs9TkAq zH(m>tdMcm~snP%G<0wF77Mj#_-@c4(V~?jH_gyiGRV+FfmIW^6UenUNRG+$}M}dg6 z1c}X^(a_m0!EPsa_dz!k(ihuua+xSFDyeRH;$LqHUEQ4f5?y9DTtP*BN{gmV@VG-5 zPRCSQCCuZp9jfy5sg|T#mzOQ^La4rY3o&rKh%3of=QKux`c@TxKf`Oi3N@y7;Xv@k zk#4=6@IEeD6Xg#ukrk5=ivLBTsoZ`(?5w=BSU_f$7`mhzD||bR+;|t*^W@#JC3PRH zQsQV$e8$q6Up1oy&LvVwHqWR6+D-$YINLD#7L~|;pbuSy%}T7dFq#sB&%-84NU%17 z@Yl93!=pL;;rUXFI13-mR-rAn0|HdI$>8Ab***a2L^b|4GcDkXEXI)Y^{FC~ zs4iBT%%C03@ze&vQIdX80(lGjvzcthPf-tgj{Y0=BdpHR!z7Er0*VM0=*HJ5za!8c zr|CJblWt#;FI~&nWG(}$sp9s#?DH)8H%WWA7;2`OmlGEvy{mr=KHZ_KEs{u!>5VXH zW15#5wH{D6`P*ls0P7m8O(dXk6UreCsD+1gZ1h3|#p6;BUYs;1@N3H+%r&r6 z#LqEo=!f$#!6I1DHc^LM{p6R(>n0#> z|Bww_P7t&4A_AU-@sh{ywk71oV>II+J* zk3g{uaD6SQx)d@xZDBTQ&1qEeg)=7t=DRV8rj}+(DB`Mv1sg*8-#v2w!b%1SwFmBm5vvONc68>gacj^H1O+PQoeu zmAy^9`E#jzX&<711=@2@+^Hq+qW5yX7Pk&xrJJqj+yOi~pDvVZ{QM-Y8nH^E$nL4E z^ZA?s&PDsxfH9QsdFodEZT?%Mh0i%4isoOVssiGjubcUK4h3iKFDw#kGsha%tLr;* z1-5!ATO0s62>jO3cDOqjG=zDQIu+CnnL;s8XLEkShKilUAQI4Da4R0ZWCF839V65y zJ8AU2@N{7Kvp%km^SLn=fIX!Hi$H5FVVoyx$iY<_^C<2b_j&x9;_s->E`~09yrP;E zF|xK_8%vM&>o#)qNBYsXx6nlRyrarHEfQz&si1CQ(%W&4^y0HNnp)e?#kiLnsz(q1 z*Sx^u#c`Fq7TAxr5D+2wrqVDI7iGlq5tuX^nt&s*Cv`_GLrx;OPCiP$S4yQp*%!nG z0E2Em=UXOTGH~kGgef)xFUJsk@-?18d; zE|$;}^1$}p&`%2{bf`Ns5y!2%(#PXRW@}*@^{{3cGM)AEgYY`uGUZ}AWhugU-9wp# zV^soZn7-0A0P?*i2G^(*3{Gns7A36;{CgutXuuiD7VV8LZ?f^n*(uFG$p|e zI~JBTil4X&Q z!fwbcUpRmH82oZ)oMW`;=&f`CCUqZYTomn7rGIU>@IjKuG?TdS=aXbh;a#l^%6Lqc zVDWwHXEkd1iT)>f)Q&R>qK5_zanrNpvBomMxXPP&@3+iiyYL;gxg%QkPpSBW>Phs{ z@&#dqAHC+gwT_?#xBZ*|DWwD59b3;lu5GeDmaFv#1k~ytfP)b9E^0Ds3WI4>@(IFC z@7tS-RT0L<0G;yhgXb}ZjSgK_HytEy)aY`3kTY|j>1pYw1x}q?{o;2%NedGR5QdBw zphvkOX03pZogIyzwBzf%N^P86O|apOVY6SSXn$HQkXik7<(NiD$JENVA2kSylyNG3 zowHhghi+RJUUBZctaAwFn|;K4vPsSph)Fophl0ZK>7?3-jRs^Bn$IOD$O0;)v<%pN zP^@X60njmEDQ3WwC7Hn&7Nv}I0d-^TA+Ve0QOr51%~su+un1&4NXU=8<511r<86z{ zdyE28?+vT6J#51ueJZ)qvGHfIhExNCiw~$$QepSH`QrU8PH2*K;B+`7V?LSrh;@gJ zEj9Mc-Rb-Ae_(hM3M-8vdoCGij5q?98MkKPr>PvT%#nd+n@$x^w~9(0U&4B0b>UHE zd>PKNoaC6Mg#I9`M$;AyfJ@X<4i-Gp8tO-Tol3jD75ETRy^n?9vey^@)w}B&(zY?4 zg$?FE%-oqzIRN#P^gn1&l9>tSi0`7eG(B+>-?Qa{)CJe$X!yNH3Lo^PP^iYdDVevR z?)mKd?mn~tE@poM3Ym`4UpFcrl?UCpRgWhnK?H(EESS6h%j8URkP;QJ&ZI(Sp6_F? zTb;9(!DsPdxjNPT=@c5EbYSTc0D_E{6Mxkthh3!JX#7zrbt42DVtq+jexS_P6H8Fm z(aPBy75xE|E=~*fbT{uG6|Q@6a~RExddL&Y6)aU-MW`)U1CX~!TSJXYL$=F&h|_TD z5sw5@YvPVc#)UBM1>pFClhO;OA5k zibUZ)JFe`)=dgGjxS~F`Op@kGTj*_Ghat%pXCcqLwap@))G;EHa<>s4_nJNpdWfK& z`^)0njooBg1_%Oa)*fSzela@HWBQXLq+2N_?7b9KQODqVVH!a$i(YPBgx_f0o>BoG zzZ^;z9RZ>W93i>C&!^hA=@1Z5O*i_@pA$?ETW>TCgc&>r7%3KRk~3;Q>Us42c5U;o zf;CN%PW8z$kM;^K#QHXZ19S!;MkiU8gQI~>8P|hh} zt5u`f0L4Gx1(ZxxLcjyN{G^?H$@}6@KrspFkUr;p9iX}FNri8Xtr9Q-PY=Gp+1Zbk zyQpMv(wd zZAmVyzcV@SJ%nRY!8yN^xN4|0EN@0+8(=>&y=!M7Y7dt{boxZWL@{>c#u!ag-KZc~ zI&95OoFk|J{m0NMaKEqCq|e{-$I1(jQ!CU@KXg=E_zUC#avrZ;0^D+dxt z$?tIBM+=owZ#+M;fkqTgos{Mq>bD*0&7MR*mRdU0q0d0*DD?-;G|3K-&<4y@$x0D= z6%gS;6_IQvItqFAKc1et&X z0KRvR`m*lef1KSyD$I*sgWPK zUnmx}g-&Ha>}|QONHXMz6-~JyTRiE$1_!3PX2Nv;Rit`~8D5#s-(`yqW9DFsP-bUJ zXIferyQxivKK;|F$Z|LCwTJ)4T$17Pk$d%CXneu7wD)u+ zJ@}+8J@GEyBT?u;Rlx=)pe=Dr6%ec{r~?mw@1NZ_ts$jE1y9xy6CTRPDB_2KooA)PYKX6A2f>5}@W^01;jg!^ zy9#}D%mDo0t`O))adl7X3z-5fXtt(vs*A~)F{TkiARo#|a4P$9DB6leNzp`DOLlBj zVp;2fZ~FCXQq6F9k9gE>)wKk4Nh(XMl1mOT8z~B9(KE)#QajO~HP@yRChcmE%pt=fzN!a| zSpALSrb)JZLqP-JUoAro_jc7Y6!E@gDo`EnnJH1ob%MdlOOfCpiV!36nCaqhzu@zh zDYmzD-lD6;W#VyD3Ou5LZGap2}TJQDDXj zkK8S6#M)J3k3aV~B7{S(_*CoVoGhrx9=?8sS(L$bOl-iHkS<^=OP=c(K27_o*G3O5 zHkuOC7ulaYlbi*Wl9VLH4MuZ`Xy?dNIbcIR6P>Y_v9rnKnQ7 zL};jd<~+Dcro~6UGg?&GWwgll9p#5wCLrJDrL z5Iub{6&T$O-9a>bZA4U)E4PdwGEGNKmB3 zGmIU}IG0g;Pqj}cN5oVmd;N$)jCZk5zq$u}=Ok7=!IeSTZ6x5gLM0imC(X`U*n?Wm zcJ>Qy-9Ig0^HV*2N@*KH^mnZ>Nc{V+IWnVx1njeqa??YC-Op$>b#kmyz#9{#uquyN zY%oxouwGnr_06PG&HG%YFwib5CK~Mm19>*xHBr2!vJVLfn!LF|fS-W+(lE}L980b( z*9!}UBCh)m$(BPommuC$Yqdx-QBMvfE@XEPC8A~2iV9r{4@l6$eOgFLhQH!n0EVGA z57;JKi#5GcV-zqk`K9yf*wnlHg}G(q&`yqsT3-;(#;Ak%D(=h)M$^wWNU(5kFM6Ax zngdVe7vp0-S#0qH`(G2E9C8@rf<6=_$2t19_%Z!UUF7 zQMd4+em{F?QDC1;HsaqL!(fKUw zx`#C=lXWmKE2@K?9OuY^2DJvGHBZyEw#M(1`HIO?C~jzlKvIyMeFBp9s4FRaPD`q@dr0Jo zXFLGpS8=CxTgXM%<---S9S|2>Z2|S1?k>nhuYi@|+T4govf|fsPm4*QNezAxXfXN= z&EC(s`|hn+1)Y^fe=OClb(tf8w9F));9>B;EsIZr2gElqx}12m%k7;M3*vO#kndx* z70xb+EAa(mAHF%@g+I#d3R^`Ce(XX+VA_IO`FBlQu~i#D zoGl!ZRrU(u*&krk+B)6&0bqj~svtcb2w4ypCRY3tpFV+KjO!5K?|t!DK85ohKIX5kG+;HK{rT zaeYIK$_WBdn)Vw(%+OR~)gmGY;H#&bW&iIzd2#dlX_BVuJxL0bNT=rlqF%0g%9y;hgx?nDXBPlRcYqO;?ZQ$MVEDM>ndyP0)l#os~;eG}qGLVhQHTTDD0 z+fT)wtP2==G$AXG{u3SZIGB=tP?Yq1rEqwyslP=G*)dzY&Uagj7zVFx$|F+bt66%O zSk?TP{=9oUZn8tmuI!n@6^ACb|^9Nn7R?38@riyPb`pAk!Cy@ zYboObm4Ux>?egpu#om*I#I=f5)7{H_P_Mu+j>wN;YP~k_d0WauRe#mzz^t^awcO~i zxvUQBn$HI%HspON#Ghh36s|=xP94tM_y~B-l%xb(yn0T?Hv9p)oSfXFS#RA~YxZ7~ zU?uU21w>5Y3#X1C?hW2j94wEvu+4&j$MDXbqAElJ907DXn zvN>Ke(RB~^G>uk-96h-T+hDE*+2}0RP^A~Hv!JH^ftxtaBzZw#cY|#|v7P~{E5XX- zx*PKAvtuZb5wC8v*VarlIDO*UbGbl_G|3G2*V6|#xByc7`?Mg@!~5j^a$2B#@wQHN zR`WzH+8Wi4#)lt+f7HiT#)ZoICHifG->sy%s_Nu@%Y7t%f^c?~$1bu+g7!J|VF=}) zm0fBi0t-f9qf|AAaGqbWFN_=~l5?fF)O)(}kT@2ugCCja0tcw+1ZIW-7TN&l)IulLRL_^Z=NHFT&c@E>}#=0l(t3ie+^!wu}*0$JB&UGucH4j z0nmD>SO^Uy2+~iyqfH6!(eqSySWC!%v9D%?d4N%3%>OJao$Puu-8N3Sd?jZhUb({5 z!%O=43AKKE)&m=t_EU05+Zx+;pW@5C2;WImqMK7nIVHsI2yMC<@l8i@#59;yv$ee- z!u7quafi?~g46cVmJJF%6w)J-23Ho1h^VT4Ayr@j?yC#koz%y+B$Y!!wGPqK&y?I9 zW7!ik*D}e(iZd#+Y9~Z2K4_Of@sh(&#Wyf9?NtB1JTXAi}ssoN+*Qr)F zgLKAJ96Vc-2JsEisD(88mQFIv>=gDLk9<5gZjv`((6qSXzc#y=+7l!jDOP`*H!mAB zXL3}L+;Pcl6!*yXGotkG4N~OtJa>&WkZ5**F_qi@z>&#%W~*IP&j|4@k-`D9zbeNk zyPI#_gk0_$ATOKnP;yA&*|IkQkK?+)t7~X5o5C!c2fNNPj-yl95MnxK2Th>VLuGQ} zEqr2^T%?w`lo&Q+OvrX?hw-b53mYwY+Y^O$i4v?nyA-IZH+6#6TLP$5<@_i*`RGD$ z3o?-hxo}gjM|H{_@p$-L0lziM>qC%T5?1Eud=Kk-1}PFuG_EU1vZy3)6tXr_*8gi? ztT+-$R7o#4s4$CunnFoxZSOyfW?E475K=etnmFiVl(u#fb~N1*Df4+2^5_I=!Dw7f zRIADoNE{!{`jY=JDK$u&t|sxO0|r6>zWQG)Km?Irk#wHIJ5o(8U+@Gm0CdO*bpz>^ zBHn}lMJLu5OyCt+!%o!=TD&ep>V2!?#2XdFUS?MTU+mbm94Nq}Q>`ai8c}R*MX-`Z zk3YeG%>F3L&pBrGY~+0P_IBi6!hL)pUTuU%v`w~6-@R0P!*Y|m#N%(hcf_+9Rr!kt zk`T^^AwrLVC5&>UfsR8=;3qbN4kxC|?L7I?VwP`jy8(8}WZ5KtY>l#7b=0PzzM$@azF*N&xXb1ElX+<0G3Z+3@s@L)XH@ZJ|NI zrfqelt1GczE?gmBoI?5;5n#9cywlgFvYDVbAAZ6l`If}kpl`LwZxR-|sK_m9ClWM< z<()QD{^N{oecKD8o&RN6sFXb9Yf@F{U2X&qN$)-E@**37&2igPw{d1`?&E}EUw)Oy za7NeaxRdCbvvDYK>y>t03{a?l?e4#i<2l(i57lr4Hv-H(h-v_b2aqPMbx%?|?fyDM ziJC@C2&H!^uk~aLzTKEx<6U(-SR~8kcu5Ql%S+B=)`f5WdRV4?OArw34dW{XcSiM_ z=@zqf*8l(tK#wxYFCLLYqH7C~|NyOC?IX!_o~?DCJlzq*MX z5z@2qQ(9IXdX{X1Ht|OW1I0T5&@WnE4e;GMNWdk0*9o1eEQvUAGQ?T60}pM9VZ2gD z?^@fQCsa3XVw81uA`TOX0kDhyY%F#utp>$&dh^5%VYq^FiHbh=dh{{g`~b5LRkQu5 z_nZ}D8Ck>Aay`#~>IN3Tl{O8kkY|Xrw%=Ca2Zfh3ZmnH77^jl|fX(d6=+i2hx{K{@ zx(o#z%pSAbn1o{ZwIF;;j6=@53Kcw}L+6~n;Q6*o9Ee&=Fm^6!&;}*)4K0+=$37nF zB;jf*G>Eu-)4fx8WFhSP3!z`sJbg^j2)*l)NgME9hX>RMo5(9K5&Y7VcFwGVK0IG!R%O;g6~uzxmuea*J0?Q@Z}$XuGxR~ZzXO|FtgopLf5Rf%Tlyje6A-oc7Ybi-+wUIU%GevB-ILn ztrE)+JNMTVY1=Wbq3*11$v7=Ge5lT~SSBT&k{uEqS4uWx8~%aKYliZs?t4HNbBju% zy+B;ex>2A6gi1+rjkH9cfZG(Z zjTFl&-=1Otqm>m}Pe~PchAJe2ei|vDoAPP_a3G+_V6OqrRWLQ|oppFa;HsibhCravgcpE+HVWK8h#2%_;~eJz_+?48+-_CkKXbkf%sHW{ba59vSrIfyGahP>+tW~ts2vAe9m8*a z_>2UhZWHnaqeM^>3(t+%GKxDsP)$ivN#=`|?C%sLL|fXFpSXFmc54))&FCULe@S0_ zn6mP37r$sLC1$R=4}p=c4EtoxHpdetbw)?$>vjndKZv?n!9i_w_v;(#+0Ns^kWXcq-r3Y1w62#oGCOl5MvIK?^FXj3}jMU z#%5II%FJCPAa9T;Yr7z~BWL^TfcISaNm4n_9Q0|p@rhX4&5%D8XUuW8K8`UwGf}wg z4AZO}61e&^K88$P@VHxO*-Z~w(90e1dFIl(sI8{nxh*^flIO6n!n~dJD9KYRqK2IE zH$jE&G}uSL|9PM*X_;3DkLewACMd;0?zNGiScx;A+{R$26Xcr!)GRm|AkIZBO9X=? z@V&@9Q|z$OISH$fY>_psDNS$o7ZyRj&`hD&@jC~beN=d0_-|^JSedW0SQXY&BEPhY z6k?YsuGGRj@cYk+jL7SsuK$fCcY+0$LL>f{RzHTA1-&a(UOTfmY7nf9B*U*raW;`9 zCT<>%32=*LPgER1s+RxfM?4Sn9j*UmNRJlrz@`2;g}B? zn}oQ1`|{UpRtWZSPuUgx^&8Y3f3q6ppI}+=7`m<|c|};&t`WRXMPCl-$|e z=>R)G#J`tHo6b}Pq`M__A5QV-Q)LqR*Nuue;B$uY2EO#^n4stz4$A2cHI><`$TcE7 zF#Vuuc9zQVdmj}8hd3MTcFefu3&XvujKRf5;Zdbsqko=x*$4(W3Iui(;GkJ@)W92i z$@{0><+7L*>em5tv+DH(gX4*{gi)>YHb`AiVFCsz)5g8VDTnMZZS4-qq4TI+<6$(S zKUnminu$3B7#{Sfb{hbu{ba!r7bnv%hydmcY-|uu+@+!-uV705HpJ88HH<8GezWWp?vlGrSQyh;~~3T z6F5TI1RULomKSf}_u3*p=5~A86AA$WiIuovsuYXH?Vv5N0t;tkQJs{69}=Aq``yqb z`DcMTjU>%6#m(H8-T|65a zPl4#Rb``qk_j>apdWs7>X&3J=<82#)RLI@=e}j>IGsD9t-rNW|R4k%a%Y$uN?TgJFbt+;UMnNP@nb^7KXC1(;n|EHS{MD40%opnTt>g1QvFxCm znc!alpvP!HLqJJmlga(fr;;)8wg!%HTD0A+x!rXpc7B$xZ%l>Z<~Q23#~{+=W? zof$dm58HPeK9F>Lvpp`Msh1N2&oa0yk7U9*YKt=i;%PbBe23w#A)X57e?|!Ep^t)p z;9u<})6Cw4YPDPirwluR)5N$3)4roG4EgexA>6jutAyZ1V zKtT17Cpmi1kHC|ix=mg(OXk zP^_UfVK6^pd}uB7@dmZiKW4K+I_`8)&3C$RFK$4KiAJzN%_ND0AUO@Pb2tZwwPz3N zu;C9;*6mCxlRABk1+tIDLKY!MEN6M+q+3K<#h-~#KdQOGU3_}fE&^BH2xaPLx*0Fp z|9ilmh|0smil+TZYvh5_Ogyh2oRE?bG^j*y*N*Su0uB2+1toW#B@JE&7XPu>GlY(X zh&neN5R6oM|@heh-O;xVnlNL%(_*f%{{6KpF zdrx*$t8>i`;!9}y-FIBc*v4vaxHcllO2kH7LZ&92pftbbu@628S{?~jO^ z6j7&-*^-l$JeGX=r@t zr}WGqEaw&SxrnWj%W)S92ukF@mo2p2Hk6*aS3+=C3%R&5Xk+UU*ku`Mjn&+W0rl$6 z$CoQ0gL2B8K$)l1Rk*ygqJj)PJoj+e#*6V@2$;}G36pcV!uMjh(<-cRQ3#hI*w!I* ze)!eGe#`m*?%9(|c(^bM9G_PrmW5c|Pk9rDF51Vb#gu*p?JUz}svz-ym!t-GL(8q* zg$5rAyJQ0?BM+6`zTG1L^W7BcI)%fk?ctjbSPtL`JX_1J!dtK97RxU(a4y zcz4Kh_*x5e-Xq}XkM^|wDa3=X!`NV&=3A6Fv}Nh}KGR%WIr!=!=@)~wc;!ZcA3WF0 zC|3Cr9WBH?>EbpOSH92`(t|Ra;GG{(Y9CT^$_=SS&Su+((_H!?Sg8RAvBAh=*8WZv zEj#O^-Q0y8CO~wTsa6N8p1usv;%ey>dFNXGn+NC8Yii&RtgK%i*7;&`VL$lEms5<<^T>T>t zv2KS^)z0eVyn8qodQkIjAuxluQ!FLCc4p9GJH3J}J-QF$_FWA8MQX)^&eiVdKN+8W zr7*H)@;_p-#k4Ar++TXzxli4$i}i2pTeja!QGw;G-K&EI6g*PEa`O-$7Q?_YUWzdu zlB=(th7~T^4}H+rHDv4>=m3R{a|>Tqf+RmYxbHL<3n)Z;@(qIZyY#-p+({^`)N}tG za@|^|s3vhKpArm?n!b4rrw61}yw?td{cKdSrY%*O3EFd;G<;+*O7C_sAGbO`T8{n( zo$8Op2nE^yp1S}fr(t$(4ZE)%(e*_SU0oK;n#+`!!9|A#$@`;N@~I*>0Wr23?q;No7sGrBvkj z6G4R?F4p}GM zGJ@`~JH4eOq6@0U|t(7+3l|-EC9k0ZL*sx z5|?4?%V?4W#1U@(I3@=r26l%*A>|8{(SfM8)R{eP6{lWgf3^pq_C3iGawXOtkvx(f z*A~{v^SjnSuYoI*qmlF6bYv_%FvaP?dZnV^VOGXw zhsP*P7w`en^PAXoI8KTcB|5so5&L<0OXs}o1_4*A0Q9`v{LD!gsA}&8{c1BeO#Z0j z{VPar*b6(E(z)q4xZRuc(LWuSa6VzOM2UjE&pCUPEbX;j^HglLW70axFuHi(27?Bl zX9ZX)bZDk8(iDG;D)7ZUwLsl;|D{;hOq0fqd{MJ*$Fzx*A1c06WL=X87&wE%%*s43 zWG`?dM!5~qg})+!(Uf@f+Uj+EQh)gYBvqGNjb=9g1cic#ZHFB;(MMW--1(vu8>L`W zc085k{LE$|SGaGdAl7w_d$`psB-sWpYnIbfvp9#nCXQ@_GCMOfiV3b3^8o^qa3&dY zu|jw7#C1$fqY;fTR1J0$HG6-9T~3d{+;S$f2Xop*;bL~|%Q8Be0aT8yyG-eZ$;@l? zwaB16^N!YGr3DOO11kJ+HZfH6`W>kuO0*{KHaIKs+3*-Cm;S30C1zke|9>=O49}{C z@@h#MvdU%s(-9KXK4@Glsk53p)~EIUbg3sFMZh$V8G^ziw_5G_xLdH`3RaZSlufDf z9L+7smJslP5$&Yu*aG)P3Snp3{{1`nCmS( ziXe3C%KsgtrEM=XUB-gQ19I-px2M#mD6$qZsk=`RY_Hz;64!5zUnfozfRXc#;H;RI zl>5Zi@&&O<3h*vU_&jK)!6UgG>ZeAguUDb zq*IUIq7^U?OTt%x*$6ndwbnQv^FniOhc0~4XWh2LW|St)fTeo~pOQ=D&F>Y`49`Dh-0U->Qdpxh4=7=#-jH*Ojod zJH9E2co!lD_xW>4#ddib|I8`=4CALR<~qnFt7HP$JtBIi#-V?_$N~x%@=r2Yp%oc>ixRC__C1W6d!Vy#!4>nov)ax*M>q)F2O1>&I09{i{xT)n zeVTzUoc*)$jQ~`k{qmNkps%6R!wD+i`s+xk#pQO=N7uS?zP8*Cs}bsb>omTCzcd$Q zy){Z@itF7xNU+F#P5L)7GN~_TAaS$rjSvAoqJpC`^_78!9eYfi+T#{9h7F4fprF-6 zFnoPjA~T${?))pMFQHQ?5AetjSOMr0I@Ga9B;pg5b%(Cc^x1Qto`9vO;3RF} zBf+ArFUQ#i9>RGZ*bHjSid@NjC(3(UkS4c@5r0#}6uzty-oL9*=jji)ziGkGtcoX` z&Q~l$^AMIHi42nfaUb5~WKW;39>a(mcAGMva~qZV$Dd*L^Tep#LWGWOZ^*;g9P!}< z5}W(XM+`fH%s;*?3}Mh!Q_1r9>4F0Y%Dg=BUJ_rcjm-q2m`V1NOi%9Ba}U)Sk$dP- zAo-)C=Q8W^n>U1e`Kv;k+b{a!c`me@q#!k50>BX1O*fO9n>Eax+X*473Xwx1;)2W6 ziBKydlsB(ODTw~CgN?mt5S%uZh4fmrgm9S38a5RS{Y zKx@|Y@X(zjk4q7D4v^xob_9&r2VeQtQ8rsnFZ_c(9JEPOiUvHsa)(!)_XnrjZLJsE z%hF!+0bo(b%{cFgSet4kCC=$-KuN6vMOd6w%^6FuI9sPy&JAIJ!5jSe`f9 zjXkF+hrhs%dq#QU<0%FDqJsaw>Wja%lEzarzgtMRK0D(UwxQ}KbDmataF=tKHGvT$ z#grDaTZE7)sI+gy2%mSlf}FQu@Ug2>8hgy^SZRM7!-WaYFX&wK5!kQH>sV@*RWvmc zUY!+Muj2P}sa%{yXPj(6i_}eIGT1e&fqwt$vB6-CSgR~9^Q<_?DRWGJcjIj$dG2jt zq&Jeb`qAB|6k3qfC1b~Tctlfxxj9usxD?hG_18p>?p78+fimFKYdjU+dO=}0YQn64TUrW;safp_di>ZV9cGk zb(St)+jDfwt+<@RNdfsqGLK5q$Dt{mSJuk&>d)w`ntE`k-MFAJO2@>9^xN<}ktJF= z0|br@+_MuHsxMpbb_v}K!M(Xpq9-BdkmP>c7>dM+JZLhUzSU(LEUP1#0?3=9wVX$P zG2#3uwFUA9ntW7h)s#ocNq?u)*oSx|mSopxU)+|cKR1;6TzOjE=U-l|opaQ|+jpzF z#|Y|yQY*`t_v0OmIsS$hII?q9&%Y=zL`fagoHBl8QVsrd21^NzDO8sP3pz22SCr&4 zo|m3hu*eB9m=fl&viU_Et#aPdCHuw$C2+WiHubI-Ys@GWf`=Y#a*Ze6NInbaQrd>@GS^fPbkrMc$m911wvSLnKV`}#Vp+R;TH?TPd^BMtF+H9wfv?!+NUu461Kuw;7v_C88y#EhNIs^g43Fc|I zikL3e_{xxs8EH;kH2gy4Y}$;@FJE+v-*z?LB}@C5DYAQ>NH`G9{G+uKGmgN(rk~4* zp_a)M;gNN6RaId%9?5QqCqFhxqP}}K_U+T?bV~F{miJS-y{pf~V-BeZ4k=9YCaM<7T%WD=>U?JB!zr)Di{YEl;%rTM5U zQvjC$W$MGNPNRmV^bJR-MTc{ZWQ?6e|Vb^yc~ zLQ#ApdOEe<8vK+Y$$$RSrc#$_oW{Q$>l9_xE%nscoErv`k^iOy(uT%H{sbyz=Mz~Qq_v&Mu{c@-f-$j1R1$lD@lv6{;h)Y;d`pS;~f z<$4q)7g3XdF{Wj~BZ85}s!qOqWeWF&R~SV`t&}z1rV$U%GfoA&zt>_+K1Mhd=$>2d zOs1bl;-#qil93WLGgn)wU2UnFN%@wv+I^p5h&u{Rv6_yZBexB`PVFQAObAv?- zr&;z2`o`?w@AsI^XKnTQYaT=?7&0LxG=oQ=={%YOx5#3iL)!O=GB^Vv1v2w_c!^1}EBSDD(K zF%T6Q?DhM%dO5EY^qE4px$-s}IFbhEdJM;H^%7Q76mMk~UPf2xniN`FmOupd=x{y_ zMJ+OwZ0*`OiCrK#!QDvq;cYczB$%Lv$_DF7lOO5M^1F*Us#`<2{>g0uYj(u zh|@aB!u{A&G^h|zsDyY7dhKZhvi4v*hI>2IbehwwJ3Pd4%>YdDJJ0KdB-xKvu_%E{ z1695si)CR*{EMjUT}xy#m@^^AL$(=j_#Nx0+a6<I~R0_EOu~5=;F6VIy>X0*-0X?JY4E7Y$35O zhYGD{MH?kUk-E6&3}*QRBt92OAd)_YgF6jEoL#R}c)o`?1b~y7fY$+1_DL8$t)C$! zRFn9Ka}f^4rK$7YcNPx-Jfd8RAu)ncQnd_wMHPVg^1s752PXNlEt}-^G>XVtx*?DY zb(cmD4DOUEnlm_LcbwLpQIYF08gOqPXqBWuA5=LZED^1&v~O-kFVdjfxn-WmY>S7|yJ^kJb0hkC2Q= z=)~KJcL(&NV~J-8JF9RR0Cuw)gG+m?LU7t=C0PaODggMHwAKII2{oBWX`Ui?!; zie`28v8Q$H`YK4Ab%RJJ&Yx|{6)DI|%j``IW*aVZ2{vj&Lje|)R-g`nS{U|eGXFIp z`F&ODoz0D{ZcU6y+Ql~oB%w$}g&?}J!r0T1X}?a3lQiR_uH&e5(}3*hpMLtjK$T2-x+fUTv9(rep3M?5XBOjmI zxJ+G;q>O;q1C`k@{!uEnGi<+sKc;y}P&)P;?rvEhHqb-@T!{?wQ%UR7pP}S4*m5wt zjyluHSUoSCyCuy=_88;!6pP!8Qa0&1Zl8VwnRSbXIrGPS7Z`<vJj!n zF$(o;f%!LU7#`}lgs!96Sl`C=?PKX0YE* z>b?T;k6M6X-b2^LYWihu#f^&o9OGaWABnJQ*a_>^k*N1JgA3RRdRj9oQxwxHA^Y~) zoYEj<_0e-*E-+H$j#`>jLnt>aLt&uw86>^di>9>CnMNtV zLSTrIm{b=SF_Hmtd`VK^dD$fD=u{EjB-}S;lN8x=a+Z-vcc$WkLAo3}l6Jtnn_>7C z9cuM~5;0WA7fptxKhohbs;?miMOhc=@!K}M(@DifsDM&8OL*qf4WeobDKRGn;ltRq z(E$LKV?(U25c|QdXx!2_(l&Mx=4<6Rho2j9bHitN-Cy^=%p|T6m^!l$m!vP#Xp|+_7+ftjrSv#TcoQni##Dm^* z68k>fS3Khe)vJGT1-#7|S^>p;jGgjJ18-`tOD74&KBnx(NM?=+mBgv2o!=W6>*L}n zHhaWXu)UcGLDcc+XENjTIexm5@&i3YZACu*z~8l*MH@L6N1ZXGc7K!*drJWY>f}Z_ z{4fWbxMV`z@b{&51DIUUemdOfDF9>-i&GQ-I5i8h)nurn*e7(T^YpCK&*)b0;?dKm zl+gvK3~!J2nVVDK=tkm1CcqQ%3;u8Y#2{3V`Y(A&K2P0pq8ji@ilEX$;Tb_pXbs!o zI+`)N_`m)kdYrw*xUx$!{VznL_0;OfrY$GMkuy?dznVF_JG=ER2?JWVLUy=TzvMdM zZohl}CjPnI1Dp1G428dNSa)~LVcdI(+GWq(mlVSV4iIE^|0L9Gvs@Bh)+ZnmG6+g# zlB2iKhx0H){9ny=*HJ+M!{NstV|DUpnQAGzD8=u%ss^?&{>5=SbZ6GxPg`@CVx=Q* z=d@K*$7Ol&oGzzDjB;!#biPY+t`=>GkwA>7sCg^}%7M*`vI+jIRG(Q*;^r4j6wQ0G zhwBaeUM0GPTymUD?yoq3zFC_ub+FY@AuTb=Ie=vx`s~^ml7Isi(_Rk@XjM`~bw_Ii z6atSw?gw8L>B5fDmWlZOgv6sTH-pI<#XKq^uR94@7U_?gTQKU zs_YEnPn6*sqf?*M1df0E+a;u&RpwXz)5nH?O^VO<5(b%21M8w-1d;!4KX>D&&?J^4 z>Jbj=P@)$z+z-SqWCcZNBOv7Osxc+sj0y^{zIAZL9F{xXD~yd) zU;2wtZXASHD0aX$%BShzd*e?T%48`kCE8Y1OU6eUko+wc{0%M@=o7oi$}^pQMP(yN_0=-EmnI~Ip7cPT`dEvV3M!DWvzJNH zCIA#7*#%U>H z9=lfR?Q;M_X?}yo!|80OY|`P0RL_kK&`|<9U!kHG^3|2(+xe6y+7L0~%Tf0gX9JT? zx>$lI5x4?lJRT0MC#Qi=z=m@gK8kqF89*=1{(JNTfEzQqd26vh zXQE}F)1&H7zY2Sspn@oU@hR~e#7KK$z|@dFU7C%Kr|g({7EZbZ#dWgGe@gXp)&s-x zH9cT-F2vj6DdXmzW#weuW1H?zDcfTXP#INb-0AV^Xm$dNIq@U*tcGCYgY=;C|3~uO zV9bl8E*lY?%iKS?`^;vg(ngj%6u7`KeacQV(J0mz?v(yFp~Q&#jW4SlK;3A`H2dg7 zbB=h_+!iGSe4K(jC+mCQ582NVOpFYog95=#AuDlk0K#$y&nV4C;ySfF?8|TGR)H5W z7w--4cM{NzPK*;@Du)<*5sdFx9)pO%{H5pM&<_nhGFDckwWH51VwArS8y7QfdH;aO z!mI^b8ZI2h)5X)$+&IiObUYgTC^Eu`7hyGX_*4d}NqSxsd0;Z8X2z>JJumjaIECnP z7-w+P*>Fv~w{cP=x@v%eLJui>thiGS^b~Hg-Y!)~1kgCWr(Y#+GeHr00-MpZ3jB8E zaP*Nk7s?!!M12@u1@z^z~qr~VSItDD- zm+{BFiCh$I!j{u3qIVrV1(!j#pEf)Q6=(ihWl_KnHX7~iY&$y+uc$$t9rGyuXE7)E z_m_M@>pCRAbO5CJv=%&6SdOgN7ByuE)Z%{0^nJN_vYZW+m;cf)AKhbXa!t|oPzR8Q z`4&FiV!&aUpZhZOZ~yaFaQ9q&wPeO2)Vr^5#T7k}Zchl}$YuWnl5Yh)1v z%{ekm#QhmA#;xD&38yChL9UoUq;aWiwYcOqu>wp)4P?j6;+QJ#q3lWYSpnvg^#PV> z4`TsF0Fngz9Q7!@DSXx??(7h+Tu5L6c13+=imN^5+oaxRuc$j$a_{m@hPdPxHvZU1 zJh!|+hYzxQz*51r{-m$XF>*sN()}_0My-iyDPF-Yfc>wmkM)L!G;Q=8_y2aqvZYyj zzHm9O7gR)laL$lf(!93P!}LKP61g=|e8r(6Gnxy}JUwSVkK83ow%O83Z2c?OYF^HdcKkLVz61cY;jnkmpTZ#!_2Oi~7*y z3^mWl0P&m2LxSbLC?62dynIN-#6E6dA$tq(Tblm_j#6d+MfBGG;aACC6^5|6`PxSk-pm4DvkTOCOVZJh&4pcaF=eS##_BPIHum6}c8w*dhik z$ORD^rWi2aVyesrte+;zGnBJL;$=OybsXNyRh(2%Q`^XjXrE}e%&A}uk3CBAy>QIY z+Cjp*FjUU5IV-c}9Uxq@heZd#jAn|{=Y=Vbc&_SLEw9*q`DbNvF6z3CgIrkn;|o+LVnh+c3h}M-omW9=|f{G z5V=U44qW_Y;YH^n7h4U6$R6v(xw#~&sU#yagu?0TdIMZ}1SM0~=ES4;R8BEJ>frn* zHz0}%aPHv+V2eYu%CNq_(A4nUhR*UrjdtUK-HaiO5J320KjIOC62i!WRk09CB@uKO zM3+OSetBbwn!zB{3rS{DVmBAK5b<@Yo37r&x;7AO>sHQ(&89+C3gqZqBNLzR2ma9n^EvnV9~U0s{bJ{8-Gur>K>vvXM9af`Ao zjKrUvmpWNs4OBYcivYMWFCsKb>!N#LPq{GV7{9zVPbxpCB;y_{6EQJfe|E`#P(~sh zP<$?MEMg!BY@*+gS|#2c%7lROzZC|za2b(bS!kPg3AHNbE`D2}8UK`&#h`GbjN7o#5Ic(Lb{p!?NnaNER&hs%X z+v;NF-c6!lTh03jwD3M{KAyO&i}gZ=Q%Li&`&2lnO|Q886>4=oOQzZE_uvVTA5iKwQO#IppH;oWY9(!OijTt>aNf2hAl^vfdbCM2 zImgw#2R)srnyUm0bMLhy$ItG5*>JIx<4*sl;Hv60aAw&<^V;YY@%Kk>+@d zwl`PJ`>+a8Q}SjDHCwMz3UCa!;hj4bHqyJg@hZ%3Fx3YP7RM}!Z$T(iVjQ>`1>C+2 znqW1*uJv=)rmv;{0o2eyFfD-VR<~X1VtKF^DVXmrUb7226(LKfi*9F7TQN3|?2fet z?t*$xOCcJ|L94n^cHCPZvCgogm}b&M=azmLJIxV<#~hE4n=X-6qNE=m_uYu~d?qJ3 zYg#C%URj+`4)G|B^Co=dXk+~u5quz^0n%K`M%~mP4ckA++_x$aH*ZTn{5XtEF7F{d zLA-^IF&);b4++cew;zItb5K*M#Vb=-t?Io(rORtV1&!4vlTj!qr5!)4AJA^_13s2C zw1=ZaMQa<5_OeU`E5)|*_n3zP04a2R6${+o;M%DZ#pfo@=iLz$)eB9 zzvsM?Kqe@7XitUG7m{JEtZO(vXASiACr{NoRdwzu^WmX=6lsyJK0Y{VO{mse?dpx& zH=`uLj*UMN#W^tuyjMCXuAUH?^(#4G`OBe1QGgRQ99SoTb?`^nI#(c9bv!HS%oMqF1$x8$ig$SkzSjhcf|Hk;fvqJ^3{7jD; zKi@MLH+L(3>=URo?arX8v5pVUYS1u;`f(}g+xKnPfi}Px1sWDdO3HVpO7?Hv7PP%G zdrB9$nuxxs8-$V}`)_pT>dfPLJ5T8Jt&Jhp0mjfN*FV5WA4bNE*QI_O>7o&X3|_+pWGVQ0 zvqb=;`y$g@(rMX1mI{e*t~~j$6O`-)-O1QiXr6~8m+tD3224puV)g#dGoYt@&5MM0 z^y_ERo+GzulEC|kS#BmrU*&m8dVq_z@b{HzX`hsKr)-LjomOR(6jMYf;2n#%?U)_ z7yuYWl5V`LwUYI1=AOt)>bG3#P6F$C=gu(_`*<ojdx4_V$3mQQ=9Uy8Huh^3{LT-)V~F>bDKz0W7b71<&)c*Warf-1znBg9$-; z3~_F@5HN*kh>{cnlc!Tn(20?Q^iIDYa(a2*gbk>iV-X-Tb_j33QTLm(bS#`e(}V6;p8{95$#@N-2E#gca2tUZIBqZ zZmqm=aE1>-d=#x3Pp7#SmI^}@|HqcIe&`P47Z#S46No6=^E$Q5Wm$Up_wC@rR!glJ z{F3*tKx;46aY`eIq-FV<>Hw=vZg?|PLmX&XL~atwJ>}Tbu0RJguKA+~&rVwE$n9a> z!KEP))cgMz3;XN3<(ypKKA*VDqI(>7Wj%S;(%0vk8hHWJWbTb1^|jT~Y?%||oL>ZK zakJKFATvrhfl`?GkP#_R)rw3oFWeJ)?2-eMk?6Sf#k1q*3X`&~ZLS<+(N-(}4hpZP zeQ6B^@(yHJQ#k_vcMZ_@sTJ5*=rWG$kqHX-H!m;9L7MD2LUkTlQwgdO`FgWFy{@WG zBn0|4^v1ifmOnJ_Bx-o0V?G?wcbFaKr{lfLJks2DuFgu-t%P}V4Y)1|aNWgeA)*TM z;5fGzQTFZ5@F93+V_VUF386FCg}!~X?GUJ(jB>#qdNaFsvZ1y@c>%c_9;a?pAMG0G zf*M|5-aGG3kmJM+w-EFEEXt)No4vIBhO?_Rr{!*L$wt5*mgXZDpshLpO_TBISrp0! zVaO{%AS~i8f~+(T4wW;fO^ouHgV9o3Yc%vG%#c(R00Y#CwpVwXlJozxy4zvfLSm$r z1i9gI6?w9y{5aM|OGv$cP>1AZq_4)gxVWNf^kwXLCO-+p^G!}W&=1|twIC8vT zyrFzX?EL)J6u5$#IQznds;WmBN>!y=IraX%&y}v01T1?rP%O<#TwJoru2GfPG>#r5wJ!js8>q>SX_)>lV z`lG>q)1VwAwDo2=ocLO??yYKMKmi}R`4uG@tn(GOP0}R%`)L>T;3KSmoXLG25X@>r z&{)(YCdx5h5%_@1@K-&w-4)$4jidY7M&-S$ljn9xjPwPqNAi9*^})F z529o*YDzLRg@eO4fvi6D6-yXOn__=5So~{*JwQOzzAs_|qivgQvXx~0SMHVp>K5Nslrv4wVQG=`EIvW=6`VaV8=`bo=nBNkj!NO8oR*Z7 zySp~~?Q^9HC#1V$5-UDIg?LIW?!#U;2@J7x238*j3vzI-?r->z-^KhDxto@k>pbfe z|LTB4Z#%$jd^0`R-}_ulBv*vW>KIz*GYw{8M125+0$X2|(3#n^>k?Q-KGu(XNhXVH zPYnSDhHVHtf!Xif)@iFQIUwQ6)}+ASFr}w86;~n5x1lf(%#!((gh0!ih1wx>!U=GRB*W;no!+? zA*VO!;5%=~c$Gx>??;dcqk_FUMX^uNOBjTM1ylVJAaDYkzw7isGea?p>8=+@^cjai z3+Yl8)ozSk7;t6WrJI-wl9vQD2guaUHgCnDlx`+E>u=kb}Tn zL6*0toZU%zb%kv<3B%mBn&*-J)F?gD0zH1HcG-;Ta_cD*^`j2FxzH89_yyxDd_t{H zj-|apDUWXsf^N$Br`2Rz2Zgc1DVZ#B^z1^Xy$V2=MIIu+>UyHsl+e*RhB&|o)j9jP5;!h*y z=^d^C7lb_J8K!z`3PH)y_ZQY@yABdmtD%6u0do`U5U^^>A)zTz%6dCcpT&@i{u2kJ z$V$S|?tnGwVrpQ}xyV`OS_jaCLq9eiaZlK0RR?;Pl)PFsO5(f5(`C#A%2Bce5@(rh z>Ju{{*1nJoH4Uk|lJE$t2`5Q*pq;a8dLc*hMDawZy2ryzXxUfNku$uBA7L>3%-vfw zE7$#}t_?L^@-X2m4{^}nE%}zFvG{{)yG3_yB&R_{P3BC~*wb_QZXs;EiL6TC`FXny z4nachMy}iVz%9E+y>k>skG&Kvw1J9{?72QW=_Y-HwH@+PYcU?cNIRD&x815O6#o5| zA$+%m0{L_2-ugh7agTGC*A8Wk6s|nSIr7Tu==6u&Lq`tw&9rNlNN+<&q6YeMX~??} z6Fjsz)!axUD9ij6@>;;)&=d!NfN$~y0<8Tv!UezW{awM7@_Ti z?H}&mpBt0M7+Lp9MH4eWkCIm8$PE;$I^Mx+eF?ouLtmD=BIMqk_z+R5UIL8iVm*(Z zKCXy^M=p-u_sC$dpGbcN%5!IsAWPO4AC{FPg%o^gdKov0^xc%p-e?Kd!-Zi0;>d3D zO7`^sjk0Gc^-&E|_+XgkaLtd?pwzA)<)Nn!DzIVcaie2k;QXPVUVzEF>ey;53q9W= zf!1RIImsE+$_hO7^0dQ$5>82j`@KRh!;te7-8YFTPe4Be+B7Tw*+NqOq zD{-PZph$=5=Rj+{c62=BsFS@06_hs!f3Dr(9iZ~TOyZ0V@2etz8j6cJAKGSTNgH8_ zWFO&R{NG}5MAOuwDaHay~HZ~|?br^H$00^kfdsJb{GsCQ0Bv;W!S z$dcCfVR_lWJ{%uiMOfTC!)x=An~8EpbFzTaSUwGhsnG)&2Th&rcpi^^<)?+RZT7i@ zeq1-MlUrNnsjrcMGZW`Uwx$zDRzJiZ3t-nMpTjJ)uF1VA0w5rUtf7#Z$+q2nSrry^ z%2VLJ|C1aGR&aoL+EC70Z-$*ywz5NqzD5RbWb4Z1u6vVp)4$$=clYgBmnm?T;7#jc zKJy=g9LGTX8mJA=9HS|I@tU88B_jQC)j00rA^be{To5-&(fAWIBH1nbZ1|{Cx8vP$ z*3wbT`Bo=(NBQH)7_`Ad^UBLds`P0gK>EfC#&;fP>z_O7FLisU9X)1_bTOGKqH9LY zoKQXA3SN(lxh^E`+K(FW)31U2vX9 z30R;Mwac@rxt(+lx`d2Yd*7dF!7Z?V)tNtM@PqN1{D{(|m=yq@Z1-~B zH3LFxI>Yn2alYA=OOItAWYV(!09dQQH8-J^F@C8Lxf9OWxa-jRT6a{V3O$w%XCvqP1Unwr@C=EhZA{Oi<55E7zymU^CvVE_&}M(?n^NAs z`Q|*}tLwi|y*w8w?{?`udc(!f$_ewv_4s4a{r4zE?oi1FNJo5@s-;Jb;ecQqD9j#Q zqH2o)BN3;hevu{4sU@%M23vN)T-7P)NZzU{`6$wMH1Y4he_8Q`;EKm`l`WFH>q!>{ zM%2b1xXrs))^%9jc9BjXy)igP!aNu*0OEir@K8vf&JEYaw_=<6Rkz})UhKl+3oO1| z*MqHa=AIThh?!VPK24le<#zTD=#EB4 zoh-&fwQ>aXU8-1jm!5D@h+|ssrDvj`7kz5wvw@lvA_^mcE4iR{ks|4EbV#SpTpljm zlD0T(neqDUWZs3Pjz;waLKj?bm~>ecgaQ}u#Hq$}qmrOtBv@QtL5M8`)_k)f{L%y^ z&!&f~Z7f3s20D~>edjb&%l@q~b7TjwaAzSo>Hh{6>EcY;vOD-+ z`q``3scr2V+4(~rRdux6u!O)LJX^<{%gjB=?cqy@1o@N$1*oJN?WdKkzSy>dLhg^H z&7oWqlEW8`Ywwo;|D7D6$aW|y@psYzegmCmZ!&K=&Sac9}1zXvRUY-&g(T z^;0MDFE8*t&1Yh8%fj<(teC=|v&f@Fn!%j0jJg+MY-^f^ll{zPvNfs9gVfJE%wqVd z%AD1iF$zL&NCmg8TK3Bh@oJsb{uOB?O}zWxz2Fjo-|@ex*`{ z<|Y_yH{Hk8eVaU=d0w?+zs6>^aqevJAey;KgZWw&br#_!nuErQ}?hY~6jXG$0r<|bX!77PFcY^BjWC26oS+P?cdO$+b zu-5X>tBW@_iA?jSx2Afp#DSZk;%Ug1n|cDcK>LPxI0X_999=JJYWA_$@aI*Yq001< zdY~$@5M7V?e)*|uJOFppuT{5@c@KU(f1T#GHG#~f-I7q36bQN~^=8ok0dqj#qgEQo zU;C`|NW3s&jGZ#o3_vagt&O%R#@0VGyKNk_jt46Zsa%uu2?yw&RnBi5%ijwt zC(GMA;V%ZU+xFLIP?@yS8Bq8@jROFNF47&(GQco^cB?Cy)gP)qY4&r1h+CaSUP?=m z$11GHZN`Dbyp9?K+jp%h9lCd`B%_NtOkJkn3Wuo>y6uI;ty6Z7!z>DCewl3-IZMTF zD;=*C5dEyDu`*6M87=j3l_l$(=iw}nkp>5A$F9!e1=}Tvm zuyH`CqL_+h*(8~o_`X|b-}JkO^_)wG-At-m$H#xZ_rN(C2& zDM_PJx;4(r6z=TS!NWsvZC?wSM4fbI-a=(S(QV>$xG`(eL`;AguS8>Sy&t`$Mn_)e z0-fmhv*-pSne&HJk1!IdE*#Y~0XBpL-=yGT^A;EftJ_sncO$uYc?%GHj2bDm(^!(4 z03vd;fE_f$y^#wn!-Vx%jv$qmrq6eO$t_OO+h=dpDK3vr^)cQNjd!IV@Q8=N`a&nN zt8^xvZ6-%2Z}lDpq{Vd!k15VgNJ%B zYlY_VfgPD}134d}bZXfv4-6(VH1(uVS!esFNK~HW{pq90{7*& ze(9DW(st|;Qc@v53OESZ1>gib8ywcWM^!K3EspvgS4QL<#R8EqkA7_kWzqfs-5P;! z<{YZ0x7r-)stQXs@E3x&c=so@LpUGKtFwer=ZYwvx3BlCwq)_CA|sptZ^wr4{dufR zlU8Za(|eMGGcl&6PxSVI_`0dvD6)$lH23_-|q8Ll~4vzhIeU7quQ zZTeb|OsvX(b%_31~x=%l&;vi*J0+zL0&6`I|z_+vaaG z%Np&YPpG~CqJmY}(p}FKU;R@Fg#@xI2pFtrQEw;UzWC0&9*((FZ5QWGgPpnre1XKvFiyy}WPr9(W_+ePhsHsdSzdfV0scC{e&{j%h%4SD!7O8dx}pc zKQJ2pz(f5;chC8ka8yrr5T;`r9=y5$#`41hPDzQA&6eVF|5T+cOe`j~euhRxmd+Z{ zry1vFR-^c2I9#}iB$>A<8S;qfP1xn1dXK6oLJ0Fsx};m`zsp5+QQXs4ET=4un1P?2 zACEO4_0E$RCLur~K-Yuwf~Sk~v3uH=skM_g8xOj_mL5enFz9Eh+L-6U=r|fog(>9|!yY zs|=F-h+Qd0T@Bs?oP=vR1aF(UKPLQ~*SlAw?o>G4XSL%mokt?nG0fLk2AUTaAdI^4 zwO!G^(a6W_YpydlvoGXa^LxYHMd1fXR7-VZ9sbP!b|^_M8gO>pL7)htWnY}@?*(4z zcu31#<~AtoHTmr-hF3!V;xGOUpCjpNm#Mg#*H4Son#uxdE5IrvWCx3WM`F<_SGc56*xVlUUyK*Y3bAZSYUHbt_j--iI*`RE>)ZLElnm<0btn>wTPU)(d@F;-*Zz_8|S9A ztW+p3@CF8m4a_pfpfjR7Ukj<+L7!pLDqyLXel|s1hG|fheC&&p9E#-ENPROGjL1ty0ge76q_ncGTbuHHySF2|Ck zYz06as?lT=dwW{uX+D~{fvcMOL(-&~mkX3}r8+Tts1)cZ1w!S)pu-ScR@ zOUun=;=t4n4~z@8mR&%(n>Dq_NBLKQBH$WlJGcjIaYe!34f2-J4^^NOJ3!C zp$|2czG^DE>Wglg{*RI|yKTVCZVbZZ88HdHeUwpE$@1VqB4a#O3Bv`gy$2JJ!l<8^ zQ!iI_k=bB2a*|>8)+d&0s`hw3_P=nLv`oG6>g#V@k;b80X_xuGR5y3ty-zP%rb@E^ z!$!P8cSWxQun%Iqp$3Ge>r52~g;=J$`nq&1;TfGofe*8suzM4|Z;_c#oXZ_pDxZzP zWBGhY&Zs>1HtQ>mpSz^Q`yl<)i9$@^fZE(Pu4q3wMn2XD<|z<7$-Et?FgnKfqfrzy7le)Bw0@_nG+&)>An*t4a6kBIS{&yI=6WICB25)vdl zCJRNf;FP`z0!DDUfzu0OBQ7#=``&>dkd|T*_d9QI&^7p+9zvuFs!t=30c`y;HIr)- zc%e461k5TTE`K_rpXI^32_7tL-!cO7X(!KCCsna6dFBr1Y#+yW93D>?DV4ZVW$xyf zLT%NgW2}OXNl4kjz&%zWUfy@3xjA-=kU{0^+)DH1*l>NF zE%<;W4Of1z#6>3u>G^co@>s0!==ux5m?xV}W?crM?e1=!Rjy3JIX5xR*%5f4^^J82 z0HyYzCQGHlY*4|xR1IQZ9LDKCh4^Et%Pow$fa&w_K(>gX5rBqrjj>3fYetT|CWteA zzpxZTuY&2YQD_q%@xy(KAUlCt4%98yAD!a%4;AiEGjUVuzfaHmM&?1jf0z3rKNKK% z^kql+<|z9Mk|*D(~_3R;+=@Lm_>MRWzjfCKcac8kTXKxEIYM75%Ir%o4ZivBf%+WLy|i+DW; zl5}eEakOrvb?Z9({SVhjMw)=kh^%i6FANt$|BX=a2oaCmn0?!->6r^>8AjlD#r}Qves;e;nb_WGX+m~z<~gSr zm+F2qGy&Q!JBjP!RSk;c$Qb7|CZz@_27#~O)N}M9zh*-+SJ=xfL(a~pN=W>&!W%|m z8h7dx<|js?n(9~iW`S=d>q^xUTOhwGMT{b@fa8fI-xisb!BS~ijCMYFFFS9!3$pAA z<*$a`1@USX@}B(>2Yi#BAJ=?ZRi6@NQ=}gXSStnOGv`5$HY5%a&9`h8(Js6Fb8gOJzS!CY*>=J$^flkoyq9FFxi!fmb5;yf+ zphjD!KKu2i+{JjaP9%B=j7h_?7k@WueiF!As8?@WL8{DB45Z_o8dQdI_FhLCfbn8!(rL`f zNR{I>!sST#=?3op>8+mR^jDH$_-yZ?( zQ`$MuSj$!!4TMhHo7)N)Yage?G#X}tHuv%IVoXoDJ{2q$Mb%6icU%*+ayyhyR~aJl zk*r8kwqox9Z=ayBtPHR$a&k|8G930E#^P5PO7td78Xbad ze=1$u_=`OHO77-2LGhKJ5ZMO12-bJs-AwXCeWMLn80@eDY-y->M7s)dMiI!?sKG>; zm_I5$0KiPkBTXIGVH#uGyO87)5}{Ni4NMB_jQs{_X4c#*EqxM&QI&=~e=#ApjA!$B z0)eUtYoYcvKH&`~UiVUI@g+U%sxD?n=4n%3h5}5uqhl!MFNuMU zWgD@axJmg(P+^_;_Uu4P+!RP>Y^>Ebfh@voYN=Ia)>7BE+e;%y#@ch8C)feBI-Vg) zJ2rYZhEH=lIwGWcJY_Bm0F8jHzx&IzOTkjO-kUv{(PwEi4gb+{mt&Pr|9qMGV)B}Q z1X3{71%POvW=$Tpf^vbA**jL!%CXP8in#c@GZiB&D%=M?rfb{@MS?m@eU$`=3QZU% zQqrnZeoF#sEi+D)UJ9WRZ=d}yd4Tvax^`Tl>i*`<$y3JP#KgjQebD>D?4%+dgtXA; zIP_#8f`Kj5NCl129fP zb<~Mzh!$wUFtPcpbb9V|e=#TI?x4WP1K@(VQ(A31W{_Bkv0HD( z#pwjL{>e;I7HV<26Hk|i8)p=jMQ59y_l)w7N&1g+o4~?3G;84Q(=$H?Vclv$y3nlPjaglQO)jAt1};WW0?QFYD#9VJ6a-x_+RCd^GOBpL(YheIGY zo3Z>(@*(SK2H%+vylZTI7Y%1{h$w9UzGv4?S{HXvAVSxl`N6F~IPnk^7Uz6&33%=L z289f(ZZevw6{cin5Vc&-D_W2L{rlOjX*eM*B}jdr7G+?aoVBIne3>=(DaNq92iX=U z;h8DK zz>CW!6qJqtKOu+gtLF-QR-nFHh)b4sjLRG63gEmRNJ0q;?yy;w;k_zA>DD}@?hEeW zCsDyh1R(aLfEVI%$xjwVk&70y&#@(~D)+k}r}GQKksc3V_^JC*;J!XoewW5kgg8kU z3uID_gr{9z;EpC*uN}qh6HW<`oSix<-?FP`;t5uz+xjOe^18A#@}L7%u4j(tFV_je zAz)yQ#=-a4g8b0viNY*2w=MX5_!^YdrzGkvN0QLQ^v@3e1`$t zkg@QBGY)Zv&3Pso&ZRGih;31tG(%<7q<&+8G~|b8D%B6MMML;I8rHqz(5oK;vOHs2 zl6m-b7#)}TDdR>kTlLBJ!IxIKB1_y7!m<%*YbVA1IcWk2tfTkmtEqUMIRx7wxZnQ0 z5{PHk2wFKDxvcDf&KPDQmFBk!`CLW=!LIjrPQ|N@g5(44LCJ8FH!eilEF1r`pNF#} zRh)CDxID01EL#hb{*=y}lZ_J9_VR@|3^5lo2e~48W-kY7@61Z>fv>QN z0usGMD~(3cj@L;d`RARrLvo`=m0Jc~h~^k^@aWA0{u6TF!(_fLH5{Rl>XYRTvtEM_ zV@w)VB7`Ck)n=uobPO#~+abR5D~3!WymLM~qcqDP0wAyid!6(_GR5U2l#SdF31<(O zW(o#p@%8^2w!T-bK5Z@Q!5;p4sO=J8W)_pk3Q4=vdu++VcbPTp)pZlihg+fwGO$M? zxzYC0)3VuiPh{2h0B^JDCq~{s)MrmUrpFuK@Z-%1AZnTMy|N^iTr8Cc>%bL41Vht5 z0`^6c0MA@#75Vb+4_?YmM`Z(g4NSS7BFcWoSG1F))nUk+Tq11J$m(eayp2j6wNjJe zJO!ylp#C+zA0_;cYd;V+i!%Xs-1?>(s%su1`e|_Qxao+I9YoeXI6Ks_HL~3y<$w4~ zm2dm;d&=B!cClu`@qdx*0Ol-z1j&7JZzMZIf@fmnkjiMKfyEX2@^QYCK*^;P0h zGsr9Lx4h(bziEu@IS2x8S?7^3m%z34|JGyJ))p7CcBHqK77+I?bMn&fZXcZ-7 zcy|-yTY}xI1UheR%O%8(TpaD^t)-_CJAn)cD`5tNFbW;aeT)ORf|FnG4x$+fF$y01 z2{!}pHBVESemBVeczRPdZ6<$8rs;L8HyM-dhose2O~2#1#&=y%F$m)(s-kqXW#uWP zi%kTsjS`SbE*du!pX&w-BxKsR5rV)@_|R0G2xiGhb4B_24ZiR;>>AsKa$Yd;jcRx+ zfm9#LO08TB7F{z*=^6z!_3(P2yH5mKEZ;IZk!-;>jB5BCdYTbo{VECgegF>@dVsoU zGIx3vJO7OSdH0QdTCC|b?;r=y^S7|V@0^*GSUG(MdsFPyd;(f*Zmz+DS4id11PPI# zh!mWd)Ej|kMokf}h8URXZrdc$ zm{gchm?%&|mR*hSJwZ|drt8VQH$~_g9)rZHJv8Q2I*2Rjmws^3gfNhCaSrfQlWcg!U-RQQ`6PB$XKGYKYS42VN?)LkvQw?^CGJl`_>Zioq(s zH+fp5gMMDJZiJ+MCMI3a!N!zE23bek?*;b}>pZ8*{|Y9D$<7`+jX&zUXlpX`)1HAiE2c2zA?+sIfOahim%Yf* zRU0x`AQXeu44mb(^Sy1Pc~--zhbg9n=s_5- zwLB0^y^r5(Sd~mrJ&d(zc)snB0BVivB|yrFSJk66sqOuLTB6j_2bP7XJl`w)@b_H< zXEzum1;S5OXv;!}4}13FCty%}#8icNP%v?PD13>X!3Lu--OzW)j6|Eo48TYMNO8&~ zlB2Vj5M)*6uYq%$;_T}H>`F)3pQ)M98>>%0AiqOKp~pBiv&KWScdZTw3A z-T>yJ^gSo{tyOVvWRV$N^T~l=&OIky!Jf&t!KWt1;yo?cn;t-_fo$Mapggt`+6PFo z9l&<4s!k3n;W`zxbLl;_7F#l9Xp&^;$Ds4ur1FKqfv^`}pxn1Q>yVL{h!2w8WHE4) zjZeqT%%`At>VE0z{gJC7%?Tpt$|z9|NMBvAO1$Cm(ipW5$GTblHVLfT#u;ihLMg(v zNmsjchnA?m%l6YWrL(D%L>%XJrZ}k)kT#id9WcYZzL&l{Aou^qDm*^cTf~hbDt2-> zq4p@rq;PvBzO14{(g}L!@tN7iR`;VJze4sU38_$i5HXf)M|st<2xUyJn^E8p zF~fhx_R@A4zIm*LUz&XWQWzosLaG!h`;BfadCBVnR&%eRp5IRizeRT72jjm;%`#ds zg2cMNh9jd<>v8QqMEEeZhOf$cwIz;z#I;3A8)_*$-zBEEpR6)qXM)GvJyVGo#+Lh# zQdkv|p#dG>4a2fZ4=sbG%y{kf@1fLoCJgNE;79Lf5 zR!Q|=odNvgj6O6k>|#4`jbkSoiWE02Sr0oYZ&bz9uue~R__c^OT|MuAa9 zpGF2KMfEMspAqEf+ss+UEvCFb@JH=By<3XZP{R$_7?B3tnqq}&~x z18z4cW7nB%@wqT{)=Dy^OX_{F;gK?I2RXeXES{2S zd|j*Yh1TINdWLwd3p$n`@(2Ar5Ng&(#LL~Phn$7rub=rwu1zyZ24Zn46h{#%87h!L z3>M@{w%lBBc5KNp9!=*v*za2K*8$xN!S6+K>W^C39!hEvAc`WmbdSNZAYxQ>!g&M` z$cnt2IVs5n*_DUDuya%(F;U=%)6I%lb7R-G0jshz#~24Aapnl%1O>pXf7nstM*xN2 z%WhT+=;pE&=D8~J)53Rf{6&WD_Qw|QcL2|h1Bpzvb*4QkSm)) zl)WKGjCu^yd3Drx@%7jswe#k9@)vk81a-NKTRT?lK)JqIdmnT-K(3^{oFyu0=5Kv| zm+;=SGQ3wJT!3oCz#&_k2s%A-1}Tr{aS{2nx1)#x9Lhi;x25NrZ*5U}lun`wAc9~4 zYK}D|llxU93nQ6Zwl&Y?bzf=1^OPubHw%4j)q$#`EcN=hM@bV~kTkQGMcJ3dXNNs3#`ZJ?*cc;rCcqL#^%W?I!t0; zWD|MJG<-ja38O2dOqGITy>XwCF(hglmh%5OZ6oZ#Qw0-$LlY^jR8j{kj>KnD7f*jYi^tYlO`qwiuIZ=#ChyIh2(rXqD>_ql}GI9M1Cji~z zz=I0z5;V5FkpuX2Ad%l0{Nq?S2K^6pk_(GIaSfCoh*M=%5qAQNgi!{{zo6o4|f zx&l`G_84tI+E&mHZ<)2vGYd^eTBsREbnI8~_X-)(Kpduo<)ewhXRHI_LEX`22E`8*~*qis~F$$T@T-%H5{grd!049JJs z6OCHRBoi9Dl*8+Hzx-!JCX0Mp90_cgi`H*7V5a(X&c@dbLC`W%cU2+BazdieNG|o_ zKi&gbRRAWG(N}0;-}LzSR$kyTFg3lBysAeU={878`x-|nkEXCazlcOu)Kj+(?Elbk zdx=Ai7Ja0M@q-h~Q)q z{N)xygcGU1bnRcIb~+HE)Bg|LuA8t zQztzDPSCQwN&5un|Ff}~ptjFZCUa|X{*IPs-iZXq=T%H+mQn*Qa)s!ItM@@*(D+Go z4Ftlu@S02@n&L2 zB%&%&VZgv?!)MhPckR>pLnfSW&j@fF3H3)G9Gq$bgC zq2h~ZuZ6feU-4o??RxU@DWVGW`v66*CQ$Y-fi9iYOBJ#^);HPizm(_%(&z&%Xfg7q zj_Sgq>b3&hmpF019VSCtw4q8@Dwm$uAf+|ZNgC$@q6);Uh8U#sTqwie+SR%^QBy?! zX6HsA*Rx(y27ZvD>yb)?5;obWjtvP-MLv=c$(t&}eFBD^Ry0M!Uck3f2A;du)b=Hw zVw@jrA1hP^H8rxFkaUE@i6z>CS?i+Ydr2}5RqO4zpXogAbhrq96YZ~$Q=f&@jWs?mASoe)&AkliYMXc>>;Cob=j5}JLdBp*2O|t zEjXRrL0^Z-!moJMtfh(xEGXc)aY01h0+oQ(QO3GTOH`m_Sa3iFp8=o?1P2IH%-CDG zi#;Oh8ym8b#p%#>k3}`j>U7sIMshjD<8M`fNLdkd*noYXqZRvOLCGqTRTbroN&{@m zZcKj4bs-4UlJ>>F2hiCIB=Kl4U9*FRvHcFpuVLfFW#7jHT?8k zL`dV31!jIA13VhrPHO(;!Sh3@5o)@BwS*QF1Z?zdxbLhePQKv8CzJ>)!%YbVDUeZY zP;&Db{1LS!3NwuhVoRjWW??#q<3vhM79YW5>5lnMeUm>%m`+a9=H}J1sjm*(ccizu ze1&0{VP|JVR4^_>s9;badp-Qz1GxP9&%nq3;W7lVlRNpHSHNCvMZ`SRd!M^Wg9nK^ zWnSCZ8xxLPCv2lw1K@EHZ;PuT&Ac1Yn-OtPSy=nzosxSOEr2|9m zB?2#{OK;l4)>IRGh?rZ(OYQKp-I`3uQ>vFM6Rj%VmtF3rS>xC~W$?2|3;nqAT*E05 z6nF*^9=qI5fm*!W|41>@3MZ`vg<}c%6L2!HOsL{HjJZ#X9lP;Bj7>*2ylccxsk58K zk&Y-&WW=4}B?ADK$qJe%Hy2uEM|$F2Ur)@)?YndUAsh(ACPKP3d={ZF^>jl9zYMAJ znT<{}wvM#$05r82j3e&=QrVmRYcPyZFU`5m2T+Lin=B_tzh%@zB%KD#DP@&!OUA{s zXZv18`YA`3{~zt{Q~Vq%L+~!rNg-CV4(Tfywe>qK1%vs$rgFp4o0r>f4vQjM81YxZ z^QNEL<-++yb;%LNb;-Ahm*;C{&?2w&eAD-x(Wubtg(K z3T=d|C-iNc>ySS=9foS2?quZAa4*BoRogEUICR09UmHr!ct-x3Gj*>E8lkJbNzcCzz6eU4+mFIH_E4!^{ zgSMQYcie^QB+4v6O;uO?Mh203kGtiEn1`>6p!fsAL&UhWi8iRh%rbyJ{jn}P-Jl?t zb8@n2%cr3>PF+6imE#vYAv9Z)x8U7iIu?vJxHf303RM8Y8SBs}-YJUcJ*7$m z{W`N<+4atoG!swu>mpmy5^vjb>dTsj!kO0)Lx_jP+mMiWf6Wyh;C`U$b%kPAB9&0r z*8(^>e1VIwi0NZfF}>W&{FZM*>*{IroHa`BX@p!irqBOq%=1;c!yOn|c!EvmU;*bY`*4ZPSB&j?|!{s1g?Yz@Au)%n8Jl5L< z`8#RqY=t1&7ZF^mHR=IL-)5yZI~0*IHL;UxZ*#HjzmRL|g0@&z zM>3jqdFn8d3~IvMdD>&oNuFJGDb*qv1=}xzO*-)?U1>8(P)9sVo^c-ob4VX+nIN+ z6}cEoxj00n!UFth4LYCb90XV;ftLD&CCm8*wo<0|nMo|9~x<||3xFq_acjdf%YXf&jC|hW-M1x{>sD_Hn6n5x#DWHE*e+0N*0>} zENp;%{$^)JXvLb{f*cFkq;Cm!J1<5EQ#28oiJ3x^vaY+uRk>IR8d0bI>=^ogvAFJ0 zEmm%A5-;@Hv2jp(9XZ{Yx$=WoYlB*l5;Qag_gQG~JE@wJPq_nH=9hX*lO>%eJ1S2K za4j@s5bhynUx&b-gJk2+V}UBlErDPqsyod5jsxj85A_E$KW952>RMz%ofnJtLILOA zt25TMlZ>1rX&A%O<2t4p!9TTA&TWe?oloqN4#PRO`2`KYGsqR@xC9zR@950UEMp&+ zSEV=$TKR2o?3;MWCnZ=El|U4IcvDf%pRI2~QV@91I@IU^hH{Gz!jh?R91@YD&lu(M zd7r_O1=N`w5YI&+S&`|OP6W%+C3jqIb4c}!Sk7i-qsNq`Gk4EQ(e;yaQb6FalXB~L zucJkQy*da-0PPXp6JRrZX}Lm5b@!ej`8YJ|O;UAWlWE)JDyIx492ZIk5nU zSMyd^#k$31Z@lIo73X}%%+EPTKyNPC3!bTEd}2YE7C-KhP-aTd>W`@dcXsu8fuT*r zKGonNUc!A;Z$m#kc(BuUszyV}HfJ!gA{tEV0;sjF2LECV*Ay_jNM{f6AaT?UV-5v; zRRsi(Qn|M39{Gd0C=mtM+9to+sSy~+J$+&IDGa+@Kr3l@;5Y4h2AmQ|CPL0fak&3{ zjHNPH&^}JdvXX~4q%9@3WVj*9E(bMw5dm;NwNw=dj`$6NUax}uhcfLBgz4$wQ4tBj zdshA|{ftzcQ|z+5ex8(Kt23FH>Z;+UKr1`PY$zS%WI0+N64_8l{)k0}kU*uUS1>ke zse6qAT^-nC<2Uub*qm}2ogESm1d^rcYA$YdW^XolM43fyFP`C1CYET3Dm__AB?im2 z`2W4`A%HjvfnlrQ0vK3x8IPnr!x!$RSa#Tf`hBC5cR(k;Wrze^Jo;h@z>>jvHb^YW zlk*j3-I-Kix{3vR_}%L^gZw8lPJUU12kE=Mq8eiv{M*qnNZVn_WUj2JAIg>YwjS5a zSnPE8jD_NTe)4ECgFNtAM0)E7tcS=x70~hV#bErX6E)q)DALY9lZAbH||2%~1buT+~6wNmDv_ z9#gm4vqr|eW1bmfIn;#faCfJnqVtJA6=FY-5p5Uu^QPxtH0{Pt@zr+l9e^nBGK0`4;OR$_5@#q|Wco{jrQU7U zMn8uiG!$}l$?)^RjYn~&@>H5?4fN)lNXtU(JsTw4UmA%&O|SNqqIyJdxP8ut?+D?P z1q*$R=bkC|Vd#x|uCOKbq*H2-ZDs0(t_-6~r1JSL`Y*`rewg2XLN}ej71&fo((i96 z>_r;hU9}C$)BFl6z_nyLZSkx@xo@WW@kU|COy{4hjv~!(x{T8FI9(fMd#NSJs-l=1 z1WU+TKwcs`e7aq}IIAs1wk|_((#JD?x9R!fa1)Zu%X%TyLDoZ)<7$^e4o8F#q)wd} zF*=xy9EJUjp<`QCxEjx%40VW6nOPI@7GHec8xts!#D3CMtUA_>b1EK%|Xpmop&IKw#+O5EPYmoe(9V(A*V+pC!6+26X z4Oo9h=0}18N>8h=9eh3#3f6CNBr@ZyJN}9ij+saYBJk&7;ivbeCrYE3Pzq`XZDMN& zeT?Fbu*@Oo!IBgo3qA^rru~$us!BB|8g@t%oXt>>u~K;T{K(|VpSqs(Y@w_o%*>OD z^XugQx}S`3VnsBDcbWd7eXkRqZOs<#aT|MEA)maj5@t`fe9yCp!sq^wJU@L*Y&Mrs zKY}|37Qw4o*WOuc|{4pB-0TPCknF zX~BDiR!%ZCRGxM5pK2F&-+Nt~6fD;3N(ddM34Lnc3)ROuWWxq@nN-9N} z82}V^Oy=sW(1jJWNXlA>OFKFUMLPeZSIn%LNyN%fTSIqj94KA!J-UwQ`;g4OooPWL z#2lAc@J@|Tv9^(}9C)l;epg!B*^x$^uG=m1>4dhLvR9D!;mpm4mys!sjzJ~mx~<>R zM-Q6em}0luI!eQm5Z_CbqEL3%S&3bGqvvu3bVAq){sus5(JSkCk7KM~p@%CjGpr0( ziG5h}El4bJ@I?sRZktXkXrU_M1l+@giu17x%dAbF3$MGFRyjg~y^Rp)lQ;>!6d720 zUor_yEV-x4sX#MKE368y(tiNr5~ibab_*7j)$pOi#RtXH2LGVKaMlxhl67UOGTG2A zgzzG|O1udEutDPM77aCme^gU^nWJABq>9mDlcex2H$iPtp8`E=uF^m=ZFnv(hi_-W zG+3lRyB8SFAH6m+`T`Uiw3#57TJOv#*Iu_6MrSRW>#jVK+B|Dh+SzllwM#C6hn8fG z%EKsvSGoo4^iy`|x0 zoU`=PGiBHWh-21(VYzm>lRG&mtx3gxg7(dqez9NCox(Lit0}6Ovdd5|;sNMr|R-+wgf+DOd{b6}z3BaLDAyj1crtn~TDg@acMO zS)pgHP0w<>adGmQf1ETKa^jHLBn8I5KR9eOY47@BFkGDfOccT-DJ5EITsyf_7FTT~ za3wE*54uhm`7bA5OBV;NA)JzjT5Otr)p{=O8$5hQ@*)u4cJJQml&YFFL%;PTpf0&@#-~FLs1ojFNLhEJHM1y+ z>x1CXT4)$I{ku+@uV8;+m6eA%v|47q=>MQ5V5?$qfB*Zx(!Q!DzKgcLyz<_}yJ>}c z?)}$tG$pk7tX0HZ#q`wz#^SBDHx@KK%|J`ivj)uZpTe6ub?sg}7+BhwNEG*ZUDa86 zCyikL*4$j1vbA2)Z8;Bx+NJj42J@;sJHDp{{?>hLvsFI8R9Xo0-yTYu5Jb+L@u+m7 zPEyNNtGF)aDsK-yps{^+7LzbIh4oE7X9ETPM}$q-)+Bv;;t(4UI}L&>uOrt*$t2ut ze$sh>SepzRd54=$zy(Fg*$mlt`u#}W2eQpM_)o?gJB$}0#aUSIJtnRaeeG>v8hJ4H zIu>@z;uX@x|DW7UB`rn|eTiM~cCwU$2^DiVjeuEXoawRn4Uj*&pKjUF4#STKBs5ZMkFW?UF5$}Lc3u6H#pL%6!#Lk8yw4T?>@m1}p z>b9%Glo`t0M?l>%b~q{j2JjV6br~ASDvb=9S5O;i)b@chl)H?Vbv6i+Xc3}4E9ov( z5@d2*uk90xE~C=_)z?b|4T@wn5t&ftoDGwkitE&qso@tC^oa^{}+ zB0(yhEaF7=6)`|)5JsvyphCZ|uz@T59H0i>g=Tz#B^9WhK1DfOJ7M^fr2Zouz(4G|tbBPs^Y&E|RtF3}#%nxrvjn zRU^v+9wPD5Tm0JSEwQtVp;6+Wg?Hbt3H;!@c4k=#mJoQqu7$LhVpjsIm-Q4Qc z!ph#a$@^n7*;@GkxCbGSN5d<%j7;paPLV?h+H!p$4PFIgSZV?CrTxcoOoiTj#TLWS zR~CQ!t0dQ3HFn;CY@2U}W6EQLQjQgv{@-;ahfIG^#oQ)mRzd#-U5Rr5IY7q0aU&Nl zTL7Ss6gkg##5I>Xf2dcFm+B-uN#B@_VGPMtG|u=sDfee~t;{Hv@-KN3Qv!c?%T%Qi z2R$|l&Bi#3VT=g&>XW)yR(HhMI4Bdrx^!6uR)<1uBiMC_$E`gSdZG|ayfL^eO8pgq zH*1jEDfDrtBR*@4Midw(1L~W8ldu`k+&S}WbgkL_JrvZxW!FQ$ZZ3&I={gwH0=`p& zq8KCLX;vog_>%*@=$BU4HK52#9*UM4wn+MyuiFw{kX_RRdzu{~2%yzn*rU_F%vp(* zp?*2guM2Ue6!vCOvtf&cavFkRqDTzv)9|m9czP8&iYM8sST(7Y_WJ^gu>m5%7O(*XPgNb9_jy#ZiZy zat=0qqOA4L(=nFT3>QWu9NnY+H2Xo-1&@U1mfzUddbij*#XC+gfa& z&p^vu0l-5&kG>rN;18!>-&eY7lvf7+F9`H=-R!0&V`qcD#;}92tW1@vEld~; z0y0EZ^h{9>N_&sphtbo)A(^R#xvSTj8SoZ3uT(Gu93Jj8N8@&rd}pjqEv>*bEeeYA zOi;M@YUxlT(Wf+L=zy5oW`;{Pm`Wo_xcTzKv8ydLQkmyUe&ukfE>w&T?Aio>C{bCJ-lP=yK4KnV;cgkmJKr1A zDUy(l;%aP7y`l+0oL@esD%%^b13z}G*u$!ikvq(lR0v*MUR81C{coz8< za^=?;4D3scLqgCva0Hn`K|2b>+}xm3Gz%I(Dv=%zyPWDffP0)eLYoslB%_&cKwXIX zp+1;Pj~!xyh<>lzPw5U6bXJmZT8iS)rTkpS7}R;aLUnmrUf^{ePz<$oULv1LFkFj$ z;nj_-8Kjw-bL8|=S&m?PNS&9IHx3b^(S3(rnVwPxx9h?iUN` z_|zvxqAy472Io3qirt4l6b+0ziL@xED^cC#HO=uX`o+*41zXcbRex9_p^3&y<7ORz zQt?#nF__vXvrxpiO2Nf_Y-3+({$)}m0fv`UV2OzEWr-2o2PXt%KanVJ>RiCv_k%{P ztfhzd;fhyX#G`dE5n7J()%fA$DI#a$RJUI5Y8*Mt#&yFo00p>;p|k)JKA`<(`af#` zRRkBwnnz7cDL$y+?`cgbM?XTzi8V(@L|?_flQ%&yGm@+)A&zDT*hJ6It{_vj5Hk4> zA?z!$QZ({rc zl36n$<{z-4XnBs@z4j|4S(~fTk!7Y1ALD?bpmqntv-Foq4@F$;#8o;jreD!>cH7?5 z0|r+5W1z(32QgK#+P?>1sTk`7aB7ks_E9@WHcP3}$f79h@q-kHdE0?y&h|!~<=N-* zK30=O($jQ!FIH_tr!w_m7 z#ZHTmWk&dfe}n_%FpA0NS7V^m)CU#ix*~&KQqm9imDN1UhK(W`s1Pt@aADdRPyYec zX?k^Snt? zk$m!pj`>7(T-tF58}L%mQb4u0@p?V1fg!x?9C-H=w zV?H(oeP}=>2VTbemYx^nTNX)=) z(VW51P*iDY%IS!9)MCF%2XzGS)@>n|v$eZ8Y^GZ0*-R<|VIUI@yD1;N6HIsI1PDcZNmX&664_%ugh4j;A3(zikc<412B) zOs00lgEIS2fERa^G?(2eihJWPw9_n;pbZwQ{^ataD~ zxbD~=P|{LyJemca7k|LcUk1avSj*HEopYwLP1UEbz>cW_BB8pa9%ZZB}%81Tf5T=d`94S!+OWl5oEJVQL+8 zgHMBl7{PBhU~6R5_{Y^fmY~=GU0D|YsCCx+PrpiAw{t&+tY~h<-XT1Tau-4I_&S-o zUJCQ*AZUvI&ss7S z;{!g}ge#pk;BGexWU3g`lg}=YY8EuWsXpOiQK+c`wbTPJQhfhN-Paydva=F-Z_< z!q`XVASE%2Vb;jUkJIC8eIbp75Aq}Ok|0`nke!>=(-g&oTCSr_3M+eOkao8rdYmKq zB-B0j=@PueK?$-RZw9tWYaG*65z+DGIm%8GIHddVGFk}sZ4l0Fds0wrrvA%R_(^EW zG7i{AbwDUdxx7`A&wBzk+?JVA-bs3=6W<(TR9ub^Zyv!ArN;}>(vDZt9S#iGI|jJx zK7UJI9aon3{N2CzCl!GI{AcK?BUpXRU^J_|!6DArGZa`Rx z@GXHut_+4m3b7km$g~Ysnq1Z<*_D_3r13HOj~5-Y@Z{s{=bBhcA+K7!WGZJz2b>U{V&F z0$`u*GoYkE(z_tod_#@hCfQ#(2c>#2tRUvXGe<^<10EMa13l{zdcKaY?&$jcuLX33 zjo{oOx%2NA%RXBmB^Jxj7W$t&6_y$PCM|cthmV`UA_=%};!-6ICbZOvUoa##lTt(IaG z`~Lgn70pb4Gai`{Q3zCS54lZk3o=~cLsITKWME0`4>>B6gWDLmcLxKyAgAh)2}p5x z0(ZSKZ|vz(-`IE2bdlKLLAhZBK+jagA^Ru^5>r}IFR2TFdI=^vG`eX-L>!{Y6yA1+ z|M2IUbm>9ZDKXUHg;SL8*LsEZ?TRJ&-{~JaWG~UknetCP{jq{HczPR;M%3E-xiGEM z7M_wf?>HbiVYj4~gmk+qjEnb+%nsYess?fi!q&)oeN{n$7LoPOtJ!Y985>|{ETW8K zGjuL$k;vNtydYHI)&Qh$7g2fajgDiQsTSgrnYgP$vI|oQ(N+tH%LFmBTk!l({_P*< z>_T6Jp9QS1TrHZ*up6gHp4PgKZ8@Sw(R%E-T)v$ z{d%SezfQy+?Uhj4eB_q)Hj~?EZ^C}u!UB$L(;TTfEY4PC&QptxplalB0E>iO z;8u1p0WX74sS8EW^@?=#(Wb@^5`8pL1D?5HuOV>u)h~yXOs!=$_sx`WZg3yDjh0u1 z8yuNb0YL5oxsjnUpnK*?Ep}-NFR}z)GS{55S{SHRtcQW8D@0t;Wnd+U3XVsWC+ILN zS`D51eXax>zDOpjcRp;G_TEJ~y&~;DEc-DPP)+*`>bB@NK&0&^!CY}yenA-tZ+5H` znCb~nA!R@2#vIOqF8f`<2C@wA{?78N!Q~<(3Y##Ob)#_N=4VTerh*pwr!g?P>;mg<-62iU}AEJo94n&SiW}!l7AC*zo!U&>3lWg6M;iYrPEWPE6oC z$r}&8t4V2$EEg!hRiA;sB4dQ2#5ACC6mtQT^2#@RaZs|9KN6Mu;G0(MKDyJ5ic&TW zJ`&E@c>P{ui)(>Lh$H(3N~!vImbcX+4_-VAp^pRxwUhgJwYmjNf^bHXDO^BPi=ek2 z_<0KxkJ9q9$1A5^$=%=-jszNkt_r`k*7MH@wxlA7iZS-ma~==I%l_ti`KyMGd(%%T zQ9j&&?m-(;nU>#J!3M;I!6K_i(gnXYR|s`Ze#KLdgx8bqL2@=3f;uJnBz8983VDS- zq=wpX*P}E1n7a^-my5&&&5>5&t~pb; zKC;KN;04X-6V!V(*ntwprlw2n^691)-oeRnPWPcT4~3tq6)G>wxJ$Q4N~65zq$uqig%iYF>fTrc@LM zv!hyaSQy7$y6H{&x(TbP!z#A%W0w&69%zK&C=P<*53*$zeqzVq%Dc~jjB^Vavxqmb zMx3EBxX4Ki^7I$L%6zpQs=b%$aES$d7^%(a`W4!3w%wF4KRf?}^b3I!SKYBIdn}x1 zdEY$b9#-A%d};#ykU)+}RyhLu3LkIZVo^Fai*!yN=CFR8Cs9UyU&go4pmB~hDaX?< zSP9aY@*{&ThnLCb00gt8w{>GYC?C> zS-yV_s<DtlCY3LAjz2p1#G|m^dQ6TD-1!`Y4A2lep$_hm1 zYxLB+o$2HM+E88eJ`TbnL}M&%IkKz3EhOl-=h z@G5W`+*uY>sqwu;?(ck5@f$#E%m3W!wfzJrYHsm*PSo+bdZt!0qpW6WHk@+ihz!uz zFs&a9_EDjqQv%9#is8yN7%`-Ii0|y zwFdH9do3;>NmjT+JgVekPY2L$3FC!`ajhFcY zkhc@B1E4;&AZiRkmqRzCT19~Z4iz8GM5ubno;f?k1Sh=a_3z~_0DgFIfY}O%3(*o( znYPGyC~VTghPqclOzU3S=@|FA<~20y450x$4OIcNQ{JzU_GT|wo|@$R2-g9^!x^l4KtuzxOY@TwTE6YV`1z%K9s^Fi(&;RjcSTcZR!r>>7?)M z9rA|s1Ykq2cx!M=2M1E>0Eu$S3)CKKy?9qWR#Iujf$d}mi~`T5%LEu_1LS42g{Awv zn?EYIzVYoP8gVelrPZ&s#t@mdS3iCy{c3&&|5QM3m62~j{4ZVY8dOV}P$Kb-;(*s+ z%0_&IA}I*qLG76&0j%L#fU6AZ<5asVq2wt&yG+z1tNny%Rc=8TB)E}v$7#x4UUv)* z9N@?QRRMr=JxdNvvJkxhW=KhwTbuurE61u&EGt|)oFRQkU=uZOj-kjUYw}Pb8?6m& zuQ*q#rLizu3MP?M;E{l)tcDrV^77lS4)(dh;pB^?Us`v9_rbKS15~{i@e#P|e_-FbyeU6ddJTK%{jzBB4`k@E!Dzbw2owZ$f>; zhvp+Q6V;VwE|lzr%I)|G2 zlXs*{l(V!pJ)4ZMAuTi4-$f=~2a(6M%j~hIuk``H)WRX$Q^j{~ z*57Vc=C2wyxzN1mrrPL#vRkpLXcd^ZAZUamM$WBngaxltH?UThy5fd(Dr(Sxx+Ru6 zyLYwvz|XRc!i*-D(b6nov65r)Ze6!7!(i3cnt6#$@B82HGZx#hy^v@@(p~JNHs-v4d_CBcXrb*VIJ7w_QpeK!%|WP~ zCu)rLr-SFCjLvR*S^8uaY=wUY-9-qIW?o&uYSY1A|NOXBI z`p0`(lZ|f12nJ#O1CbnL9FqJ^1<2>X2*}6~QezC0()zK=MQ&if2jH+%=5oH`Kh@6` zKAA^Rp^r_C!!wJiZ*P47HO-}2D5<5O$D9lg{qkaH&IWk_J%B}ileJw1UCXfVr|l&! z^DoGC6r3aCk>lGOvCdrW+DPWVutQwRY$$Avy^Tekzv2PcwpW?7T;57%=8-HP|1;W- zm%4Ky%d)2eNPPAyec|Cj{0jsqX`ZBPFXLEBjXok=N=N0(DTxt@;cXsRN69t!zgr>X zDuD2e@lHEZTY#5uQH({#b@<~BK2M0aN}^J`di3cg-=}~Sav7D4;ov!3(fZF!2Fdm4 z*MrL+$~J_O!d#6&AklVzHI5Ct#UT9ZBYqT%s?4dA=;#WQxlc`Da zn_KKo{eTY`@sL@hdT}`D9Husg6Yas?bU8`j(6GiE0q0+eUZd%!rkr+pskUzIeXq(p zXyHq>k5+r~c|p~iZ-^R|$;TA6Y%4!!KLb^;!F0{*|B0kDivpY4P88l9Z=!khAk<$+ zMr6GIR?t)hd6k4AoKhpJ&s2u%@oyjZg2XIb#w=5PDi0qk0^V9fT;}G>(PwrEuP&#` zY3JYx22~EDZU82Cw*2^^fgExZ6u`gUT}Yu1?UivJHL@edU{Ro232?iVKvQPp)oq;g zUDR+3F<6>}&Q{}-CU7XZc9VlG9h1VbmqDU3ixkv%#U3WojplRumN7qN5BprQ&H7Mp z6|`w|Rei;X(AIiW3ZS=C%NIY?Btup7ie2){`DUG>m`Tk34A*elyas9v6V=jgDtrelj zDMJWluutzi>6+_W7fAZrI9}RjM;QU7SN&I2LWdf;OFOljX`;tEZi2b@=z^uufZ6zM z=b>iM*GShc(H=6Pc?&K{vqqxRNmJ#YPoZf*@=EzfKNI;o)gI4&0XmDz+J1#CpV`*@UP{spC<wZbh^@cR8?#X8nuK$m*6OeiUD{R3Vt-5EmlPBzJ? z&_bvRj@9kbaT4lb#Xew}R65LoEH`+YiE@75xw}cF$F#=$6=ZCHJi(b>mpG?uN>QBz zK1!60ckFcnF>1waT2|$vB14vIWVr}r<++VTy3#{iq`1*;TSVZ3$$l5Edd|L&a4GNI zc{e}U|M9TS>NYv}Ox`UxCffU)_$M&CQ|@I^r+LI5zw5Fy=>U?;F@rSFIk?p3i4qOM z^!McVu^f|Tvc7bzaz2JFWrepL9ePM9f=Ef(_O@O^et|BIOFukfHPvzWZJgj8()k7L zME)Lq9L!r{C&T0F zpUNv(y~olub+^@yMCNOI!Vj0pLuWeAue-t?%LjC5MG%<*$$`)IPg3=j-;GVMMwmcL z2b%H}@(FB`UWEU>Q9CXFkwr8Sw&imAnHP*1=|@+e+*SpKApyRI9<+y=&K>!153y9~ zn=fvkn+NkmCi_SnN0*=k1t=c+BDp-{NoWej4HVIedX zw1xzO+M;R-ZHlA;(~+63Q{cSZnke~L=|~eW7Z_fsE?Xq^q90>$*qTNOvpt%rN3fqg zvCIXl$nIe}reHBBM1SCNTZJa%kRHDJZF>}3&~)ft!yz3ifsy?+BpYB-4;v6=zP&vLjgi^zqALz6@O0^w5!A1K1Kxa}v z-nM^2abH$5HH&6MT#{y!cmiAO6?LHPbcK$`sJ=B}s4;2+i^6Jvv-pSr!%+~a#EVG` z#pM~AuV^gxI{9UFa&Y^{6~gr2Ajqi*P!}(B*VL~}$jm$z7YI%z)4{#_6 zM$D0IIze!pZM88D^T0J>X?H0d^4KPaes5G6Yiy8ngG1%nE>;e>7Vi zJJ^mkqPW*>WiLlytcExNgBd8Y0Qu0GiHq9|it!GylE_#(ymP*_VM;^jz^KjN?gl3M zYq_#Jw19Gc3pQOSf?r%PzQz z1Cb7eZ){91a1dT#(qGy1RL85XlQYoHLHiIe(Wn}3-dpsJ;R7EiZ>Mg)b~tW>B(BN;+}AZzoaRbUu;KR4|}-HbEV-HM61y%OuwXmh8MO zDkMeTT(SG*iU%;{M%srNp84)=Fkp6r(}Jp0E`<3&nB9+c{^>%4bz$hyNmb6{m;Nba z2026S>n>)WF1zmyT#j;W5dSm)sr^!Lf^$=)_7l{*hB^qLu>k56&I3oRj!7-ctW*#& zsQsrS=Nel!<9d*LU*uOesQ_RayrfTqeV9RkCj?-QS_*YSLE;&l?&M*GK#(nSwfh_!2APD*EjcJ1D`Z7e_+3HfrcUK=f(EB z>b*Hc!!b1cP>ekA?q$!9r(wXsWoOZA{n~CHGwA=ve=~725ZrH_P5WY=*@VERP>>u2ZgapMenCFaSbkh6pCUqnyYrza#KpTCe~xw;SK zyQMS#gF{1p969DMUwCN$zfP2j9do5>X;`H5rNQ1_`3r^XI?n(Wo4k7Fd9@LK8}m7k zC52iP{ii9YZ4a1GpWYU=tKP`cTm)y%olKnWoDm0&PD@>R&f;v15XMy!n_WX)A8}4e zVr8@O7xY#9|G8=3HCHy z)Qf23$F2)T)1@i7XbRM4+A9EVW{~4ATu(EDLlEa)I|?!$qSqkD*z~z1Kxzp|T;c?R z-9AdGdgwk@jtXx2?V6SC)gV$l5slYH)`+6Xd7ufHAG5ZTr-%Z+iQ#QqUQxn zQ!Na(2@hOd$V2^V7zZW%S8JO+^$rN~WhM1uJ;~*#(SW-@gqUv2Lke)ZqDsA&da)gD z(tWT)J74zgy}0lJNlmy_QX{miy!(>EE1$X0cM&8R_nbdG!Ia!h0E>RsEe*A_G$g?D z-$X-#)^sRUPpjBAw_HVq(Mh}v8&S%WA@5}}Z`}#~sXQeKCZ-_nLJ5s2s;=uNN{@F! zG(k8dau_MKAp@#q1*AjRn<+cdzqDDMNW>&1D$)G>05hTo#b2g0u?bVPe~9bZ;qFRE z8FUU)ZGO~rZE-gt9hsEY-{GN70EK^(482QH4iY>eje9&n_49td2%j?b^bDGWUoA|^ zJj2D=hkxaDDiIvy-gnkP(D_>+=hDt_c1o8t7kV@lUqmLb_@b%#@_6cp1GW3PrfYRE z3X~dGAp*+8m|f>Z1{*e{@3*ZY@EWJ9I%l+d6X=IHIbXM;>i* z@h7vt^;Qz1F)GIv;ZdyLDr+DK&_>_ldQ{73n=u0dzHMaZ1%weC)AxSxcpMza_&}3*F&QQ!CulFLq1~2f6$_S?{0k;Bn@@~rX4OHPffA> zcmv((AP(rz+Bp#cO3=Iw@TmHA19tyMEzg8Y`f|wp>x`7%2N2Wcd6^2g^Xb<@MJ8c% z#bgtblz}|h+#{CFi+clpQI7^6Lvob)6AI=43Hqu^vDbFYdMrul#I;TXh=YL`XJTEe zLm3g~zv%MDu#C?}`P_~10hQlu9ur0d&3@sm`G|2aO{}v<`n=R?1^5_8UlB0+Nd#V}0xxfhyy>H6+n>8x7+43;2T4EwW{yxE{BuIGVyeCAK zQ~`3x_;!vS^^pp~p`K~^^dFS)6$s${*1X@G=|#<1+j! zyjkeD{euEW`$IL`oci&2_VRWkN4V6%CMcNmRu5?#K>7sv^pm02X-)-|LxNAB=x^y# z0425geF0oNdL~&3$Lil#2PsQ;Enw512=A?N;)p`=9WcD}Q2xUnQd!SN{6#kzA$7O- zg-v^{D!b8t(V}4bmUpsN*bp(VWP2plmI5hnWlEH{L}nGZXFbGnaYScd`+E1Y(hjXu zOSsuc9?V!GmF;;-_(ar;6_xSzDf)_<*%F%NRYSgN#TBV?oiK*Q9W1_PtbJy&ZuPI|~Lrz`>WtGJLhP zz3=WfIgzMuwae6)`&*^&Pk2OpqdhJL;^{*1G~@WnUJdQYZc#FbPg8V3I=CZKcF2w3 zE58HYDo*??waIKG{qf?2G+@iQ@6G>qm3b+Hj1ttV_N5xiQK#+rph$Y4>!}9J(*%Xc z?RObV^Pro$A$vE3gNan{N9;_qI7z9R000S9u(z$?x<5ISAERbeHIXK_(tVkhbHArz|4<}a%> z4+Y{XD5bZI5-qc+I|+@zVj%Y`1CD=vioe?DA=(?UTdf;BO-&ExfSd)v2`g}B2NL^D zsh@n0K5V;=F50=onJ(){W!xmN&P1}( zOJT7mZt}^*POTT>ubu}KD+tb2Vv@RXArgpkGK2NDWQ^RGK3#^qm`m>6 zKjtc=EuJGIl@~-?B~!)qKIttEDP!`2_wBb0;k|sVvB-rSZmU39mi4P3>To^TGn0wc zd*?*}<%pqD{fua89&beUN97DlJuQmijr$Tav+%&6c}Kr4Ehv6-3^B@OpK>I$6>J)a zr=aVQ^+dx*-0V(Y)lLn-38bHpn>GRAg){iKjAz2<)!!puMBT#7t3U79H6zC(79|*;awRQ~oR6V;RX5L0MgI^#qIHi^PG=V6mS2&tIc(=@;dr z_e3O50|{&=U_%OttkfdeoqU1q(EBlD(it{ZxZnv9@e$f>3(n^R(fgpEgQ@aCcE~_0 z#UL~Zm{x`CITRA#YN$-SPEP%_;d~hkcNe3~Y}JM>T0FfzK-2t3!r!(0X0^DPQr^sj zr9ge9wM_R1%M5+XW|j|HA(c~)x8^>$&zm_%M2v+wU1#RC`$btX=og&?aspD+7Y*d* z1rr~Cqx?Drbip|g6FYx{Bn^+$4Q=?t<3^7t4Ylmqu8C9Jtkq(zX#8}oGzf+hA>x!F z7wAhi8M(+JPB@PzUqqNuf_nK*9W!O-p_?zeEAEE@&qF_+Y}S%ICyy~{`&99xjSO2V zR5u5a7;FRCQ+>9+n6ASEZ5FP!+s0)eOY)#h7c@!CwjBD> zkwpAI}=w)hK>4{n;1c$tBvo_K(0Fzp!u+tNv;+b_-aX znS3C*6O<4`XVi|ZEe`m+<_e@vqUOp7*uwZ`>=7HSnP}+NNgi`xem^l>ihxp1A(Cxo zO}@zYnl{prT!LqL5FLrcWtJzh*8e$P;0< zMwzm<5w(<%h5g#(IOtpx9IN{HpN8c}9(Oq?qZ0%y2bfWu?*t=T?b86@k_!&g>gsX!o_L*jHeELhgM-dKo@d6CiI9hoTNNdQq}hq5eKI+n6V zscl^yJSbWwa}C^}Cwdhip2g~h_sD6yG0JI{T3unZ{hJdTmHR2CPJm4zap|N-PfB!( zL9fd55?XdEhKm3sv9u5%wext$-6P>$&PC_G)@zpH;&7qKshd{oDG5vZI^R$vKK(t; z2HC%2XOEtrLsna+Rcqx~uX6vh-He@>=d&-GwyaQ5{P%^B{H=SqYa3dX^Or=Q55SZ^ z^&NZ5|3TsV5FLMk&o48F_9siB)6RozBdCIW zSJi?4z;E1yKmZ-rzsu+~RS&J@ueB%4X!+MTfJ6d&zZf50_^GuzC^5pZV$lJoUM zkBuoU6wU>ySBL+OKUuwrvON`}m=H){jJ4m3)q+;7+=ymX%b28?2)!~RUGADXd=QYn zlMK}HH!dwRDqMOSQD_6%32m?5z`69O7AVhU^k0;6eXgD7`(SU!=MW^Wo+rr;Z%r$* ztBhNjR7A5X!dXvWF=oFcn6NW=6A7VIhx5b@;~M;hDSwQjS`7Z}mS4qt9_;sd+yPaa zUR!B%m)6n+ArSO#%MgtWhT>3J4SSMSi6J82$qpi6>4xlUFnyPH`i9r(d@ts)DUBYi z$dThbtFrE=0RkeUbl1E`4xYjEjaS9{U$(5a>e7-PI!Yn=XgWK0Y-F<*%QIv{+bk$d z(LoL*Y}S!{%0Y@yjI*L=OQu+I9Y~XL8YAS}T_?-9`K-#)AI^{BLrVCRo6M?{FfJGa zM_-6yuL`H9ytZk>c4}x5R{EG%(etr8odEYqkWU#<=&fk8dVx#4%EI3(pD!B}iPLdq zdEG4lx9n;JX%&geT08$;ASU{FS4(Q{uJ^7fRIcU@LA>PQ)=Fq!DkVVwb)||`0Tp-I zOJk&wUrfOISQ?O7MqPF)zqE0gj~p4;iT6R*AfMIQl0h|TLZbg}XtkC~Y$Jw+w`q8f zQTD^34S=v4a5W5@!(GOg?c@0hxti+1@}jQ}dW>+YS<`e@AE>PS*H$(%m-4Xnzl>gI zH{CZnAn#NWG7D(k#NMoO6*w1F_zW8s1Xh*KT0y}%k%yvLAECKUlx3_Sj_6-cdOTyx zR=>D9Skjw?B}q{rY%i`;A(i`nfNlxW;?mJS3yv zE^TaZ`p6P`l%H{y#Td)7Q=Ue!+$d{YS*?7>+CCJ=WOq7KIXCqo+V1KCWIJr87&IjL z5u+Rj$6(NMc(XKg1Vp)*aZNDd!PwquN1VxIVTn3sQ@4+#0PVpVssC3`D z-rGfdmTO@r>)T1+ZjFTX5WUUbFgk+uf*G3}mzUnbypVQ5-4m%C>K=(w2x9Wto=SbS!biKvG3}kMv`ENc&-=+8~4Emg)1z_etpg1`z@ClXGtaXclE}EWdBPJzBJHgzxT-@A=a1;qR>-`wP2a3 zLXi1||6+?~5DD|fIpg`>EyN8P7(4}%Ww9QazwmNyI*^NT73Et+sw*{6155D)zy}7_=Tp`{?h~FTA;#!w>z`I3B-7{yQwu}Douk`URQ144!vT17 zaDMQ!a>04nC-e4cuBU1XD$oAotyFgdn@U3uORzsavE)AEAib$tvJm-AK z-xR%DvocVa*61!iWz*0^C>6xmpMLsdD!qpka{|#cF#xjC!=-bMMu4E**QSkoXleL& z)hMSkO>hk1;O)RV)#z6Gn2=r*x3^D6(YM*qX8Kj~Qy}*P)pXJYuRNn8M7lp-xm-W` z)PW23eRrv`2BwNsw3PQFqV7mRxEz=gbWZfD){9(r$SxH9vr_XZr?g(a>>_Z_TlB-H zsJPCQl)HL0Q_exfaZ-E6wGz@wV=O1grsl)A(^8NMZ!6k>@P;g1*T-<%M4W?Dl8gJ% zDSh6`ix7Qm04K2-lCZ&3URHr~eNXLW18t;|_VC+&%2?{3fbV75s-{F{(lwhUD0KD7 zd@c=zMfwld2R9VkbaDck%MqS6q&AMX2q}Iz2tqcTtZ+ecFG!Loej%P8DKT1`_BlX> z3;|to@fqJe>x2>|Jk^!B1lw)kFec26RB-j7yidTGkYi@D8B`V{F_n=6@S`7G4vhj& zg$zBLL}#>w)I-A4JDn+F?83gBWk0K}M`&vsOJEz7QK0`ddzPN*1AdKR>htmyXXC>*In+6>2=KHL@4qJycH2C>RI-|4_vC?(irs~FJP$s9{r*I&@-19HwFBa0H@5OtoyN5J~5__|*%2a0!z#t5UvEFDZzJPsXj zyM^aK;4$I@OoQ0OBK=qoDQ!cd(@~T}6$V?Jz&$!}1JCwod_k zxWW(UeQxWku@`7_YTHbg zaOv;O5coX$EKQyau+7J#6Pr$vlT3jjHhXgy*{}&xsPsSpSs;Ekblp!pbqOb#B1Tmn zXejsV0c1d5E18ex_FZ7;*|43JQT;KL` z!Vj5{Bc+em_Z(hM`O9eKc=?y>;8x>|!HDX6AckKnHp$5so*WNW*`|Tm0!2GXR4q}` z2oms=*QhT-5aK2TJI?AEd*_5|!~lq`X@f%;8zKZVw^O)UgUB?iOO9Dv@iim;J#jypg%2sHHO{if@w& z&vXq}^JEGtqs?@Sn7D3h?!uxx{huMkk92+@T)kQ^1@oB#nh*eILr6I8rL%8oh~M{ z+?V>@aK*z6y<(-h)mE96C5CmgpPd&=roDeIArXkVMK39B6DicYH3P;&mwU1|+>>EF zOQ7F;QEJNBugiMJjsXim2pkl&>-N+9YtuN!0Nd*V=dc_kSiVP z)UB{}&990NJ(@!KAFw#0G#XSiDOpe5)2QL*!n3>4ywTVN$bjwrCYPNTUj{!fQki|1A_cl$pf=X ze}TQvl&L8>^ZEN*)-zL)t=a;Po?Y0Qp*L65tIq=dZW6{DX6kPtg*~BJ8h2+&)fA(n zWx+-d`=g~q69U)-6em$No49;-Z6=x?uydr}Vusx!c~-M1+T$-&(Bc-Nm6#stNtOB< zXP>kn5g;jj1Vb~7%{j16I9A>zUBoUuy9~7u#h@rnE)We3iyAi8gN;xJm%BwJ%7?Yg ztCb=Y~j)7NCf?`5?v)PjnmKPF%+H_TM6N@~E)9Ji|hkl*rhF-w4Ox=VKUP#E< z(tl3@gAh;W)%ndvr4DKHqVGNMRlfkY_O_DtQNO~_QAqMyX&~5<)F}CTDy67q zKD3*|N7rFzn{74~Z1sXLL({*^tG%Rj;;;hZei?xmh>`{CIakOSblFDxz9P6f1fgV1qZ7VCxstrgi>I^j;c5LDh)B)R zDebIDz-5~~wm#dTR1M1`Iv}Bxeq>+a>0}tygvJ##aIz`(HE6jHtY+X*2OY3Rc007) zVrKqw1U)(PceGfJvhTxk27o0QGNER3{n#Re$yz0yCe_9xst~fFk2W6$dq6)C{#AeAU-h3Ij<8gHB&e`v zNdI%tEk{{)g!R2&**qe>$h8(;qblL|4k+v?xz%rNzw!tmqenO0eC5A*9LP>|aHx~t zvUpLV!r>PPNPOH`B4oD+y6IU}^KK@R&1ZFMW4r)lYu z{?Ayr8yb5YMb;*mEjDhm8EE~L+XZ1!tk_N(|5~C}iZ_*O4qJ&?(4>`F+k^7hnz|{o z3e2Sr8l~UGY55fpvG%LTIRr*+z(v+!Kv71Fq=kv;rHyU#t&b*kt6$ngEUKWA*xqp~ z$wKp+2b}uHKQEtUs)gn{Ej|Q^Pv;+uQ8TTa|IsEh0q=2Rm~4y2{F+zNE9;t^mGkhF`Ax zM^>n3%I#MGYq@goSFmCr(XS;7Xr4GXKop+MG4BS9-t+@DXCv}oi&cJV+;8hl!i&TuvuqYUuKG5s(k|t1{ zW0yx1OANC$sfHZ@)tzX$b@e26j+B0S4M+>LdySuu%_I*iamCH_j5(?~s~ggU^(xB( zeyw7eCIg9+{{;|KmRMflhP+)TTguZV6%d8i$lm&(td__2`|MmNQjm=^#)9YQ9&@0eyktr(lGdWqYAHm0BI#GiX2A;!cAC0S&LGW9dU?&YTf&7?HOxgI!s z(ZNSFKr0CfLn~8zJ5dB!-?B|QPR@5+g4j%>hUL6etNRe$2^sN1JDJ{TR3kV(b!y1D z?Cr93o^gakqjB`X8byWBVkAhGzIZm#skXb;=)t&+T;#-*eZ{B-y2v6Y5?x(A-e**d9&*> zGvlqQTDBjhC#P+^7d9tGb&LY_lnu*R!MsZONdOR16XM1Li7z1%z$y|}bwGVoM%fyE zUKuC6gJ|I$y(uqH#)&232l1L58t8==2df0HF2C?=G2=K}8y4w~Zmls#hl>nrJ$GJR zrb!i9wTt+msWHKf36Oqd@eK#6k#=@@+XFaFS?s(K!vuO8Y$ii|KkVmXnRoZx`+l=u z4r?KQ*3JmB=*H$GA35ZCtUz{eg@ySCfKG4(PSZ#dWP{WCn4v`N23?PZq29*2uE@1p z8bD)5;qcENwh+#b%&G!2s-A4_Q6=B9V0_vEN!3M|*;x`t?X-x%I+UHr7YX)cM{a8^ ztgQ=8j_CT24i!D}|6nQ6y(DP?Rs;_K_o61zAUL^rpa#oc$fa0MQu&l?!lrtCDJ->R@-2GP$-=5Ac#vSkS)>XmgMTPF+y0M}fIz8Z`rG zxDqe?sn4ZKI-}vq_T06xrqPO+pgHs)U`nGvq&M$NPQYkAp1b70(ciK*}}e`B6=;71uE&abxE zoAs+5&i6tk!=P3KC!SRT>lN+xZP}DA-{(XLO(WpD&deD|q;u6rF!j}}!uy_wnB%l) z!BDq-G-RzF<_N7o)@f~JzilFyj`g_2!+q?7VVH?OZ7M{BBQsqYWMSO*<}w7U`I@FX&yWPPgeXdQRgvwdqiJ|KC)fuOdHs1Y=JT8ghN%No z>?kLP7YUt=)HJbyo^gmoiegp<;J9BsQaTCh9ld4(Z$~RWQ5*~BZZSNJsL{4lv6{XOBOkssk60+-Dk6oy6 z&2kwmK1416tAUj?t@?zp-E##71U0HOYc%-ioX z32GM~%_&;ljOtNx;jbad`;4Cuf%rU{kwW0<0*N{$+jeP-Tpc%$Sj-a86v1nd6b?08 z|EvtVjN0~@b)jqa;B(EDYfNCpg!vRc{LeZ(S%ueti=?cZ5=xEQbyfd<4zECUwVPM~ z1h(n?_p8YCyVFR&3W#5Z2n1j3OOa&P$I%vcff%yGU5+PpmuZO#AsvzMl<=t~Q(QK$ zV#VRpvTEvBzrqB8n049pOWKsp;SEpO&I>Ez!t)cUCeZ3#yltBDKO{lR7)8~4+>7Xs z8w1@mK8W}_!q1f?#DYQ4U@e}DF8=5VZt1w!0!=TiSg80*yTJw>Sv6xLJCi1z>TuEb8AR2>i&*zM7`pNbn9lNwYIPwTXj|jtQIeo{OE9s+-7oaB!ZI5A8@=Oo9Vh zFjh5Q0TvF;{|=To<^c4PQHT49f$Yzpats=gkza1s)x=b6=Cv5GPzUpMiVc!^3SZ;+ZFho^$^~#pzzusZ2&R zs!@t2HofB4dXly-W(e>3f4eDFeo}WJCaU~be-cgP)&^SEKk~uaYLGsQucwWFJc{~m zniPG@4Y7YymgM=J)SJ#Ruv!t^wjVv&qu?`1{}B;Bjm)VU2&V~|gPf7EOJNi=y}ssu ze$pXWBZupkfaUU7VlBUkw1sKd^z^dd66axa^}2>5nwaS(3Z=WE7? ze3;8(u*aZ(j$z>3A1-Q;QP3uTrFgc>S^CW@g+l9F!u+Sbk}y{DTm0dCWYT5j1k3k} z-iVopkWM`{RF`h(7zE-3?VzgdtCE^yF4tdXtj;{!H~Bu z)LWV?E^=99>z@Gdm&yFHYULEe(kS3UF;eV;L2s(FO^tIlFowC(eqNJ9^ZI(6S6So_G zV0l);y>2U~BtRhhE9cbZ0IJyN z$N;K-o?16(em9Gdr}k&i&z;~##-~{5Lu*SN$8LC@s-8_oZ)h_$Nj~c&CVrN9D|yqf z^^#<3bvG3I*m z7fE7r753%S(T&bkiP~CbjXb(c_JT|+5t`}}EbLVJEZ9sZ`i&lg6rjI`C;gwj-F|?% zFbi}%rb%3_!rot4M8yS5+&x6RPHOQvaizcy37cS|wV{eCa7J_?4$|@+GFtzcB{BcO zv5Z~_WBXl)MBep2&=cuFVlQXoM*Z|u1Znz#R`nEb#61MO>n{5Rp1EDa8V9|<|57Yu zVVSp+VVleCl0!LOR)Eq`Qrvysd>wUa?OvN|Ssa}_{_}Hb2taN`dl7dPjr?hYj zVYx;e=LcD9%ni?&?pddTM|Xs{hR{uyz|yjMX}eBfQxn-B?%|Rx01p(+?T7ecV+Jr>|zoFznjS^H3a7oWNdkBj{QZc1gC>O_|=WRxiW)uUXd>=ns3 zZl2_})zx;ap6)uhoCIyvxbUS8o4Als8;W+(x^$HEv6sCcgYC%SR#CUxd|*A8QO!xY zLOTx1#7d?qAA9_}qnXEjCqR!fU6^0a9y_m%=MSqwK5~a{ShgLflxhE5{*`mJNUbo0 zQNR!0nv7JK7JCCaMj!JVF#W}vy-xEGNB0L`tp!oOSe>8XPr4rpmg7Ng z>>XNqs$X9@Ka)3Ow#Q+>0~aH#e1lCuc`hr+MoC$mk#9!TOWDSf*h2Mor#YR$>EeL4 z#}L;Hs7XjHXiT*U=sW$D+t)d%59sjZRt?^@mjBR%4`r5?lClS#$xlG($L-v0s zi1wUwyY5@@N=H))QTxI2fCOZ%!Y_j4qE7r0xii0|y&VSxl{feAT%gm|yiZZWW#Ww* zuZs60?mA5l)wHE`+Bcl2smrTR4sj&3_I(4JyCkwdAg|w)Y$faV%(4Q1O45151s4vl zsPoTU=tY$8Aa*6N52HDD^m78(3&n|vp3JXfzixvSJ0KFspUGLLQR{B@TEGm8u@aTO z&nGduT7h{KKY+k<6BnodPvc(oJ2NTMcx5*$XpvuKo01l+tUE8} zUR@_i$TOY$<&gsD+i;B1?H|D3-_cPcruiF@h~bwlRjM{-2eK?zs$I#eH?}K_ISGgb zP^Nro@7TBSaZc*o5ufr_I+dFd0WS#Ka_No0QJJ|**EhnnQ zP4qXpK+{V%ODOENnGB%TWFPw1$b~#c|9f?u`Uk6TYAZVhSjh=N1nz2&SH9>$Y$^R3 z*z*fz1}Ua+Ji=GzV7AoZrre}G(~(ZLRZ-@AWs0wLuS>xgx1!zCI8}|irRjx)IEU2} zDy5~01X%_(yzcx;Dc|88R9DyRU(^7MPFA-#uq321fAEr>79Ou}7jh8pO`GUYCKdWw z_Yg1NG_*|Cdlu#u)uKf#TJ#60RGlolx@v7>@E1~*djMe;-dCAF@p8a3*OhVD+;TQA z@oTYJJQG_VoQ=y^k14^>1Oq`#+L9=pQNG(EmnxUQV(B|F64%Csp# zEw;S8nreZBpd8Cvd7(~@%2xi1EE=#c0DyNr(CV$}#Ka(lWUCJ9g&MAJu`oW> zN0lO6ye-OZ+p?|H!gU_&XRy|;#igV^)==yH?5G4QAF;O^4R&HD${^aYGx!%dHc_ho zzGS}kSw9kO?pcNPU^zV|22Wn<)UWav^C$_g0|mtn2Dt49-v2WyXZqjd1&;>bob#Fv zn(?vB(P_98qkdWne!{kS*5;LS#NwMaJaqf}~mH02vjcXg)m0QZjP(12_1q0z%~jK!)C z3TUqU&U=JcN8(a8a-{JVPk#ZHwqWD0f&4|dyPTByG>WT@t>g^)=>7Up<4#Kdm2Uju zSrdTH&-UMeE({6-@5M8#w6?!-kwBZY?^J#}Z!8uUl{)-e10>{kM zvwXF$#KLh;-f&kF0yU@hedGXY(~~db&u#d6BszeA zMS?Am)>vV~qEF}Xss9jbJ+B9(EbEg|vIns>`(H}jUE!wKC}eci{wWt{xXMtwP+u&Z zmO)d6MoTf6MAmC)1Bh&Y-setRYns-qE3~~|sbjBShcdZ-s}Nnc)}XsVGKY#vm^E6- z3*M(**Hir`+SR-c9{S-%s;K4zKab!o+l{$fWtY`Yp-JaABNv_nVuR#WGvVXJd`ufW z=gIR!zf}KXCYJ^@o!_7O|LYLGf)g{@T187?VD9{BnKvJzUFo?ZDmXn#9SUtXvsP8* zLH~Z*&I^WS%rC&hl1ZE)6m)Q+LLjBkg@vx(FLfpMIf&1wDrqHctY&fln4W;=m9~Of zVZZ|#QnRuSUZXCK$0O%K=kb2U>-tI5PmrHPXk79WH^x_jW z335V{5}hsIb5tn6g~kF1cO06k(tdf3C07zmlXfb@-(zNUd3AMuykRPml5{~f2zX_) zK#)byqQEx!GAFjMOy*ZR*R|}xCa*Ef)N9DOpFZu*Ct1a*AW=_~&ybhSC|iaq_A1}w zLEf7d)*B!@&0&^KW5aI~%ElvEp9$*X1l zOU=9GnI_YZ>wE|G4MrG;FA%ih-WGuSJ4Rzw5-bgCaC3!8y}oy!-g;rZLL<}k5s#2! z)zqE3QH!xAmRp%tCwm}A%dzFq3@_cG%$nXBJp9CU^lyUoB3(1THWo0f-Y6mLvk2D- z%%X@%4MV=D^N1)|@$Sn@3DEZ&WCX;WhL?fMKR^p7dPv$y-0SdLiFk6NIwt0I=6Pi* zf-OO-7z)((;=j#C`EH?#%mS{ z6`5<}APD7?Wm~ub4Q~(7g~|NqXpgb1y$5A;MYN-Swq+G)*lc$jwrywSNCRCAwm}uP zPad_y*9D;krYBq$Gs4A^HV+%@#phIVwHhBRQrX_e91tM^Z>@&&?Q{z-$15bYOx_Yk zi}9LC=ozv{Mf}7~d4Sd}ay}b95w;QH-tY|er>U0mjyyl}<|t>NqtDRQ3@@b4x@*}# z01WO%qUXc8@bp@O0)TY$5(279G}9`o-yIa~F&8TOD zfGtZ%s6AU8#*W&-%O+P)gnU&&E4vF2e96kR$a_1ogJjN0eY7#DAAgMZ5+^h{@QZPu zoiGn~+pXO-N@3azSdZRlrjvvz?X83a9?=kM>>?O%tS&zJJ)_*Cd+9%`j^q*0+$~9Y z{BM(wl3uacOz!d43+6T;w5XXOevE=wdrJDRVdg;v|KfE{^`gDz6-?}k!HSj0a5A>R zB~jB4P+i1yMeWmo0m5J8Z}%iFz^{ayLP9xO5q6r+8qYHGB-ha3Y!vE(LC6YiOanx`q6l-q-#yobZ1 zThpt~r#>s8MJ}=sRWa;#hMu+Qkw|wI6al^gE_OSLOJ~8FX4_>3cY>(V+3*wpaMr(# zqw(QxuOgRlFw#nm;EqQAIhtLE;6F{HdR99d7S%z>oK4o{{(UUVW757X+{Luoj zz@Z(nH*?E|4HH$wZ=M$@p%JVJv8n4ent+e?e#ii`p4_=7M-)KwxW1o`E?^%>B`eL6 z?u}rs8Y#NFIh4Xf1;&2&P%-YtEoW$FB43V=z)TdIw_MgPq^xX#Y(J4TH4dGV%n@YZ zR;~(4`V>kT-*BYtS};D49cIyFeT|vyOCgMq-i4*jK&@g2njAgbGU5PMb7_f%K?}u) z?ZICHYgvHfT8GI4YaoclpJHJ64RKlNQt~!XUOAu3CfEW!K2PtFr)Uz0i;*=b^Ay~u z@{tKJObZ#`l1-=07$+VW05y<*8imKMOgd!oS>C$_#XwNDSJ^|sdR3bn#t-6eVKs|R z_kp1=O`$ywITZ+uWNR?I=(0Qx_X|Le(Fnj~iwQMTjn#ncT&f0;M1U54+q3b`x85_v z|ADg}rIcQqpTxh?s4@b1MXrOYAC3T`X*$6h|4ouNW6?TeD{7pTU`VvAnan6c^*g{} zLN&uvgW0tKTwD@}6m<&7&$5P4t(d7a*6&eKp4RUPGdQff9K|OvV+5}XGp3?^#d_@T zQoKnL9_jZuAFo4kSHCP9@?#;ho0l9RC_xrk&%|yHdf!61HN-OvMajK&$h;AaYxCEt zD?raI^lU3Qv*DnL=GsfOwtVC=n;z1raQHtH7sFSL0j1TXqZsd0oy9v(E=3aPl`f=C zD~u`6bx4`gXEQz~4vg|!(1h69e*oSS;w(^CU&s`}%Dh~LX8{t-<#9t0t?#>J9Yy!1 zFA1pQs|DS^M>_UhiXJxD-iTaW>i4Kx4Wnj|Q@EDm)Q<>MFXiYr;4_ck{A@Ss97DLE zJ+oq@*P{Y~(wEX;e2QBQVP}+vnAA$y`LS#(UJ#9y-~T8kp|~=iH3Z@;NCSm>aMf>H z^91XGha*5rKYWluDx!Pgho<0nf`7fi*B;Q`AY~c`W*XZjwtBBA@P6`}k`%{{&^=D7 z*7<&J#G;=)uj;IiiB$&64Y-jN^xkVPfX-C9pMF7a@kjZ(G_C7KplCXzmZ1)15gsm@ zW#%FzxRef8fjzkX$T*b>@?>7!cpT|DE|n*e$;5ZbO6!fuT-`;+zyg#-SYI(GIPWP> z4p))yKw&xysSQg&>#|O|elOmK44j$A0=+g+@4&Bg!o1b3fV9l}Pmtu^37>*q(HIjo z#+*UW)obnd#CAZg&E=&93sR2@lY7SqXvb9u7IBD>z}k{ZbWj+5F^-Esv+QI~1%6rM zQrlTFZmFqp=E@FXpDh2VLT2qZR2nnV#lv}bCfTi4lr`7XdxEQ0u%f18?v(LdNdIZZ zLDrZ~oyoH`Eu2&*XPE%xw4rT0i~za~twi8$PKSDQS6s7;az|PhO^e_w9?=`U+MP)- ziakU^q&^i`&nHAB{qc-@mHGoI9Hib3+wZvTKr!R?#5x^P3{ZsWDmAWGdJ<8;EQbX6sl4v*TCN6W> zE1B`>e^VWwqrHX>Q1<5tjpw)tb6b}v@&nioS0}u4C)C}0Zb4Ei&tK4H^5PR_{>G`c znLSPaO?BQAsp61|4?xPvX^61-smKWo-JwjV>DXgZ-AQM=2u_u|dd^MhR6J`ItD zR48#dK+lWt?}$Qx<+^ZNy-SyZR*>9XL8xf9dIo0UHxj#FQRYCF>cjgjd6TVxLfUei zZ;Of$M>hlFFt&VlvUbwDg>*Uo3&>e%=i7E-dz=UE(Ibt>?w3{05ZHj=j!JBc#|BuY z@(I|2+;2vXr&s&f3`Ue4K+udz1Yfd%$!>tr6E5?>yI^RaRf;~cnKvpsXmuhjHuFhO zUsQQ!!}BYJjHj=0@k(H$E>@tVwF6f{&$yW`6|geBxriPIsbk26V1Im#2L(D7o55a? zGc%Z3^EB7*S}w^&5=7DD=gF|e8K-fJ=gP`sVGk*FeoSkX_=Lc7_VIFO(abvaJ~ctz)#V0RRa4rV&a@Wuz zd{hwPao2*ZO9r;Y?~d*vUt^wQblt=HrfUkUykLG6Q$Md+;*D^4o+yCh_%^<_zBkK2 z!v0AymzNq)ta9T%9@RK+hia2S2r!HUP)HZ zcZJm4Q(-Yr)TLkNZQOlmp+Rd^McZ$o)6|LnYnNmfuR1?UGD{q{ifVJAsJX2BDL*8F zRc*~l2a#mVUzm(jd~l_KkuiI^XAQ41PFuY?S36}$!rrJF5g*<(s1=g77p0$cW#@YY&Evc2jwxyQF#guJ=iN3o{u*cMHb{J>JnL zMD9Sv%LLSQJnBotI)?K)<4RI9yy+I3n-}l-&Q%B9fo9uUSKvpBxWr3-%FfQJ_`Vruc(0vR?fj#`ISs;sON}t$X*txE;-+EF z$Y8sc0kZ+|uVfY_q0vgB?C!Q5U&Ji#bhXE>v}hBv@Cj41X*CNp5~;K8%$kaBt)vg! zOCgT;%zHEhF%;rl4gQnR%C^t?Pfc?@IGIs2Hb-UgGB-_d0A+#M+Yz&!*alY}X@_eF z35&qgl0kwOEE54wZE3K`XE7+_ZEFdmB#Nc|T^xvV{utcwT8o`uiLE%xErg!5%21fW zgjQ8N-~=Ty87nP8%eFT=4lC99L$u)Dt9Hb~vTqCqe|lc4YFBcjG*7XLKhWp&lE6Fs z3#B9Lq$)O1*@3VzG3jxOJB!JfUrGzpd#0^2AJj{vRH*fF>Slh%1+~6|T)Ut#0#X6m zJg$JXCk>Wgxo}`oq};xn11Nj2zMK^Pb?vfg(mw^x`zF{6aZ+Yu7^w>q&}Q;D8Gz}Y z_&sFN7>NEw%Jh3B{e&?aIu<9}y%(p?U=z5aGh#{B5gU3+wbUzeIi!o#HkqWk_81yI zr!pGD;j~!=+EZYXgCEYxz``)Q>c#3vEmpSL35!I(KR-H;4bwbVmjs1awZ?~$#{vTN z6A(7KoeOBReE}7;X_5*c9%g++NSUt`lQgr!Vt7W?86IFq#mRow+2|{p-_t?X7bt#% zFO@NQL&nLN>;1>WF` zC5noJ!WEw{>c*%txSLx^^}Ew9wMT5ORs!iPu91cvy(O#=#rxy*FdZS~hjufbJ0dQ# zplDP7JIGBl#oFCnlJ`OS zWcWTz%cX|Y7G7z~;ssG&KlCNIN9{ToHP_$hy4v7FV7EjmB@oLR-{X>F>ygHRp%4-f zFl;F97qDkE*9}HF^3MFWC&ctxaSN}f5u>_6iGJKo_ zV4&c}PidEp%_}zbZ)`n<{t-T9f)Cp@7Sa>lpGmQ>1ApAQok!`-NkKM3n=9qthV&{$ zIOW$hJNp`TlYJPA$w|OpXYaLePlsOC2G@`D-BtTEFYDqcqMQ-e`-I=0A~pGAr)X3} z-2B(8(I1niLJxx!+|;ch`eV0Qfo>~X5!_gOcCo=4%4HeuotV*!qIk^r;EhKpyuG)i zV=+1{qJrS<)vUehr9$*tV=sZT?aURfQtvqy+g?EF$(yKm9>t@=*o@9oYi((-#&}%; z?TKMOs+XW|Dv62qv*4NSP4y!|rJ{N)C1%;_8_eb;=*uWZ*W4`HDN2t|$!n?e+czv- zGnb{&i#>L5df9Rb7K$osU4J(PiLQBqI4VDZ%u|%fPe+y&fpVu(-eB%-pHU?wxt1sM z_CD0Rarrlrn9zw`L4y1ComV#hW*SQ<01`Ski#^rbfHpJ*&H$s|TJti=QP)A2+9Kdw zE!>$o$=KC3rSoZy2~+~RIfuK!&@~T`?=m3MM4QbdAI$XgzmueWyCPGnxsMDArp*;2 z?84^s?Q+G}6A``*1WK~mAcL1|Vhz1xR{G2$F`A|%x-h-#mj9eTmNgR2|API(uZ+}q zITWQv1eD&IyU@@3^Ksib!vR7Xa+q21RkxNZmSKOKIJ$LX`}uRv$Hk5KV z`Sfexm7zayGUzxr6?0jQMcd<=if9C+O@JZx!O*3=uw;ifeX#)_@t(F7sjis~5O%GB z@k!Yo{wL>|=aV^WAAmf)TE+e7H_gtTd&keGaIN7iq3N=cp6aeg!7>5^3B_W9-}Em8 zTi9G+yhAIPWM0jF*XLCYuL8H`^NlDxKwdOR{pBOSF6c5UBumU&0x8V;JUog=Q2CEJ zwK(eNLPI4uI!h4M!%W-%2WE3yVE5bud0HdSmS;wEJ5D+gcanlgphseyz)yit(ed?2&MP!17&F?Jjg^_>PFdGhA z9rN-%w+7*K5;PxkY%qxLBZCj-Av|u3&KG_znT7dZ3~A4D)LFD>LOF6UO8G*{#gG)s7PpcM^=qP zI#$LeV&u7*g~0Gf$4EzZm+NmuZ8IdWWQ_6X?KWD3^A%_~x87E&O23Npz6<;&Q+zO; z!gy_MwG(MEs&%3+P9{LaC<6&zC})5*c7wyW(Gm<4o47&({EA2N8jgMg~VZJYoM46m5Qgnq2%q;by?yk)@!y|TU!Rm}_8Ud8plsma2bfz#W1i@>gk>G-t_UlPYJ7L&kJA{?^-4#}A9Q}- z@G)PP+87`hv>UTDat^t{)qakO_LB(39(KZ_v3^_d?^M6$KJ~{nKCi1Ver(o)qMyj> zm_YEduLZ+GywRppS*oss$ty5_|Bx`!wI5&e*UAVYyI7k+WqfAEL#Zir1SG(2OgIX= zBCusKZ?A|oTY%2_4bF|sm~q`+UMWPAn#R*}<;d<&U^eG>1oHt_7QeP#EBUkZ9k2>p zy87`&h{(yknjY+jB_5=*#|sV3;BCdYPrSgQt5fN4WxGf$t`pz7P#1&Q+ia*h{E^CO zg?LgQMP(JBxF)@)y#RFVQ`?qDUrv8q+$_l~y0fdIuKain{v#vRS6s=YZC6uo5aAk? zlLmg;yYxXdS}(9{nTH2~mb66y6uHuN(4w~rp6iGLOMpcSaooSyJz{b~_XtCmd_(BPAS zNTTfMzI8=Ts|KtRv( zb`=h4LU>WNsS@d814UPS?|KzVxYk$x-E>$Dj5@9xhhJrxneQabRMg7F0ij9v*j{fh ztw&LOZ@U5AVVxmUKs+*k|BNjpwBKrJUy zAsN>>-@z-BQQp;`WTD?@%##@euRQz5+;ZD-0BHvV_z5y88V40BHi0v@a1hu|avSnb zk8+D(MQGm9=iPGEZv%jaOG4MVG?-&kI&|WAzB@9=7}CzTc!99$PGL38C6zH6V8Kaf z!sVYsd1Yn>_er!~ns_>4Tz@9-160qffL4pQhJhNi-j}8VZQJ^tm;ES8#*xs--Q#L4 z8py%twy7=(f>TN?ej-l-a&j)=B+Q1PD{?U`Q)-iuu)7(0lhsW<#L z^Jbxjpn3e!K;csE<~MFoodMbH`Qg4v4k7Wdl$^ z_V9OCk6R}eTb-0s54L~=H7I8Yb)}!g5lcz2`^*WOr!KM{y4Qv3(XcS!=4+aEG7cNKN{R z`hWK)h|y547o;4ffZ1-pytbx6kO@omLn4;7s1gY`IO=afY2U~I$wW`V1w36uuK8LwbWEU%+d5O8-oj7fr}0#q zY@v%}SyGJ^CQZ+ucWn!sL&0rA_z+AI7k&)Z1-mN31374_dG#X~&iyPDj*-+|%7})* zi9Sr*Bxx9{NX5R4f$I#NSViKP#}vJ?5vw2I&RoQN>PVFbUuwXtu?kkyd9`K|iX<73 z9BZ}Fl7Qq1H7&XD1C-0RRyQS@cY07U6(sdc*@6--ojr&7cDG)~HvyLCjVC>~@`drs zdg44|6s)$8(D-p?yc;zt{Dm?^PL@(YO2(fmx3sSn>&$?+q+$0qHALEI>I-W7mHpgN zAB&JCZNR&8W$3cTp0h_ck#Y1C4sHLAS;Fa_2A174rcrO~ z9;&S&qM|u&6$>LZvOUIr378;>#WyuYRj{GqYq1egh>=7++Vt$_l?!Yr;!W%gBI)ar zF&LJ4E+;H3Q!Zjy*{RP)9(z+e#YKgv9?Yd=RArF zX^y{&m4Y*gi57c9Js7_hf3A`dth6$!I`m|gl)#=L_QQi7P3JrD494yXoRe_tkb9^f z?$Wy3xx!#pxG%b>-3MjM6 z?h*=d(>T&S#y464hpafWdi%N0fSSb&axrO#3ek65Iy)KwEJ&XmQrW49NcZHDjSDgP z{y0xYE{EE@+QJm%0Q!H}>%JW(bh7e#QoEFeMw6)XDdn1B52-o@E+7;5e#NBjU-@k; zc_jafYH*o;`eu~2UX-WMM$1y z=aeZfylHmj(#SZCvIzI)IA(gYO8>tPW#pF!lrQCrX zytC+))dYWv)Vch!W2-1Jx-$piUxQ|HjmU{&mUi%CghTi#OzbAUy4iS9_JC}AGAHr{ z8-offhP|ce2+~hC)Ued_w+cE~?D{uG8+vLNLbVGkyM@YP29!dmsh#OEz!goMsYSyI zWzu9BzJwq_x{b@bAT~$4G*^fd`$)OpHdE-^dAz#)nN#qNnm6F7hRd}qKNkjJf%lBF z!;xKq`i$7gt%Ux9J5YC6E9xW32)3b`>Z#Efaxmftt6f9}9EKsX@jw3yxk_;XB4~sE zT?tM7Z6K=8c)OEzdM-`sy@-C>y2MQnOA&(BEnnRKp$L6GS1gQnjjAZeeqL|iopTo6 zLzHnByYOhw3ue(|#V;qG8}^6d@ERVeEMjY?Q;U?pU?^ndUQ}-{+2wkbAy#C$9UGfd z=DpC%1HQ;jvP_kkuI3Sxq_c|0-M044ApJg-5L>}RR_^8++KuK@cF)+x;8a~{ES=R@ zsiRozB-)%4Hvso zwto-5!_;y&x2s$sT~>0t#J9Q}kypJ`sX{0YP>aStN8bW|6$LEvOT`s9t!z5UetDH! zAA|v}uT$=Z4yQl&ayI;4txR4B$4DfhC6bUC{PQT;Fm|3+(Mf(Ekpbr!Fo51>pVd74 z!UVY=lvqOE<0*~wKYGpD4 zs+~mQf|6GEG)^rECDQ}?P^7d|N-1VGou*O#mhW)7B5UI$yX^-*Z=-yO?$iZX%4 z^r5uB(#vERjcdHdEQRS0(VWnysp%sCH$ce08|kMec^-I=X_A(h)c~WZs9{e!I0Sl? zC&Zo7YGas&#o7Im2l`x@EcwhMpL^I1*tV8D1KIddo;L*SeE(}Y@y$yGNeen1t|%|5 zQ(1|tdz_VN`lVukyw2EHnNJTrDUV$oz(j;_Jez&<>8)g?yeJigg6#0EuP86Jh%Khx zTpBruOs>d8z$F{nF59a4mEnpJ9NbXTy}LrWaujx{S>h}ObKRWMb^}MK*BhF{hSg-Y z^@Q4~@fBU8BmuCY&Um)U`^eJkj<%c$71iJsxN=*QCTd^l!W)NE_uqRDGJ~i1+IsUN z!-o4lAA;hE+RA%q|G4wMS7)sXz1p)gkJX-E9NH_El0UdIc{)pMw^QiVO zl}SF7&aCbf1CO(zwl&OKNoOs!ObVcdcu{?ks#Yfb-Zau(zIz%L;(+!ew)mh91uCzy z8C(IwAtgu}83Y)bj}&RbjE8PIfY-qW05P78j;smv4%)=r%*&|>YN+hehbgG^@1Mj% z7U^@r7o2{mXE~iuj7K#}7`{HtP77EP+0$_PR@c90eWOaGTEg%4R%00l_0d`hNn_gU zh(scE+1iJ6iy-~sbGFKw?wS9MYFE>(YxQHHsj2GO5U!r0ICQLUL_5+DqGGo4?qws9_ z=YpXBJ5^?vwU<7S*?f`+Xjq;}_`B_H0$I5vC@fZo++Z%7VjLFsG*v2E%LN=HT)I$= z?8q=*+Hxgt9>lq9vF8B&s4W(BfKcSs6j@90C4nP*web~YM{KNvPK0!v>p489Pu0kd zxY@VEsQMeo<2PKu*+R+orOT!JL;W`1nERTHG<@p1dRoBn9)6k99w+tH`g4M+?yZSel_6tU3W}ZiqEeSqt)N3TR*2x!$B{%QAs;Q{r#C%68gKYJ3*u0UDwk?us;@-OX)MTp)V% z1bH@tfau3P2klkMoGC~4GSjJZaoUS=%>RDDlt)EKnybkp6Hp?zAxq zKv|WWs&EBk4km1dip8N)9iaW-88J^Q}S)u~B`*Z>Krk(C3WfFN1AF4*Mh+FBHhta#T;TEyw#dwJ{e z3ZZ2a2bDHXI}B)Lr4r`^%%D1a3n5{WkSfL=L>#^iJX8hyNpx${&g{2MgWQj@IG1Gf zI@Z$<#>ihD^ie68W*OH=qJy8-zuOVs{u|RB2`%r;CK};HH zk?`V>;w?x9jypDq3mn@g+K=6=Q)Xn4_1^1W;|RX@LFENJ(}%nzY56I(iXjZ&7(J+= zNDU&cEQvqwHQzei>*Cj>?XM!1G4pe9*-r#P;)R#LZ|b9U zn_T948W@6P{L0Y@!sc&I3wD+*4f4}ys{yXrT*JZ1&~MfMUmVT<>hFD6A*jh?X8VP9 z7H-i_U;m58G8pwOCCMQI4Vsn~aP0QuUrwcD{02ZFsE5&R*q`;8hMk7si%dt+*m)h) z7osdc)Ga=7;0awp*3bsg8|j{BAbcNOwcuf7S(E_;%roUS%_iLZ1k^O<$Tc(?mlbaa z1V&(|o~pHX$e%0x`u4M}pt5ktEgn4HpRaKxb=3|E>^$bBzO!%3mq-UsUMbgTvm3CB z{L1XM&SJ-G)%5m|!8PcMTxoc8rac&4 z>gJw(jqaf-Bknn08ar5D35>SP!R>16d_5$dK5#xL^au=)^X>k!ULkmX)-USO1Q!kR zDz_StuaBfP8Yk2*h`QU?%!xV7Ld7sGloYT{`qqtITn9CUEKyD|lnnxT8B_wX|R;}x#b-w@YGW)sh6{hy9nD^7xUs9+FS zf5(@>b-S6^79$5N^nY)L>dPxOa(EFF;MgepivD~n(F7Z1_4;H%Ws%%Bxs^4U0;srj z9$Ka#K2MR##P#HOP&~*6D56wxbg(G$u)@g&@i9}fG+Q+Iyb}+mHz^$4S7=80bDJ< zFLO(w_d*V5QuS27dm`Oip+Pi?ns1I5Ol3~z6iwPL3@@;XTy=c#cK?%VMQ=!7)*8|; ztBMRR_wu&H!osv_`;R0%0q6g6ymdisidfm%+(<|~^UAu!3kiRy+FT#x*n1*jTd!6y zl6SXHr1qa3MpSfzCLKK(Deg-6du%}z_paQmZLC5&!q^fQ=t`N9Y2&AXh*(`<))g}q z-x|shgCM&!~)Ga!mO{_y_R1F0-RfaxGOr&hTOHElZl)D~$$!WYqy zo<}mEj@TyrI6+CcwwLc+OY7jX9jzR%DQ1d{9^K?cv_U}hr0_oax2!rc)u=jlv9wH? z-DwpK5(E=5q6`x8Zq*qrj@(75GG_Lt^c4o7_kq{C&VlAo`VekgFo;@?_}Sg!)uVkx(aguy)LhLpmN zQ{fA@Mm*1Te-Bb5WS!o2lI>3=`d`a4W1L9k{LEK)8NPeAUoO0}lu3~Hl0CdDkGH~x zAkd5|0*D`uF>DvYo@%|#J*}0Gy;NI)g4z9&P7YlLc?pWQ?e81TkzvFAEPe-}JB9op z>?^%Z!<`#1iG{61RC#5tR@xN4UYgo9rt*Gy&g=7+`mp${*H++ivK5dB;M2F+coc-J zzJdstB%+N-t8DY~WeZrj4yzd-yV?L$2t;3xATrs3P-K#Oaa)T=K_ML}8To*~4-GW=bBGZse6(+^~mhl^x6T^Ik5$HZMfnQYm-G|4z?jd1Z&l|B^RmtBDaZ zINv}uFz1s&TWgUnvMHac%p9X)rLWv3>%QG+6qIqA-GDAMlQ^UP0n~>m4dnEMjKUFZ zy57QB-G_^M8$tRgaj)@5QnO58VTWr0s>}#jq;_`$g39vgj$ckYx!^avx!Sb2xp|{k z`Eka$jzYrJWF2@x3i@!Q|C-4#{Ld$X;GWUr<%=NXC-+n$aFaCZ;KE7KrI{8Kj zJH7;Y!;#hJ_YivFB)AcuP^@H~j54R5D$xrAM%|ZFDM}E60SWt#b?uz zzJb!wrBND!eh>`y%{|9iV$yc-g-6=-2n@6ffdjAEPw%;Wh(UM4&w!zrgrZmdwge){ zuwtyXbhD9VqJ$3+_oG*40dZ~iZKiA|tfm}|7bMjzb514(;8+S8axN}6e~Qu;Yrb_h zL;fCoky4GrkFvV|#zuwYQO>Qr2}%``U~e7`RZoChhj63oYBcM^xX>isP(LDCV>}J@ zuumcnec2h>8 zllPrNer@^AT8ij!ENgA*3i0vAx6bJ-ASIz=!|=NN0~@s2^RT{vJLU*`kb<%; zb&})E;jwllImCrn`}%~nX&iQEa4ROf?eYHJTCs;+2kzPyl+11veM(=KK-!2H>B1eMNGP_l6YOZ2HoK`%s1+bsl)%T6-7O& zhh5`cr1rjDi_wbgjU20`SPq{_)2AP7dI-0Aef5LEKsNSWi3CGTS1!w3YG`L*)w2*z zfIgz=$dDVte7^X9b$}OU^y;SBefG2O<7n&YB4&ajv=eV9ze#}7%#Qyr8r+b+5AEcJ zlLm~Y-#p2C&a?7R#GQs^TNi49mk0EdCmZd^DYVY#&0w8*8mujQE{&Emo=2rif5NOZBDK?WIe%NK z;YDAirEJUWs78Ip0wdUvKH+ro4K>*;a9GGCmyya;s*TlZBj?kj94Km+w3uYYGah#r z&Nc(Qp_gv6_x5bSVqL3cRtjfxh}V%4-Vw2u1m_~^Vu2^@F&ZUnb5uO@^rE_6&bu6ix#lz-C!1WR58A=vOWdc@o{guC0eT*J z+PYFZ5fqC=DxM&LE1ntNn1PD-ir6gvk_nMfzl73Thk8DDEQ1AGcx6*WoCi`Qy~@jv zsFivr|2a;KCbW^_e68ZvJt+U~2d6-@z9C8KH}bcXx^Mjpn4`dq2$$fx35i?C@#`+? zpDITa94C*R4#fB5riTl)tm7A#{>->#Y9-7V{T@d1D9xbY);wSFlvF4bHe_n>r1f#p@ANO$dDNol(aq)#%>(O)BBYGh?fWbt|X?HpJ+B zxJ`xg@uR$yVx=8m2Sq5kgsaBP3RksCic3kc68$onBKhxVjewJfvtH^qADkeKX;!ng zjfnZsv^`}{CZWJk@m^E>y65W){C2>T3AOB1dft&?J&Hh~X*0d-DBX&3)CB4SsYtqD zGTcNxe_3P}u@!=Z?~EY^PCzSNnICM|{KCF^!!n!P{;s`+=3AOSuJ9#(5DTH($Skt- z>8ul?EtE|?G$*(RV(tteEvsN*xuH!3&xbH95eay>Tz%Y+M0xZvvcUJ+yGO3;V|NSfTomf( zAp^Y_JCv3JZWUg`48WuPCGE)28rrC`$i#dAV$4FO!MpgQOGF1>Ud?JOU+Ofkhi24d z(*OFsPcq=MN(lH{SrPET*7g?E6I+g~B*upMv>~oGz?@?lti>Lc$DN?mm@980C z6Utpf!81NY;2|2*UTNKaF9;3jVTMlLe6m{G$e>-`GEn1C@~ei zKQg-|ZW9JBjf1JO|8M1Lw49iJeGOUpJ{cuAj zn~$GM3u-}dqD}bI229{nB(X_bZr-C<;bDD$bbv!sX_EB7^zyF_jw;Q4%rHsILK4j@zToK`U;RE;n1wjJ zk-7U+X?sNG#rymyRMl%2(1-N->tY!h}kQJ^1n=6)VC1B&HEmN9+!z;JLdzSH(U!JrY;Y53N(_@O2j9YHzB?o&6s9Nt7j=BxnYxlZRBxwLTPhx(#LP#ft$?s ztNqftan?W~L-wH;=x(MjqsqvuVi)oY9M zp8`vvsRqq|2p0$*6mYM%akNcfubLv)mutJ^v2Laz=kyB8MNJ`J644qc&1z{N=n$78 z{1q;=r{_K9QlbKmkVDq7ViFFqSN+HSh>>bt>$*gru-dA!INT6}qjkcou^(vD(lzw+ ze33`S(yF|ClZ5%M({$3I4jbu0;I(Dmv^SixDC>}d&XxKs(l?1rXJ6OYcpzvSDW{@W z=X0+L^#hJ2EQ*aiuRj?u+TTuRB)=H-37W$Y_NsP1!ZKl;aZ#0`T^kQjkQRAegkt< z+l)oY#M~FevOOY~lO#?1Q?@w|2 ziXLRWa0A$Kr{p~K9M1@DURo!s($Qc+=8H?h2Yl-QtQ+MdnM}l{J97hS8rc< z`jHorWlb9TzEH9nrGEg;(0%lGbH#ORC2Nx0y@o?NqT`<&aOmJjv2Z1sXfllOv4=zN zDK&IH%-$QPS}n!l-Q))a)U20c0e$l|aGIjw-7VXN@>7{vE&ejOC~Uh17z@{2f(Z*g zY`=uiBsDAv*L~1Zfsk=ES%NJUof-buZGOv6^cxk>`W}WCDaUv3`nIgz=lW2@UT_Lm zTU_J$`+J}WUykXA2k|<(x-3o%4e0kUG?~<8#au7?MyEnkV0?i-Rb}WPH~&*e?xf{3 z1}d0n>Ooq@;pxwvZq1M`n0E1u>lhakY0-+Ir+^=&7T$oms>cS^*EYr;e!~nX00`H3 z;BN3U5Wk7B_%uq^mUTf%_lBYt)nr-1zc4wU0b$Ye)?4x4q># zsD7vBS~=y2LQp(PE_w**Y}2;1X&~aWpr9hYX3B8-{z~UvehHCa@S*Fla-$RYBPM$F z>XiukymzG>mKoO+f`5^~i#r=I5*92h^XfTfl=-#(%@3e#O11Wnp+qrmmdp)?3=+{P zn%JkPPJC9}!F{owSDnDgIS9E6>%QyE?0z0Pj%b?V)Bydt6@SMGKMvJoRodk8@b!iM zbB6u4M4LHXiw`a|`xn%`e;z@7TswRo(7jZEz}8B=V`*Nz_LJX1Q^U2%Gp|+_SqHiM zxGlTnYx6ocN(UP9@fx)jYK~IZK{9?{QVJ|Qcre}o3u*F~86}vtZ~z`_y;_ddM&{uk zYyK27<0n*pj}mStc^~5H>?SFpys1=2+l?GA@_P8{56Nywr%(|nBkXE` z;`mHzzLre=;$&S?gKhfV z^?%pP8K|3)EG(KKfCsD~i8I%jFlru)|AmW$q7`cdK2QD`phC{BUo6~kcG%s+P8IEN z3&{L~l`%)2TDyHRG8>0-hVA1N8C6tBACzLAQ|NjPCZ|TO^86QRU%O|3dnT9WX|*8P zl5(dLA5mANykXa9DGtBI(BHslnmdAOe57la&(HI^z#Hn7g;YV0 zpX3LUF&eX{VsR1ZcMv^8$aYM1V`$Oi()-%0wyb!ToK^$e3QzYAt2QyYf1ODNi9(;q z1dWTR!-Z_W2bWEUM0~xgt>U!2%t=>>ztT7?`e_+Ea)VXRY>9B--%t%$J0oxI*N<*GRDCoUoJla9RR3hi^l|SYg}7Jj1W(n+0W;n z9J~^%kh#3MayU4Z3uGHjaQWa-A5Flg*s7v*(>jghCZbc;@4@Rv>m5rmDQ6P6(YPI$ zsNxO2taHJ(X5;PuPtvuSReB%Vj>|2OqscTWrB}S#5qg70`onyjwbp#ksBEgFK69iT;wUnF485JUROjJI;X~^Vv z6%X(O-51wJ=r{|4BZ%`n;TEFz;w+xx+zY>RoIv!i7)U zDqN+{1k3XjRDFAUJg;y6Eoh?EsMn~Oc_}F#YnC_T(shq#I5OS!y(Ew>G|58^lfBL^H%Z=NI}>F;!EvQrT5o zLHA|dBQADoLKYFho9S;W(``#go#z|dhBW#ZolW`T@CCTLg=z@IloBR$yH^MVj}=cP z4a-IRNo33?Z5S?w8v3L`x=`g3Ou-a6z9tclYo;?y9NRkh8Nb@4*2o^x znu8Tcvd2O9mqi~@8KTbEM^|)V2#aR5nPQ=sS%1YohH6afTylwYX7)lqoj*cX`@6&6 z;_&W1gRYqvAL9I#&u{B{@66akx57#F`@4bcID5X@4(llaVk9*wc(*`fjFsEoGWa!M zXCFpd=b_fM6U|v@K0qG43g2q6rGUJexgYfdPpv@p#tE-kn!;mJv|WQ@ ze#+4sW9fn)Xp<*V{|=})No(mIWOyr%BwLYH_Qr) z+WR9B3})?nUm-uN{MRi^G7Kew(N3tX*(yONoFio0 zRC*6k@*RY-NTCt=0)6spq%@qTWBsh00t}8`jBR5}Gd{ zv)F&elUwTrYM7bTex3X(Kxos5>(OzFF+a=Qnb`t(K!d(4vD}AIP9NA$Pdl+sS+MpB zfiV2e1P8-~04(*!rub%w?1}^m?|YnU(>BLV<^UUy;n%%iTsKCYuSJCSTMlV*-)GV& zbiYzX28Gra(6G{uku-9d22govV6QZ$x;o78mmMHKq;z59oCjk_Ha`)!f=7F#oVFG( za0H-pVLNS6q}CTqHrTs;#HzGDfc1CrL)58=7{)2Sii`rk46PlNS7DSOpc7MZ*Ij=sag0SkW6{J~6pR8(|HOO}z?}XDKNWuAg)i;T@FpS=@ zv+Ivkh0U}=`y8+Uj-2mcHYe7pEww8b`n?goQGyzA>5_xrNtg#f@Q&}$Xz}{pLH>B_ zd&}S>1^}4CK}xf_vsON^#ydborXk_}6^ayrBW0R^rO>uf=JJk-X9h{JgP(dQVRX)5 ztGR)Dx`dBvuS+NE*>PWG4#+q+JulE4D?l*u($Hnb=hrFURa&SbnmjCJP1={H=Ze7T zpZJllxk#%_l9<&bdeugYWsf#qU{znn;&l9Jjcg!AO-3eegV@6hjXr?$W*Cb)0GTgFY==*G7O48HRlpk{gV5Yyv=8 z?umd7XGj_f$`o|=K5|7CL3$(zD9G_DHMA(^RLy@h*XlzU*SFfxx;G>_?yq0 z9#8{^G|izdo0$7B`2gl3a+#owLytVbi|lOr1dabQ$09c@bD1JA(;1~8@~}n zPil9TLdnT4sX9N9K1eGb&lfqlYF!wP(O9q%=prb_vq%pk?0Y-o!6&c942A12_{y3+ zI;{0P>U_$tR8tFYGg|yzfIc^!_{W+uxM4b5yxM18o%Lrv@Sb*S*&y1Qh#)d#RfvND zLkYH44J4e^Y`v~lH&LO6ZGXeWI^We`;@*AOus^*BhK2CdNDA0Hu!QG_lq)B&F-JFK zN2+P7P*q9w3Kb~MuuumfG7jJR4eC&P zVk&!C78##^Bm_~|jNr+ra+JA`#io;J?)mKy!wok#JI1~nnxh&w)L8M47tsr!W&ffn zUo^Rp_AE<}h7N)8(VtFMeZuNrqa`E19H#oc)rWb@jl&s{ChSRXJ@M;2oG~?a?1fJB zA4vn~Q{vNUWY_l#&TWta*eN+JUFAx^{F|W!h|<12&8DTLVZhrFeA#l~XXi#U-iA)s zljBx6a!JnY3c}-F1vgeBf2?B95Dx5E`jKNh`sdX%vSWW)mxi>F4Z)wib49hzER#)8j@Uw{5~IN5kJsJu%lIn*(lGq8!@vGx zm5}uML$Q~Ko|Z5T6eM?awY;?;g6W#AQd$5+v95ce)!au3YWUnXaRKW$l^!Xjm9|9X zWs?&_FuRHYKmmsvgGTnhSIx1Lm$I_`%;cmKrH!vj!T?>i?4B9m3rp>*0vXs^rNa>x z`iAah_v+X*ei(9Q`HiLvY|m9!lFmfrmGB{89~n|>%RRreHy2T;q^fgO++t8GV0&3{ zs_g7k%#@_X6@(EB+p3#H8RQwfxWV?T1d!}=M7)n%QE~V3{Jo6rzk5a|0fMGlakR(i|3jlL4l#% z7{T0`s*l(%=EmDh>*R5p`Cd(6=n4cG=WY z$Q#*kLP4>yj6DMJ0!S0m`4Yd?nvRQOH6Xpb?zA${_6R|koJL;G!m+8pa z$Y~>pNv_UI(jvh3e4)O@*g?ps*gbzTTCAXTi^K<9=;R|j`;3I&S|cBY*~97yaAv<- z{ugoJGvG+9?RXx0{ioE8=ni40(@So3p(8fm{x{KcEDn8?v5<{{?F{pPUSNi3V@IJz zM>;q}ygAsN1r7d}-LT<_NGYVjR+Qyax5c`SR{408p?o`t#*DL^x>k-UgC8*NCF zRuHRGzyH~Et_ecj=029-r*mjEH7K1Vlk2s<=WS^e`qd%X)<*a1JT_g-QC<07 zi{pY#MgFP{CBIyRMX&frD2BM-C<^Q%v&hG~9WDJPbl#-qdPkMt8cPXupu87;R(>>#))^O2&h1pgqw|rEK*c?Idr`yVVqz)% zP6nMBtR$n3((CSab}1~H^3|N&Pl4q_>q7VHr<~((MKe==i(O5U3C(!CQWpA^RS+tj zq^c^`F@M_2D#$2E-0RjRUXI$isWPLX!Y`51lG7J7(W=XacWAlTA0ROd!rpf2h`9MY zI>;PqBNDEh+ZEE4P+>xe+^vBhHSODbIHZB^i%%=%Z`F-5PTTZsMK2$Z|JT^X(I4t# z`L?Lq%yf(O+S^=g=~B**Hnn8MNu}*9Ml(~b5+%GE#zEcdBxe0+;~Os`tXByx8LK<1 zJ7M|#uTpfY4YjpD9G`5OW8XQgirCXQlN9K+7;(x%Rjo=CVmoXM9PgDxa>~j9iuPqQ zg2l4N(R1YF@&%~WzWElx=jc(6wEdTK_~G{?X};1q_eg4zkrgmZI7U|4B-n5YMmIDi zW+0p|7;n*~pO>%66~VN#%23p1ZU<3JDosw*w~p}sOz~?SJp219Ebc%N7!3QV4p?T+ zAKRY|X%SD|XGT`A90gixZ_)DB$1&$e=(d_Z<$*qnw>g<%bNQay;L&C;hcGN0K58hA~WB_^Oy_A@<}ilQW{SpY}AH18#Ci ze}VtFmbPcWiOl5|vZrWdPa*ib@NY09r~Slf^Sj9#n|yk(4u{bLeV1JbgyZZ#NdCtu zh&JNs%C^Ns0w$OK<9NM^ec`y^4ty$&DOd55Ok!mw{RsDkO67rgsOo-vnh7ngoGH9n z?txCZo)rK;0B)P}$M2-X5`d6{h>MOO%;G{h#J1O(7+$rd<`g}kt9a|ZF}-HkSG)%e zXagQ1nBCP!pw$>2Mm0(MH?ca2AEp~Q+m!8(p~O35wFaZ=1pkKnBLt)2 zg8d4Wc!>-+<0THOAv!S|lS&Vz-*0xejfwcVK653J2h_K?O_GyQHa#KVfXc1`;61V% zh?;Hltue1zDoy&8w?eZg=a98Nqodro}g-)TBbpcQ&3?wI4(ZcGM!c?W;NH8HIl68r?Hg~8p zZQzeCX9OVt|4)__3zk^XPS9L48zEb+0HwpgBhf$~{mVgyO+ZMRYH3Qm8h zgZ(rhf`Gh{g*NCm%#b%Ck;$OBCuoAqrf-d^bH29pk@=RPkk_v3cmtV=EZ)^ zd^C;>{OGY|FsTiOo-rzybl)|qaL9P-XN^;c_ZI7}#?D^gJt4zr?aAc02j6meh-J3{n zb;<*Zx~V{r#Gw^s_hBK!5$#34PH=H#(v`$d3d5o&Mudf;h>Jh%BR)H~A1=XA;Afui z=I^v*A^{7Hwh>ZB*y;&q&tq+c1-BjyQdp=4A>7bBdAE*X)ol@CsEy)^yWz&;rBY!; zgG5K0s*!buvp-X#O#p|4!MdS)RE&(te(Nlh-m3|F7R;5x$|_^BNAiy+onS zHNBw8;5+GG5(B5Ko~QbM&jp+oMhbz4d#jG-Qf1)Nci1dYV^a^X_Kp5sGENmtJ4Xqr zYUKzUI5VqUHfy@Y4kQ$cRDznQg7Gv@tgqrr#+N^e^M8ZCXNy)td383=_Z2g9$zBKC z9aTF^wcr@H^XbJ}i}0AYs;mAlgna+QW#ZhBalc6J^_0{LnQY{ymu&vL#tG{EN1A&3 zW=5V;>GaVOQlwaO`-Ksa*KR`#?@B^}%3RBkWfvF_cb*heIwv)ktRDz4CFIJFu*<8PS`<(Mta$?YgWM&(=Xj-S-9_0U=a7l zU3#PAA1rK9ASRtRv(5HvVBP`jJb|}QZnYRReDp|&k104(&|=nzEOs_>8K`A&e+nG*Mf@}#mxhty>WN4EXOX>vt?&9#-eYBNTczRiC%v8R zfr(JR-PK51kIH*~sF2_FfKY8umrOxWe!6T#A^RY^C5Q>y`kO>du%3`_Cm3Jv&#tHg z=lsx**#HlvF6TybuE;y1Y+xtNkov-|b(EF7__&8J!N8Y08b`I6?d^$Ds(uA`VcDx$rU5knQ$se)(m1_P789w+|x( zzSO>Vr@&o!bA>#RI0&}>o9urM)Qy(7h_~sxE|G~*B`}a!HTr6O>*|^$h(Bh&Dh%|I-Z+k7!79aa6frwVyc44|4jmK zVUi3t>piS3fod2SPDqSUf9vXxoSq<}P^WRP@QX9Qok=@9bTa_q;WR4CKUIZI<6C!L z`x;ruv;^z55@8Q`++oUI5l-&EL(GBJ|9)ubxDn_t74T{8oY+w$5V4UgJS_b+pTKZy zs4)^X2_Nj--N3lZ_;Jr3s~ob(4#59%&L2Cxmk#JUDnst!09f0^dJf;vHxrW;T|OG$ zB}{)WQuM6EAXOrDs5-Y@SPymMN4Voak!R4!Gu?#!%4mwU4i7mPrTWO3wEUZYWFkHT zGWJ~n9*o+2Sf--~X`+bc0C5+#azs3w0gJ+qk0=VRdSwj=056Xz%WGZx=s7*cuHF8&mt(f zpm_1f9IDLBfFC+e)i7=y?tc_YP3>5I;EmMfZ3cP{#9$M-xAtISK}pO_OEt^wSEn_% zzRmG$6DN}-WjHSpPrZHlv--F9|8(10B~C<`elgnCEr9Hf+d+aE{m{N@K6@N4ICTjG zl=F9;He`Ed1Ih-As2#xVsz!Zdl?c4`(;vxL%v#aRLwE;gEqTjIOgI4teWk#=zl0Dz zckVc2M$_z+B+?lCTG^4|=b-yE;8fF{P$X=S=piEJ+#Dr?ziJ>Hk{69`78kPFQ*~6+ zkB1;=kB!+dL+>4sCVY&TmY3Ecaw)f$)`!=%=~Shw!(fA3I zvtIJKW2v7Qekk$G9ka3EISZ;kN)#FbV+!d zfr0YIovAR;oc-P&1Sn7iSyC>|VA)cDp$e#)p?YjQ_aQz$4e(dUVX$Qx(^psoWJ4C9 z=GO4kH~Wc0>JB7;XQOSQpP4;n+i_ICA+#EYawbP_4igKHXXHEq-EEahmrGoJLFg1n{kJ)Hytn6GB_kj zfEu3*(E=rtFj_wOaauhND->A&fLD=4l&fEe!SVEIVlfEv zYC8Tp{29^9AfO7xb?GXYl#WPp_jn5sp(@fARZ{UT8t7QTG*DTCG|P@xQ{m_UW2k*G zBO={hkWt-?{?p!?pRaWa7c)05ufd60CZw%+N9CLJO#-D82oH(I4jbHI#l8Ea_h{u} zZOBOiSP5|CH*yR~Me^@ucUEDZS;W=XrVR`;!$DJz;KiqYMnA;*?1|aM5$1G-83JOu zbYPhf?n$9t8XL5(nI#9CFDFUg9o zYqi8TAUuv*G1lq%zBo_iDmi|BOl|*w?hHJq|2S#K@yy~Crz?3VK~3%COFmNz5lbGZ zq=T7HK0Obu+302iHdyD13DI&RIBYijUdS+J%itkb?rV+z(?RFJ$b91wBn47c*t8&j zq31q<({)x?`WUFk5Ao+ZKr~09vpnqBTvP|c)6IZ zBmz)V4Ej0VN zosfWB-FID-9IYEIObVO}Rm*eoZxvfoTFBsju~5^@hHn7v9xieFLLnv5$Y+1yC*i)8 zGrvU0i~Azv>r^G}Y`9M9^A9Q6nvMmB!i!{~TkU>DeC;S|!p36iOHpJ}j^;~n=ZQuP z7rY8*H5W=)F=Ss4#I$?m6#>=P;E-eW`XxZ0^)1siw2*bzsTt)Z{q$W$iP)e=P}B(` zuQY>dC`^lhc+41_$$iLs7qU!^aWj72Id6>j>L9HYyJ2~dsrOc8zdl)!yr-kL2pO6K z4hqYGZuehyRnj})y=H>I+HzuMJ4{B@8RqRy;rgh@x1lg}d~T~UE@hq}8S=8-PYa8j zaDAH%jVJ1vsW>b6(P~4DEdf-pTEh%*QN+|Xndo<_bT1toT)|pQq9XRJVcV8W6$g>d z%mgVTVdpQx{8JjS@F_&tomYr*rl`nDW*}`f8$k5g^u`t050fAG{2T!kMsiMB#$wU1 zs(2NHjC=HzTqkUF?DqPcWl#QuNqGkd>J||)UVKkSPAG(6pAb-eJSy=V$ZnG4V>e+f z@;O$nHYO282zHRvk;UO9oIo!-V0-yFDu`@z1+xggX>C(G|MK7&4FzlG6R!+V5X0)4 zT-BOiY5Te;JguDHCxAYV=pAZk7AmY(2cG?WV<*`xB0m@wV$SwPG}eaZy7Vl{TGtlY z7YLu=z0bP}9#1vl$p2}omo(uDr`D$G8R+l5n9OK_8?nxEVS4oNQiFr1VwolLzz!f| zJ-bGa;MB+mW2NY41*5(dJLmTdSXm~NoAYiAfJD!COn@k;)8?ZC(8Qk_Qn@JZ&P(Vx z(*!ryh7>JfOd0L2g~eU4IzK5aOH~V`#A+=f^#ya>2L@>uL?PG9mOA)7Qg=3W(;a;+ z7M)acScdHbPwvX9;@XsO`s5}7rY3GF8*lx6(!2AU=hN6T+#)y5#YnOE*S#BI)n_SZ zjs+LD4A47FUcy9YgHNmzni}M({wnS7Sa&S;WA*DX>m5+&0Tj(4=RoaW;)^W+a9iHD z9MQLjg8L?`cd~@j1NJoSf=-bknWc?0Q>*NXtRJm59050j4Ehx%)i85;8xaQAn9vDv zn06JGPb%0Q+66+KmjEb7|K`LgBt1KY> z_?E>5JV+pnO4(uIY!$Xa3gp0~LjTSnh*J*GP*ww!-H)*VJwU?0|64j&$5O>HTEveq z)u8Ax*bG0o?Q*!1G(r8Wzl)PCmfQ-N2r3(tB0(qU57z0xpMba;>444GZ>aVMlZB+s zCDmLRFT_S!I;|lU>M*Tg-p=MEq_2Sr+;0XOU{fhDI3JuhXlT}~pJ!ASba{YP&;yAj5NIquLp53g zdQkPoguo8V;%7u&8FhmaEtmS&MC}^6{Vbvbsak7e>W||?@K>(IVU&qKTcFhcQ7gV5 zFX*G3Sc4N(7O{AI!wKMee6CsbGf>W6oGy~b<~J;A_*VU@&{OKZWU}aSo8`v_)K`19 zzQn#~v0VTbgI;R+v}AJN?nVgo5|HTLqEvjH1AeMItN$jXWB8bv4&k$B&{dKbHtz9w zDH*|dYi(3(`QUa^@)#YIC9j$6MIsP%d1dQLJB=Y+*KK80nIe~=x{6%F(8H01ryF*4op2+1jN zt|%kC;OX2ASx8YjAevEFs2~$|nW}98;mD(a7>*x9HW|w7!T@-RbD|n@Mw0U%-Ey)hq`a zSOv@zv3BJ-tx(nTF&~X)Xh>5X`TfRu{*&}5jPgzvZM?)q$QK?}zEzH#;yNP|u|;0H|#D;wIoHid$o z6E<-stjc$lR({8}>+HKGY*q}r>$z*oE+5kR*|DCIUkC@%8c<+Q6=hh&;0Iab^=yoM zcEq--LOIpFb#5)Rih=aOebCil4+espJ#VY_K^-s&Nd~wIkmEJhrWP?nN9Be1lB9@^ zO+xNZ%gW_^pA1Dz#V5V4xCkO?a$=`Cw)WFNBS)iQ2Qr_9BxAUxJv0JxtNgXXA1>d1 z&%!I{b9DH_Yt4Cv9}mBr?2cZSi&uJVDxAIQ!5WW5NJ$$+Wl$wh>X4F|6*SHx$V@wk z6G|Xqrt*WK(&DwYVY!bgXz!mst9U@(UK%CJ_&a4XgscHj&BHWzHa1JpQC*)3mIW;J zo~%{?h}iYu_xWL`m26s!Y1V4lmF}Lz)BHJFC67fhu%4(bY@ku0FtoCAkIXoFs7{Yic?-{?4qxrcx40Z)>(V$)<2%uey8+akD0%Kv1jGm`agxlMYu4iyl8CX@&nBJ6HbH7hdw+P{jbZ3thuxf^qhLnC0C$O+HpVk0M4AF54{1Kf46uR=QB}FA`^lLo2q zLvu8UXmikKo5)koOvs?|}fY}O&Mwbj;kb)u#!T|{| zU4(o`zULUF)($1OLWfx`X+(Qg0_s5v|22}A@C;5%oydx>*Kyd#e5Gu~=1B=+V4O@z z=3G0L9=-7lBX2qo>RuJ=7{=YM$%w8~?NlG%AhusMt;x8?u-%$Dsa#sC|ET8#LL3}Q zkW>3nUc#n$KCA?@OOoGHHoV}`rcx|Yrywkmxo(~cOVUP{wKIyfnyW$yTquzTpM&!=_B^|_F7xNHlSP5Fi6ekR>)I*s$3V9!7YR3jyt!-1{`)_7EZ+yWOSd(!i2CB zQ1AM7l!YE-^`anZ(=9yq&Ay-|IK0K+5d10A$b!w3+Y4V{yshv z+4W~yQJ1HV?nSggTUay6rb*gIH7dU`7fsWy$&BFEvgn5K)%*vQBa2BI?qu1E{#ok+ z9va=&7Y&0W4TY`Sa(R2)#=6IlsjmhUU+q1I183g0;g#*uR-hAY1F}xlf^0PQneZMe zVnG--)Eq*zp=*St3EF2}`q!x!WD9uUy>!uxgB1eKczE`Hw()>lkddNK zWI1Qf&4ocg-yxxmxdRcn%jY%Da#a!~T64VGL|{k>BiVbiWsIZ*q!HAv)_{iOv1kYB zoRd}mT7B{F7?ZoK`ciC7*d4dF9ZPv0*XW)$!l-Ldd*tK`EuSPcV6u0N_el{dYh--iNMCw@h91XNSAkAfY-0NqaiG1J*%66}`3=uXv zgOU&GU5K!&Gx(CRWm>vIw<+y^oRnAQ(r^&nu(M_^3p?kEmB>e#yunUw0E?zeWF=o?&cOAE%>ZnzH>k!Ecw8&BBffj z%&y>tRPJ1Gt#mMj@m8Tb*djW3S1sAY9tvIM2=4__mOh8#;7vM3xO_dmCE6YSpFYl% z^FD?&)_xO22xiLG4{Xx+`INpR9O#1;L82W|DaRnSRSSS@wgITNdDiup8DpVb*BFF6 zr~=2YT<|-`8CA&EG5f-zouA#2>xTNGpk@XcZd^~Rr_ExHiL^Hu%T{EqH`Pdin1E*~ z!$(&Y4(w8u&>4Au)Q+(xFFlQR-x*X%Z}TUW<9S`5qKf|u+4P>7J`#}yR6Zcz)W!H~ zcxqhVRia0Jj)wRaTy1R5aFU*SCpz~pW2*BtL)w8wT0rnxDK_YhutRK=waoCgQ1h~4 zUI=NzqEw9-MEATGtpXY-bkldng5$>b|CC(*fZe|^Dy@;S9GiVRW93FHTx3}CXW6Am z5Y8!GKg~E&Gv0TM05!-(=z5091Gd~eDJ1ye$rbWsuhk8O5zz-h|9x5-K=I{Pp7IaB zfVw>5X&-GZ#31>j!=-5lN=SsIHq2Ca3Q^LJhcff=C_e>HGSzFU9jk?+`y3~mtqWN( z6B7M=3D05OxM-Q<)6rv;*5MSTE8J!84QF*1Sz5D_S&yXcz4d#kwmLKS4$xieOPsrjz<`dq}tj8=~DOliCXXYg4hnq?K*y0XT#*T+L z<=DBK_G?T5z*_Z3KW7mHllb%$9_Fhvcs=ECJrpMLbz3u~CPcY<*AsI(mb0^!MHN}N z5eia$xu_4CDpM_}^yjZ_wCM{+$p#lO$ted3V6tYl7Y><&^ zSN)lcMqgg3JN?9Ml!(pWB7YPwhlv4rn;M$4+G~;|%s)g*VrK&KK~VyvYV+?x6$`1j z0fsLC>V6emP47ZImyEVt&9n|$T$eKHl8(akxc&+)<%)w?fqI{Iq+Ox_(y1nRs+QH1 zaW0t-nHbjH4UBBdJ>s1!nYbDgAKpG@2wxf96aPQEuC>{8&S`<>*6Dk z(3Y1wq|O^~)EI~Pp;lx#0%eBUt&eoKI%}Fcp#8xLu?yVS8kb`}(OEf!%5j+k3BV?w zaD;J=wF5DH(GYDw6ueTl4&4sbV`tTGAkQFM)y6se8-j+L|1lM9y)lE3o9twHT+fMg z^{(*o$38NrFD(_xO3TAU9ENGmGF@$YNbGm<^#@_5wW2Ow^k~3R#}QhKQ&Q?1ep8Gc z6aTIFr&?tPL!5*>u}UK%tpc_^ZDW-p>HVbOL?L(k{qsS+VIHGzq2l!SE3@x2!72Zr zK5T4AC6mIFCrv?9BDWm<#wSa5u3JK!M!Bj5s6-XOQ}y|9ctm9P`A^3mv+v}|O!Vf3 zE)H^HDv>6%HV{&?8pYld3QiqXoJMHR^lxfUDTj478f(U14K()OE}>bBz0A@@qt@7# zM`#R8OH`0m_Yb8-du}^tkKC}}8Wk~EdYqAUwpS(%Ygb*o(&Zxs_p0=p34#5sIe@29 z?3oUEWZI;Re5>5iga2YOrRGYZ|{4 z6jsy#2x}Hf3%UG<7e3#p-BM{TUgz;<`AvL{jM{Fb`+ZBpl7_M2{)|M>o!NknDh^J; zpv1A|>_!j8Iq0K8tm8^1phkNUmQemguR!Ac&L61O)*4)PA75$Gnz-)=T6^ISIb1)K z-7+b}AOg(V0{bYTafo-j4`%4*9kB}<^7CM4SS}vz{Q6?q?x1O?v8iAp^H)9${R{S+ zl$}h=Nhb)oH=*~U4S7-K>u)@WCHxa~oJS0qKFqi4YoPro!@8MoFYfsKSlx^4t)OM~ zUHkfXnXIKI0Yh({lUkpWnXcJ1o}moO|J(xdlDWmX7}U$GzAR{x1iCTZRSQQlNgOX5f^%Ek_^sjQ zmnn(d;_i$aJ*4Vk=Kl57Y69%FmFTEU7%br&mAXxMB?|I!Y349l{&)FZNg#`>s8UAj zI6EE+-1ig3=Dss&&U!bMI>keo@l53;PuE3XsQesb8m*v7`hd{G+_=WvGZzOhLxAMc z$_sz}mPOWHfD1JCJp$ayh|URRPbb=%_&gSnt)j^Uv1OqVN(KR6drQ1UKC@+q0|^;V zTo}4FX$^*}G_d*ZFI8e`gR!mTuwmvSU0|LD+uHpadl1)qI7+YIyxd_F5G5!^60C@N z#5Y*dKK%5S_x8@-&#&@w_UC;1V8EaMD{;Q0Q6UPtp~Y2MuUe{DqSYZobGx0S6gZ}9 zu#n$Q?;eCW3H8`6u-@7P`H1D7SoVqMj^uUuKMh{VjJ(>LYNMEUJcE&gorkg_jEyMf zpU&~Gbc&cS>-BbB>KxCRKJ!Nit!XAux_Xy;PCJaVx&#fwf zYH$hKYQ#{tiXynr^|g4(@XM<2#n`Xp8{Bz>sh9YHlCLNN_0J7#k`3uzQ&vnz{;?7Ex z6A28=kFh6ve|CQ5X*ILAV`9VZ^sEfSt<38J78e2mNnV z+Xw|nROJa#Mqrat6Opo^!oQ!Be=bsBPTdo>oWd4#G~42|1&#^zSLD%|=Ti0cT=e;! z`M`yHRlFX@X1PzC&9R3>tUehs_N_;)wo%tV3uQX-7NzsVG(t=hIx|@g9Q@%!P2+VR zrTmr3F6b2dC7mZvoah$Cm^X3_QxS-l1fA|IW}JD!$c#Xh=^qi%LZz7FJ@Iz$>`74e zM^0BX`}kAD1D5P2v|~{Tn_MZ$0>~5M_q7u6LDjMmYXpuJqgz}P>1u`mrAR8~{WwL< z=RdSZv> zJX{geo3BPdHT$`^M8;zqjDx=B?hr_}%sAh~KGAT=taW?FSuA$b$RtYssg6EcP}~|> z*{d*tqn(+Jxi8I;nbU67*A;*$srmWEx5qamPLj($d*UCobhGMPd{xOESfp!u(;T`z zqUbO2Y#y=D4hG8cL)z0KvT!>H#F?*Z)-NKSYH+;jO+t`90U`d1GXC}E`7A90({~{) z1^)5opt3*D-IUs9bwO=doxg-4jtZjJIreAJ5qFFaNwG=?d-;eabyMx1XH&QiD_ej* zqU+C>xkyl5LA)VxmtH#HrH-BF4dl>et6-w_>|CI1Lc9DBLM=wuu?%Y*LY~zpe;rWfQWuX&M z01l>!!7yXXQ8)vyht8TvoNGxgp~(^3fybL|2=8{B?AwP*{W!=i*a=|uLPftcQIa(q zaBQ83`bp)H9eb+v7dw{H@<=K*+w)|kmMq&hmgL;}^SvUpcNQ*P&~`%279x>2A~sb` zif8pF6=M0-YTa21c6TD3fcIEVqFK2$iZ;fWG>|R)?WYVOGMeM(4mGGyFt6%PL@!J> zrT={EaTRdIy=b?uRQB5Ul3q>v74$C;ka>!Uq858Js(PViPB$)phJNJq1bLR_L4I2^{+uDoi zLCl*~S-S{MV31Hl-d!kWr-jQqfYj)G7o(l6yd4Fu7a9qOX}|t^qfAc*EyL=pv4!Bk zcs@_pLl|hlDY6-2sxyzi(G3*^_PDPg+A72b@?MiN`t)>6s@=3 zOeP_h#y7=KT=Mgx#33*L4kV{gINqFf3s}G?(Hw!U#8)4&Ai|)j&R-u5WWQR_(y7)f zTEE3`YHy})bhAdvZHtoU#AcQ}`iNjQX4cS%j6Nzx5i8MQf_zvSUbhOpO5nmP9hgGGk;t9fvM)EiNgDL*1){B7 zW>Chw>Z}o)i^Xp)d4l7cFIbf7T6%?GPo-WXPudDy+feC$2TZS(!~xs7vqaLUop4p$u8&{7F_AzZCKe&~ z5xHg@C%4nII~VtO|VDMO)yTbo0 z^yI;chA9EOJEg{(r7d*YAfYh*(VdsN#A2f8Eh7jfFJ}xorJ3O}FsY0ITA5{WgD3(> zn&IKkO9vD7rk3d^@+FtVH%JJ!x^d|rLLSwLAGE-BG`<5Vr^J+}OqV888KB~1408ob z75Eg?yj0g{Z8BAp|8`h*=}ASFypFYS@F*_fagM(*a}Vjiqyj=Yqy8#qD|%S~P~-4=4YnOoI>5EWc!q*6%dx`Y%JVfP9Sl^!q?b!;L-2 zSR$kjcAV1-A|vgS(a}Sbe1O(;Xce^XiqC_oL=mRsemx7|&d z#Y)aCF+bN%24UAoaC(Aqr=ENO>liTwuY2_Mfgr{y50Mh;3-LmdPoUU3v%1?)2sLg2 za_$99+NM0#?G2v4{k_)J|Gtoty@l7yzyhlc#B!R(xLI>T?piN{ zG?RRm8Ui-)WY!GYu!}xURljMLtj`z^xd>LWJ)LuQA2NaB%GJE1zJWg&v8uOmZkp9G zUFT2c-Dyq~`D(M-qdDi8E7vWgz5`@8T37J>jyDPKz#N8On66Y2dbc!aY{4KLiv{z0 z0L12smG%ffEVYYG=Yhvlr}rn4k|C8!R!?jNDHPGa*DNalJ7EALvcI==^yyFNzc;cY z++qmb+%cAIb865*IT}CYnD^R9DicUHuV^vb=M>cuQM--=ua2c5=oEG*E4j|M!Lb-n zBf>)Ho&i(8f7+=&EsoJsQZ+&G3@m9gT+wtL`ln+!GR?@N7 zw{Y0lFwrp24I=;?Z8jLnMUO}=f9ba0c63j&WzH1L3C`Kv_baO3FLRY$3Li`aq!B%e zlN>mp)toStIYB2rQoq^+`1WCVv$lA0&Xr$);Pg=#<*1sRvp4x|i<9a#!e*-2*Y_=I zvl+&kbXaaN1A_O!e~r}Sev{PB2<&M(~kDJLDvJAsoWYecu45@RP|6 zRH=oG3DABgo4A;9cvIuiUrK2!8ykBy^iaEerw84qTk{{d&8i7aK{*9mwG zT>1pHb+tdaKu)r0JSAA4_`L2iI|zxb_$8wGq{Lvo+`-waS{uoW(en9RF(ZA}oR0Yv zNc)N9hGk&?Z*gvrCz<+X+(frP$iro@BSbMrTgX%6%1^_LWz?Q;Tt^?u_)1wP&X7$mG=h2N29Zf<`*Fy6ild`6X2pxZe<0+f@xNVz3Nu=>UX-4;6WjK1B|2}1 z2lUA^h2xwN8W7ckP#0h@u`9Ry*_Q;?fQCeL7)*d%|MC`OsY4S$4}o!Q^&2;!DBz&A zd0jrJBUBv$vkua$*w895=vkVSj;5`Nd1>5lWh@FJ1tne^=(?a0`B@N^Z#;ekeZZpe z9kNfEe!ivLh^D~N3!q%OuUV&cp9A&&DOea7uap#e@^9fXKM?yR5asrl)GW>`D{IATH37D?-vdl#TT1r>b_=7DQ`F3SR#z#Xs?ai$0yp+_&(JY{)_ z92tzge>2cU4-uQEDO22JdxamD>{#oI07csFH9$|i?WXw$W2FwE)}9b@gpi%S79Fa^ z)9z?6XYHsV1%o1sw*!#GlVM;S|1iOFQM0*aw5j{2$CfIk-wk8(I|{1|fBfc$^*}`4 z)00cKl#jQCGhky;*sF~pJb2nw$1EZ`=wV~B%oV(<9ZP6eN{2~)fuph|Yp(xaJ!6J= zUn7dIqxHfD1EOE`v5@2qfl^k?50mJdwxo=~Q4v&8_*}hT7eQ#Od`FwQy8hAqGz?(4 z_U{vyXj}=5$VSLw3!Y$gFY1*qwiviK!WafFqO~S9KjC*kC}3v|{uN$5V0j0--B*hn+@KZV;*xan; zo7@cOU)fw{J_h5?9y)+ryIjh*u`XhBzOBz?hLm0Wp321xQ+T_Vs`3-iej-d6dK;{8#ls8juc=ODf?Pj>iK0z++ZDD65e5zI^zQKQZh{f0rpLh zsl4{uzAg$lZWhO%fD zb>)Agv0ptf+nbm)9@Qoho~&?5q_^m1jOGA>u}m*ZY0Wtp8J`dE! zmjxx4v<&utW{k9Nu<}_72JH!(3bh^`B&3@D44oi)#;iQN-_Rgg ziweyhlink?oAqqgpp&$_xUxOh^^~GJKD9v^|2;+u#V>U0;l|%y{b4bwJ|?1+1_WoGy2B#51V!rbfBa8} zqWGK)lfgQB)sDl>L2L+HJPXuWaJJ%Dj`}XhH?G7CKYr^}@77EJvENHZvVF|emc(Mb z$i>tT86G(p6o^%M6Bl|X+ngOExiS-1fn?tEmeBhUV&uMDCKcX~%m~6VRC#<3+T>vq z8ujuXm7A`iBuvR@rP`3ZCWLYg7sM-j7Q6|ZEst(jJH^CHd&f^^`2_TdTCDo!|b z_x|DZ7~U5=Ge}O22IUEsgF=brKG{?GsYt0qiop4cl6xedL${_OAL1k4sl%%tY_a2( zyu0I-$%)d^`up^UvBX9a!_p5&tPa;eH3~3LY~NK$U!iD`RZ$BMo+JVL)gH+|Vb2E` zU3Ytv<<}TQB9uKlXoo=FGl~MuZFwIWOb>29z|ib9RqIf+98=Qh9Ynj685o?3RF4}8 zxHeS|1(QdnkoD*+TL0U)C?x$6$U9n|K=kua$7E}}jDVS4AN3Av&AfZN*!vLt%3-ab zQfTz>$8I0&yYR=I$)(K3mdsq3XjI`3r6%y={Ci_>-ECi~m@%V5;g6YLJ?g`HVd#C4l->Tm#k+UoWF zk{XqG?NLTvE-e#fIOFPUX3#m99!8pTH*BveTrZZUUUZ6uhD_~KI5XYnf_P#ZDwuPWV~XtxMa zx*G;E)JBg3s8R%w+S^cL5Skyoxm!bMgtdMt%p&4Lb1cKpjk4pEI zTEkNIYA9EXj^MSf@U+HftznU>*FeL4G+ z%g+e4wB;SBGnfi}lmmT9KRozT5@^xRVoW@3)`GoNZ~b3wIh$>uN*E7ca+tf-q<*e? zn}C#$P^fo72C6%Zjht@@H;Zj6g?bBV^(U>$+2*i7{ z-9j$QBttL0#+c27Y-~@KzQCX*f-cCtA65duJWFx{o*n3;A{O_iSa!(E&WJSwj4Pjz zB9tX377%%NRVet50@U!_du-fMp8jPTo6)cO(%N}D!(R1BrM{~PBtur7obu|N6~v?_ ztaH=XIv6$xn*F)@VPu6=g<*fMx=FcQWu|Wll%$_Uy;%P6i6D@~S0mV?M!jEI`bYT` z@ZkvIx;aCvZ}gf))L#Z%oEbpZ6UMFpnFoz&Q1Hu)+&YF*h-#tbwgYJX1_vpZjhs*Y?4{?kY15Xz!oyWZ4bo=FA6EKOAt6&bc1q!4m?Ca zcA1%wF1hJ=hUN-dza7P7!34bGxVbYYslibPD?ZUn^=JRQ;acf=!MMbJlGAf8tgM(dmV zg+|YM0gEu3rZDGu(5d945E~x>1QWXC!{~pLpmvD}{0h6}Vn2|=Bv1{ibni7ZLpSV5 z+_w-BVb$fDYm-&#xM`)k2fr;ZpI@%h$W;F!z93T z6@P%3xKnazld7cBTn?o={n?`zEG0mIKiemOp+_MC9yO08R8)tQX@uC3-w416ppG=L z${P}uLh&VaypRu~1Cq?@B}GJgmP*E}y0gnA^lCYKA3$GVYPF&;=zvQ>t@^2TPdt&j zKOWq-!c1%h8u;*`CM>r%pg2=Ld2CjnYIwDFt}0Am8OeF^xlUX1;z&9oZ8+^>J<6U8vW*O7+^_*kv)#Ks>WbB$f;=C0is#Ht+})+FdUZ`Va;pDOWds@(wjHe zY5P3(2-NlT9cb$JltJGu31Nh4$Cd46>AM8*gla&RjBG?|XmPRa!eAtT8D0wN3=(`; zgS3=_QGU1)!H`8b6PFux*)0vrse(nGssBPgHQk8=;ArT#U(38@E0IFT0!{Pipg>g! zMD-CFn%Xro?FmFUveME|O>k39jnt8V2sLH8Bn!vsQyum}ns3qe0}KT>rYr-BL#7#8 z+j}PYEi8?@!`;sgHQy>d!jRpQ`x%=+J~>e(A0|_I(XjU# z`9q-u8M>{daudEt#Ne$An#`_F{S~?H+#%4jqcaa0#Bjv!n4@f$Xc*el&il}Mw&BcY z=Ly+L+{c#2lK5AvDhmQQt^$0ju6(NFZG1_a;?(2O*VW024Yv4wlw7M&dq&q6|#J$#TV9I@Tl+35#6zY z7c&m&WVz%W96k<>S&pl%dCIukY{Sc=8ioo7UikJqhiLZq3TM$cZ_5w^<$u-fb7hPY zk(&NwhgV|L(ru`ma|fV{2LAnuWQDa7UWk5w>U%{6PBa4MtnL_8En(if5b)1iJK|M= zr7xIB=|K@t+7H#EJ2Tr-tv)W?h0#|B=}EMCTFx8AThAJf`p*GAL%GtVM!b+f5gh)w zjl|GSAeFuhZ-c0tR4suD2^JAV{K))2kxt1@3h|htB zFEA&tF~7<3YXH(T=LVU^S3g>JR^aXXG&SXf5Dn4vh)9fQ5h)gzBCyR{Kw65%10fg4 zjC8M1({Fy_&0 z39SD{eUO_k%lzB#u7c?F;v__;D}c>sZqG`}Hvet76FzZsAR^9S9$ZpxR^nv!tY-CO%qFQ8>l5&M&L=jKCqg8?pp(ahlhOY` zBns0*fzCC(frk0XhV2$hy=6f29O~=pN<2GYI8~z3o>{QqF{r#3${>fOAEi-0EnI0i zFt}S&6EEK(v8~NsV8QuLPt*iU3Nc3%TQUOO0V%i11Y%2`cY7=B=*I1YETd6cU9J(@ z-Z75d(U(}~zD0$Mp1bA=I$l&LnHV}yo6hjoM+RJ%@MR;V5k?tL5OO`|PBc3@*J4z~ ztT%Bb6@Dvnc}@I%`{^5wpU^#XsqJ1Si!!VVGwPQ@?4yfP{`Xk(yAdyYu6}#f7d&=x zr7XkGv+$Z&E)A|CVlo+fiYR(Xb(*rW!eE}88iBn_!GgfZ`6t^wM{3q545unfEnJso zn2??CDIH6Rd|enc?DE4Oi-K|{G-iYihg4a_2NdeEC369K5w|YH~HI^D;8A znPqeXBwzr@GbrnKEExQ5Dxn;&9UUqXL5E|@ zp^u@&5G~k*gUfK=gD#$wtORlA({=1gCKVCjVn`FHlBOe#FeX3M*?#_MV{$GK&ay~v zZQ*+eTiJMlX|Tx5>98!ZQ1F&t;xFPDL8@#zZ`&E-EsW9O)(2T|2IBBJCP_ceE$C7; z+L-##W@>IlBOAMOPnm{LMjqz4cj|$FZh?=&YS{uLHK~F*rLde#uXHh<-NUl)BN&&P zIhQkOaDo}n6uo+h)f59>Wb;lG@Xlo7aYz|!*H#VPT|{}ia6clW`5H&`mKIz>?o=;#=Fgw)qF+1wGE7@%Vyf@Qr9a znjvumc{T4pjOW)J-T3HHSl2k0oUD(V>xZVS`G@UlsG!!9Sa;SF;8{_HK!!~QC2rea z3pFcJwg-E_WDm4SS$OliEtzXwma$GVDWT=8)LR~=9Q>g9tFA&AJJEXXuSgFW^ah^U z7imZ7&mmjjHZ6^Mc^Mh)j6XiXSZ2^&DYxiCQhys%z!tLxqvmCdi|bYVi2aS9y^d0N zN(h2#c)WF=xrb^>gu{cOa++5ui)OHORdJJR)UJZ1%!2mhVdBYcX*banXi8a- zDlFHULs4oNjZ$FrYX`U+0=x>k4B zeS?7e(!3@u2t(=X)YW+T%E84sL4lJS7wX?Y$Yw?PPdB${YUEssVG!8%fBmlSj<`Os zl;+s#>K<<1GhIJf+Nwu~h1e&`uH1Ki(j9wIs)2CM0*gSxjLh|yGxJRDi8q5)A#s#{ z9|kI~lA<AJ`T6hz%?S zQ>Lb%Ek@`O1z`rS0aXm3OJ)6o?RcJ!>VGnqB@golj4t7;?DC1ajkx^i(ht~_NlV$)>KNo>_-m8zP@fpS-+ zh%7_?OZ$xi*1p(5@6FyUY9y8Pq!Lt;ikh?nb2VTq$xWtDZgOsa{9tq<4KiheP}}4V z_EGY2e?+}{atGm~l6QR>V>Gx@2*`eWXolSnyPHvIAiw+Ib+B4V>fMMa2dsgst|@U% zc{+QH-qceO?_(sS-<*KuFQYCmdG;GVx8X28eHvPvbUL2pcyh8tTlpA+1&XWE@j$I1 zugh9Yj{|3xQE4luy^aA2HdAHt$_6G#467!C+fbxShUP9Y0K+SOy5p&JBND&|5&Ik5 zGL$yOy~Hgy&Jh=`Sc2OPG`8>PC#I%;*81>iceHtey6k$w86eEsEhH-jDhis6*~Hv7 zr-un>X0M0)Ctuquh1q_yA;`z{iwFl#hae@OxX5XED%4^wOBXpOe9<4-6W)A04XIan zJQm?#{mZy6_o8t#r7tWCT@^gl3maZg_NSoF|0Fmj#HXSR~6avtU@*TsD z@!kbRGw(Bm!1!u`eyYm3-T?fQPloa#V^~u$aM5!WfcuVMUcvTy;@;u?WkFy)iDFpWkCe0w3cvRcsD)$-Vz3&Ne@T#P(Bxk9LtOzDa8JHqjl4OF0&N8Y%?PZ0mN zq!zVUXutKvPcYrN?M^$~@mw$7ZsVXe;lh0kR-G|n&aWFg+v8GrlZILMY|ADHluEWt zzuh|$gCJ|CyY94c+z>T+XMd)65mNZeJ0 z0;gYKWu`xIRLJl|bgSM6xeMwmJgq(rJ5f|We1^PJy^N<>o;Qzomx49%z1>k>yQb z->{s9kcdo%13Y&biaByB-}0aTb2+Mf$jA1*;>T`SeY4dm>zTnyOKi$a8iu&~Wsg_g zCyKQ&O9RC^iGD1`pB(O!Fsk|~IJxMhZIY=y4*6nf1S}m*1x=DU>{+%OADdG&jCu-1 zoTsHCK>ut~+y5IP!uty0yrx$saH&jcnAu}12@nDomciqBGCIAZ4W+I>8rsI3dvsqm zDHPTJKy4gMBNWKz8S*KL@XEEp5~ByvOI`1G!_mQppIXsMM#4{@5PslMR9Zz z`B*vfdCKzJqaVImq+Nk-J}HlVK8zaYT%&*?cXOFb3c@a|!?(aVz+0Q*(o$F}Fo>Tk zYeU>^{d<)>`61g@_*LaR!=WnYouPz66ro^O?Qc$ULy)}iJxw6Z83^Jfc|XP$z-fM9 z`#_Y_z`^@wc0yQJi2;P$GjsrgTI5%9t(<`a!?6tc@WoVU*Q?_i-PpTr@zg1fi-I+U zl$k-kwi0aY>sdwL03_3A+S9d3-iy=6I3F}~Q|st;>TtnUr8>VzdA*(iU^M5FGjQd5 z{+yd=4S;b(6$PpN=eLR<00u16yw7o+P0u{qqkDVR@4A+QG}lsZd)^u^nHcqe_hc_4 z`n*RWY+O_E_cWJ^q_EjfQkG{X_-omm{9zEPNJ-(yyd1RZ=HBU~g0d`7uYB1VN?)Pm~rgGQpe z5~0mx0y%i(4%(x*Ey#_HXS?bAs{UejHI8P=Wt6-RRIeg>8$vp?5%S_dZ#Q?=fPt`f z7Lg>T&y9x9bX6YT*h(`&Q6qRV!hG!b8>H2hk#lJh4SE%;zjAFbm)#-K2+uuK4S!=i zls@M0LY%&J#-K*9zjTmnRL#sbV5&QWJX2BKkf9ed#PCQc|IhYSbJByIij_mbNMb|+ zZM1Oqrk{_#$9`(;0?aw*`IGJ^c!G&{O%a7)6BVkVxDMk)(}$dku<8K(pVZyh!cA#=5a!Gp^?ZtLtR0tC)cF-yum{^xSx<&= zUlrT>>VXNg6l9?%MD*)mcYu@}4QEpli8tDU$R;-+plDXF(9(sU4%w8FUYRxv0PkJgQydFVTaQm70S$@iEVZ2>#twzz5ot(2j(t z$j2i_2K?^iSC*6PkI0lY@lP(ztCjX-)VjsR8Aa(}byYob#^Sg>;0a1IgW7W}pf*bq z%pb%U9Sal=Ev&BmuHitu_j803`6zX-YCD~t!~<@}GS~@m4Xw+y7*a_+CTJK9^r!Mv z@x49P@_Ba1J>n;Og8&=ZmNIU3{|TqH)GFAK^sUhnd%%k%Ifmg^1SffR5&Pg|e2aWS zrcz{F4f)sOW(JnL4-3)|D}(qvxRf9Qz<6(0TY_($eg8$ zwdPON6CVqitb{S&BTE`Oh1rw1ZxNFtvTJy_NG)iV(Rtm3LEZ}Fl9u)8`gm+*%UlW-1gb7ON(4%X zc}^&L9p)I3zX3e{ZxqCn%nERU{4hXu^UUn>Uz*`Np*@Zs=46un3u(EVsUcT%8RxfO@> zPr{$^#16!61a2ADKXj9PWt5-IV*p_BTnXcFGF>rp#?o$U6EPxq3Zw3h2`FhKH#&QQfeQC4@JkJDlr^2T3h3fxa-h>9F|o zB=(zp(slqEe@jqcHS*EzF*Ee85TEM`l@hEEA-vJbWdqVgowL^+pNkR=tCbr_z#o6H zSR$J9bg$72ktA}0gol;`9$Z<9IQW1h*cvfe@Zw29Zr+5Q4GaO7!cU)fNo$!rYc!vI zRL~KX&3Ct%RH;vA1*EZb8NP*j^k(BSjA#}sCQ%MUL@sdX1%aN*7@UVgJ^{H8h|k(O zeMDsAyvtN*Xu+#FeuNk6TP-%q{&73tAApl&S=shndp9+2#S%e2TrC^wVI<{GT0NK z^*i6nij^1fYU;!6`tBgLHGxM6D>WuqJ>ws3sY=aB0LayvT3|B_0Oar+9%OB)c)3F7 zTDSGcY?jAVdiBEa5!7( zIU!Y}zKubpt?nGy7DpS))o4d5bZaO?vn4!QN|n1{%4&k-;?0FKf##czQ|bWQ zziAK!6@3sajO$AS31A5cYRjOV^Mqh*#asU|X!G`~MjW8&1+k|u*G8LemiT-@@SvYF z%^BE076eIhBkj4H-zE^bpos<_yH3rNJ@j=;)Mmc>IHZ{vyTp-mcHAK?IU8Q@MBHZr zbkR_i`al3Z7XP{EZL0LXWvhE>?)_B^UdpNP9`Ux9WA+!q)z$_FQ{({m{e%~RB0mt= z-ebv3m)^aURE_F#pMH0jF<6F}z)$HR=--3AZ1AE*L$|mZ*+{2UK2LO8!m2) z0xbxY{@mA58NSDcgRDpeOD@Pt#qEUFSIw<4YZT*iSx&tHWmmCZ1^V_}u1fUegQljv9d@QT2dgx!Ul6n{OgnV1pHCsdph9@{2PR5>kaucaebSCdV%r4$2z+3?J2Fg$3K>Xctw9*atdgT@4wf9^W2CVH^WV zh^%U}0@vfB7NVfqb1 z-7CXp;paeRV;h}~)>NcIzaSJCxtiubG{SmtDAN8vY^78nbXx$r*XTi^OJA#bCI2~m zEr$3ojpkvD<;v$on8sm)$1PwP=k5+jX$UiA2Bms1`1o4XxHJ!_PY^`{g>?}6f(ZDyu4_%;h&pyWOz11ze5|Z6VLCA(v6{8TJ*K?*ET3hdoW<20)hGZCXDL3{}?wvZ(nWTZ=pN)0R zAwoPJZ_)dF06jTZzCd2<`H811{g7l0NFg6MJ4`_B)I89&=tB2g>fJBBey6~J<~6Zl z=sh688x+xOYsc};6|kH?|9{yM_X8Us9yWYJ%XduND$<3bU2w0{Z!HXs;V$KoZlX{r zdD^1z*lqfoqIx@UBfS4lSOaGt8FnpUzpDngP-#$ZlqDJe0Bh~ z{R_xPWu{}SGgZQqdh9dHF3G05mx^49N=Gv8heWD5iWjr)JA|`uer1%TMU-jKT%pmu zm<{{=dUVHy?u_i!;r7mM^|8C7FoA$OOM2_J`ay7`iQ!ig@UpEYb)@<^x>!ErgDqW$ zSmu}r&-)7i9=519gtjsQ%b%`W#%IF1lkm(kP9987L$V!*uIG#b=Vnw@j;xPH_@Wnf zW+<@~iHLUQ>C;!O6xy2x)+gq|aK41?pw5YZ7g>^VnKOf8!7g{yEyNynoq$F#+Y~%poc(U)<)B}ouU6ftR%ONfs85MImw&OgF z5O$jap4Y3|)2t{@;yMYxhbpnPu$rViT`I?|KFE#N+2iI_ZIOQ_QFF>6buH)uUO^rV z2=IVPMFd#p#DJ~D>up-jKux|k+_>OLVsGY8QQQZ_I@e;j0@ScLPiWPzBzk^`>Q_jD z$v2-$>&cIo)rL|#g=dy~X6$v}SVfeIr1*f!SZ4pAiY%@yt{DHuYU~g}7TBpnc0csB#5=`mYJ&HKToU&HnJT)AVT^MtbM)(V zA@+s<{Fd{$p_wQAL@u75JqJ>bkjXbe_&vZRBWeZB*499)zbcdYV9`8C1YvTwuoGIEbek{~9ts#{w}prK>xQspXlEGsn&t;92*v^hQk z1KAv2a9Wso@vuD9`^xY2qp$*L&N|1Hcl%(oo7mzw_QMfDncZHUzT4ub#8~REdjD#n!{yM;Y=STw61xSNih0B<=YPQbRO~wi zu7a808}csQA+0)$bit%+SxTQfcGlMdDF|_XR4gt994g^?{1+T!TN~uHY!K*OD#_U> z-o?!}t~8G|S7vi;J_oI$#To^ zkN5~{27`qYbB*VGlMa^S-5~z>-kk(e=wuS#JaW!~jyg&OQUd=!d@chp7@{ zjt1rnNzj-Wh+x=+qgI%E{+s&&@Or^X^7M8M5}vunXpz1%{QWTzJbsiCZw?2WX^e?; zH{*IiPnvz5o!uESn&%QL6r8i?d8rH6+WF+L(wrWstU28xE7iQX3RO9cZM?ZwC&;gK z5k}6HrIor??O2~-{u-GDl;3LFtf#O5(9tq#1TLGHa>nKYX+2MpN1O~qM(v3=;<^u} zV~qw!qbg4@4m1Dk;Zm`exY+z7O-^-8=Q*q6+Mj~7*=>(-%kX=66U0)X z^lvTrk14Vpz;PbmAEV_|V;rC(CDE8p@TT6PUw=Nf`dc5Jr_#wKP!iA>Eh1J*c>YOa zoR}seo*Q-V1hGMngxR-~pV$Q@iE)t(h6gv{DK#+B0OZUg7tq-Y;kzy%A<48%#h5M8 z#iuRN8|jH58n{>>piTbMEcLS6g4j{1)crKgh$-Lerv8}OTixkg?(D~8>_Y$N*1~d| z9#qr@2SHlxNY1A?rnW<|nN|hme=-5f)40&+W0JN~OU&qxjNzxHJsq^ChO5RA<`I>Viy8gb^8>DJQkz z>E^;NtfO4i7~L_8S^>+fm2jD7-=AFgo4GC`rxJyi{B-yAlwVq+y%)n1aq=sWTlTnz zp%#v%RX_zA2O;h5UNk4yrn#BtNHl)HEe=4KVxG4Wfe=SDdks)VpO1_D*nks9!*>i#`Fub5>jEwc-DW%1YyQF+!hw*_eAD#@=ycf!sNIU^Cd;f1aTlH)E$9NI3VPmTM=V4=6nO@^5N?rHYWB7+lT7KE!CvR;Zhv zp}_@U^|$qGA`hU%apLRgB*1gVi4HV;K^_N;Bl51yu2(i+iBzsj;Uv@p0C88^9}5P+ z;&0@Q0D5hu{MvMa{GQuYQ^&<#aC6=gD`?_=mi94Moj4E;%1Mc*oJ|*9zo2C0rOU7% zQD7(?@V5Cl!?8cxLncqeWr@{MZOW1IImBYEszZois`pc$Ho=aDhauByJCRqi&Sp>-t^vV zU!p&4UTfj1R%^)m0pDImj3CROA}C8u!Q1Vg_GFui#7EQqVptH0 zH|*>zeg`x^#5}!98302~&$6IwvN_3U0IDOzF;%?_vSp5S`99di>K(lP2M;fxI1%I_ zy@Mi(T`*NA30Ak&l&St++DW=RE@IoiFEZ5sK%$4+G(Jz}`kzQTN8Q#oOho%*=m{6H zfft38HE|t>!k%=OZwyrX4>`er#?D*Pn8cdAqRRgW<&=t2T)YbR58g=Rp znXkQ2Djn6ZYxih}W=LGHM8;iM2lYIm*b%&Yo{>i|wL({7uS@ZaquIpt^@+#FXGf*v zNICK)NWlQI(HlmQAni(aKP1r*y%##z;n#X9k3gV&eB_O9#4isg4Y;O_iwhgarCr)q z86sOy1utbcgcQUZ1{2l0Nczb}?*RM0T-`G)0HvOOD z9%WjL-(Y2=EH3JN5&MJDZUaKlT0JzkM$FT&kNc$aKe+6*i^^2k=yC|u*{-8xx?7Sf zG2&{2t9@y&Y<9PeTqJCQf|TltpnxgUWHMviC1jv$)E#Nr%rI}fpW4^QoO4Cg%}5%8 z=40r2&6c#Tiu!-WGzMVkvXb+P(2j8(R}J9w$8n=Z$*-HdEu~q}kqg{QrdnHba}$T~ zDR3IK(y?ifXmmeTIvO7QQ4Cukxh;mc2t|5(+&CsdmpqUkA=ebCN4n4uv25G2!)}lF zRFWgNM3;tw8cVM>r&2RdL5BTYr+~!^dfmHtrvPB4sY9hEf? zT6Q-rKQ4O)aZu#JgeY^obq^7?>$#J?}s7b*!ltpiL^-Hr%7VMg^(U?%D~lN zW8c+xHxpe;uCt30kQ>+bZ`KhVGLXR8({Lt62n%p-Z3Zy|mdOorVq9J}mVm;Y6y_5a zKq!e|Xt1%95oX)RgYQ8FKc%E%Xh@Qtrsqyz{X`<~8J&ec9?sX?$ZT+FKcQd&I^M$BM|r zBU7cW?*=|*9Qv{u2a_js9NCjODoCe0{5k$0WdbyyEbz(WV$9qi0 zuqZ_SB&jaK;jDCG^jjD%(te4gstm4*db-O`h|!WL&8$LfT7G7ZcZc}Ux4$HeXw74E z0Jn{nJ#mx&{$0vlehGLrTH6s2K9y*#)KPTK^~cf0?W0k%M}SO*$*fg~|h2 zzg%!c=)LJv2L~V(`>cm({LvPaVX47R(uSH~utxM_cT>5%wc!7`9p@EQ0oHX7$3jO@ z-Wxl%>4E_*=){<7-zwc}%shpf7~G(a_*n|@ap%RI%Sdb*-%*)fwl>vTkdxP-xrOG~ zu}xCqR$e9+EGK>EyO%<__uIO|SSg>UGMG81Wmlw>`yfY$=<7K3h|k|LiHWnMkV?J| zPt~y4jHmZta!If8`|BK%9`zNvqEu9=f&&Jp?nTji(WFU(in}$S_^B1x#s22={3v=Y zgtZ>4@|=1tEHFQ7D}jSV1euaj`R7SFePx8kcG@xS$J!fjAc$9-qsmjXB?nwfaq70} zXz<^0*AlJ%czz(-=}Z<~AzPvNrm^m8k(ompQFK1YDxqw$a+B>crFMdW#d_3g@`;=; zv1D)uhRcLI61{KdgJe;N(E@bQ@#=Av5L)WWv0WfA>LK#{ZV6}hbdlyh^z1T%TZ&Gn z{5`!hBg2EJZHKipFPHDqOIqyV0oN#T@tuJD{?UQa{B-{N=RI&Kvav2;fMXg?3`0WfDaioNbU;Fws%J;PI}mGIvfKAi_44@>&*)z_ zHOAO;1FroPDMqsVg98ay@aIU+vu^bjB+Oe22N%=Z<&-~OvEU?Jot$4}uy27dw|mx* z2Gn~7;$%%>@ndCt?`KyvQud}yMqldOx23c@BfY?eLUc%ZW&L3@N;wCggr)bS(lNKD zgl8J^SH;f6#Bz)ys(_os4G+xkiufkzssSA?78(98c@IjLs#cmgKRJp07TBXrKGLw$ z>Z>}8T3glM7$8Q+cdr3uh_yT6ysHlV35XK0y|D(pmV6@Z+|w8w+jWdj4g*0Dm!tn} zi19#JW8e-Lh3xDXNFlqCsBDNwaTq1er8+J<&N;v-3?f{KMvUgYB~v_xj-5FE0@Y`rsD|sG0&f%(xYD~g;0IdV@MzdgDhWu{;NYq>-m!m@F<@T!AVKlBTi;9msyzUzoyvR$r$VLF5*cGF_eb)I4Tf~9(Qk_(T*b4jisA_bI&PoRC1&~ z-Vg^IvRSuO6QUwr0@)1v-l_>IMh@_9dh>u(0S2ZDnJRmIU1_j=nJLb&45kC4VFpe+ zK3mHwiGA>_MO4yR9a=UK38yJrpwKTREq8%Pl3yS<`;f@o`?3sK@UdI4d_Nco* zCT2`Sw)@cgf&4+P!Mno95BdHO6)7aI4j!OD>l@t8&GQs@HFZ<-%W@!6mWz!B8D|n=xv|Z-LGS_Ur6*9vqgz`iYY_WB#EyPziixafr?{;W( zW~;#JT^MB&GI9iXs%`Rr0AvtL59p-8|HAFh{tVbuu3!8oqQ_KI?hK!08?cHsLSbK! zbSxyN9~67z-QKJ`xaOGZxwg6<1*a)!bA|hnMQ?Hjei<+&*%XHc{8nAR)$Y&XNKI`vBCgV5De5F|xL7cav<^4l<>PjsVmpLjAQ;G)_$kszLJA^> zvx}9kMt@BbCf(++Ny+9(8<T!4`;^C^*KT6t|$!K}_e{Nn*jNq+BzdNOPw{40D6$d3MI>oLXGv z1k5;H6bwz=&s%<8-xHBJrsA!Q18@mY#05qUY!X#?cnS&7ZbiMGo{{ z6aycTi|5%WTS{4|Ao1`j4z)5Ufx8I=7!YrN&U#AJn6Ti^U4dvlow}bSHE=m#KU|d( zQAwx(&SUTSq3E17MX>BFY$kQXnX zvLwn(-gHPu^a5Fg6+LP}_EjuJ--}cmzyKq?TZ_s!hKK zdOv~}3I$3Xa{fHZ(I2+gzAtIwR!cYB)N9^DEW^tNESHKKO?IwYgg=FfJJUVACJRtnZrRi1ds zX|cZ7r`Rgs(8}s2sBEJ4H089klOInl*ww9oCsbn0yh4Z2{Ckk);zwjX1$R|l{!_Bb zy_N%sqk$MU0Ft_Y=y5+aa2|@kDy?R3#p1aaH|oPX2VNXI_<%*4SrKn4yuko!kFS ze=AI%$zUSenY8&epa$X~y+rWvOF^RP5Yov9rZIA-5^A6Ngc`UryQhN2%oe`f-n52ugZ)UkbaHqpEpD{^{d^y0IeG=2a*hYe!O4^o>7=# zWgl5vZxJJ&$|+wKu9cjWx!9XBkGP!wW)1+3%YzXL9tAOsI_ggqU({QdbjU&*Ygxxq zv2du598%qGc*83pK*kf6&1tS{Z_-s)i{_k2w=ujF=yUBl>poQ|BLf%E$mo|yaZ!+z zKH!-`?w?vDsDllHFV{(@F5>>ThJ;*6aDF@GuR55F0hCo!rIp4M($$|`{?8Bo zq`*&;E^3kvrO7xOh5e(s(_oIkcb!?X$fPLcMleTXUPRufI~V%%$5_ScVul3Cqsm@O zt&s0yCU+qYLmx)XDxDCEy_imB`wpAn;#snB44x@*gcx5dAE6GX_$H3GYxGie3F<;R znKfo{L4D_EF=WdFZzsISrS?mFG{`~L>UW5c{I+p#23(LS;&9}H+xtY8rC`)#u|9nG z()v5B_?+E--C$3|A_>!zU|7!avkM?mElW*qw73H4Z*nsz{cka)swOtN7z}F$ou~7T zy3@?ddgSmh*d~7P<0JvmJe+fc6OSb5#1?9bDtH>5tV&MtQlQ~?#S9`JCEIg#$V?_= zegRq%2B@vIci8CUk8-!Bzxnzg3R&rY(FF)gM3uyiqMpL!f|!aVyUw^GbbO$g|B#)leVAxw9~Ack z+I8{z%5w|&O_of<*{o$dv*{ZNKB)}Jo=_CYOMxL>566`2nBV-*gIDfbGlQs(P(@?( z@~-ObSA2wo@%RwM4i4IT<3u~~8$gsb%l*&s1FUi!?F$dvD`Wz{6jHQ)9c$vV7xH_2 zO_bd70>gldYh3|78`*A}-(s>~A6tqU?cI27%t=M{PrP;;w!h8m9G;WxryP0O&N<~e z^U9eZXZohLkRQ)GN^<5W(li)aZQ>_tl&B?W1hv@2byj&$iXO7+itf@zCdZK4Zdg*< zNFA;6&@?rvT}uZqXNx0rD8SDq>?kh7j@GnOf-j=p08D}Us$2?CBVz+I=*HB90YF@> z{B|_)+nUG#ACTuTM69|kXGyl&(HkWi#e^i0_>^pIa3(0-+E`2pkA0P7?cI}LFct7f z6b@wZQM?zjNV4J@bm22QM;9lsJOn@XbJd3a!m!a3%G89rJ^a6f?;1yX9*0M0s9Z0| z+<~H7IT0$9vHdrS&9_j~r%TU*&9Fe^u?DZtBLf~tHRBdMYj$ld^i<_q;<5P;dAcL~Nj}ild6q^(&|rnQ(x#`%^v@B!ZBs zf0vtzGU%-j)KYPt;5z%*fo_dthZHN)dyXX*ZZ{Q!^ub}IeYHwxzt=o$_jI7NIvQ|U z@&Tv>;5GSmB-GO!5O00PUMr9ujli_ss&oYbj<0RQneOW_OKJ?aSR-9An!n_r)o}eo znyVySjzR$fTK2LE@dBK?aYasC-?`QpkD8AOBG!*A-H7^76u)j-3;vFXl`#O}{q$Ix z6jtOXc)k_ep6SU(r|5VItE8U2XFlR&@JA`sKRu^x()(fr5bj-TGjpO6|BfKKq0c3L z!*VN3F>ToQm6G99KV-k*qC$c!>1G&8LFNj~_+|XbQY~o|Qzbha&+V`9-XP>m`NE?z zaf(d+N`FFg`b=l4qR8ksW_-ZT&-gHg)%WZOmpna_ttu z=K`4x$Ixv(x;=OrMm#FGxnnoC$j)ef%x^P$_~Vig1xtd_j~&2S2$`(Cg*@%3-bT=@ z(n7G|oD@PvbPX^-G28>on!%0F!7r314hr2Q+u$OtqX5GJAs4S@NPd#4ez8$zd4`wb zMs3xgm459+=IM2d1z%jv**>wco%N|V4edub#xj0@%&;#@cJ;oDwEQg2spL=Z2s1w4 zPc)}yDFA1Bd}r5PNR{&ztiVBXa!dZ0F>ByHmc(BM;g)UU4Ms5E1Y4>-G2uS;kmjZI zK|~MJ>?{GUAb7@omdsZTv@ZVxFFU>q8O?U8U5veIoSM_%=eW@i{&%;;U^q{kGwiq` z9D^-b`1p?SJ>Ye=^`J0sIhAp$?>jI&Q&k7i?4+5L~UD_2Hbj``%yhgKpBZ42eTwphe; z@)$2hBixFU-!X5H==R8s-YvBr`4ta+Gbl%;4L6J0QH28oG$!%uG&gxBvK$uwMlQ_> zVz~m;t<;4BnVn%^eI$%6Mas+7`V=VuPNSii?<_5k842JqSFrkx8OlmLDIgrFN3mj& zYBsGAgQd?ABom<@%8|9DE>~k-cn_gA_(IiVEU)gcmVLy5w-_zn#3+dRbA04KSt@uJ z)|;aDXc}>oPN$AcN?cgrVI&AHUEo+sC02hFsmM{Ru17!IxjNe1!BSttH%HQ4K?5Uj zXIGZprT6NrrjkV_xh1;f42DO5B=gte4l-7X5ywy=3t;%Ur+%^hQv70pATTl$H?qbp z3sc4;nyEad5>{Juep!afLH!fcAnU9=#z$~6I3?tua2HqmxZcpWV<0)bwM{ zGZ>We;5@R@3_c~_zNsNmv}>wP6Ha%qv8Kmh_cN*)lgUz~77HRq!@e3DWrp=1+)>PO zQ#wSGFMjluI&kPq(1J_0;d{RN1bjbeN@TjNqt|{F{l7XYup0AnN#3#%&#}&6ZPrl} z((QL|%s@$R&_PE$kJ1Nym#^Fe_+VcVk3?nSJ!9r%fe)L1;h^^`J&7;bg(w9@_w_RZ zC!7owIY)9zlJx)GS~92l9#&-d6ZRus;q^dv-8p`jA(06g2&Y{p%UKgfPy*5{oVGU* zQBA`VO0JJy0X2`XYIt<^UaAS;=Z@tl@m}Ln6m}amoW#3Cxj$-+Tzu@2{NX_ zn4ZMd%UvRz00g=+WZc^@|OcLeg@|RF^L**3Jw@FWgj_`X{GsknBnuc^C9}KVPDg(`KbWi*> zq-ZXxBh<35D!}5l3q!K_g4*};jvR)VHvX~h{MPg$FA_9qO(>auiEdNpP*QNwx&QD) z7qd6fYgiC~@LYewYkcg(WeT|ovzRQpZ!HP{PkaS;L5ZDgVmfNS#tKEYW&vu(>5Z}^AK}5p;DHprKj}KOq|7eo^6{}?E@Ok2zY#{IbREJ)ujCQ zXz8JN`#1vR7~tCTGfY2vXLLzB(?RMcz7&+Q@S3}6Fjan3`3V|~d^j`wAkC>>>FAe3 zI8b0w$I{<7)u3aM20E&{-lcjARvlkM=*T-kR*;~~H{f|=iLj5TlCR4blX+0l!_eOF zpoYVrUp|ufvrSC5lb=7u8W5VvpI6ty3x5A$M>uKX>69=a!R7OGz2R&>@o83b@o#87*?$%2v0EqNZL(WqVV=% zt1MHElR|Fs+d zYa%!F2YOkp32oK=C`$CVw)7V`C=3ze%`i%H7LcBizY4i*!Q0Q8@C?RrGd}I$JJ%)7 zO_ac$e_2yAI2r_(+O$L(fNe&+5n6`b0*94pjD{W`j0Srfc`9Jt6oJeSZs@HCWN6*i zt(3r0l@zByTG04A4k0OkXlcM({w2l-yo29f?ZPir?gcR{llFbn)1Qrir3%!Sl!4G_ zBX1cn|IdIE(rENrCnm} zX7|?@l@tJbnA#*dZS(ZiZXOHUNRG3AS}DBybD}lavzN`YOhFfQ{K?>q*_rHiq-18& zBN^ahkxNmuJp+VwO^a89)UmkD3M8~0+#H1*;T!zHW**Q68`tys=b3{#7fxVs4b@9)e<)zMAaIvE*lU4jY zz>JQ)`HjsR`rlZ6l`EgiCJiVrmTRx?;=qQ3)h8pMN5e=k~A5T@&W>e4{ZWR+tT{|;gt{pRk-nD*$WyEqJndd@|U6N2PZ5d-- zzJMxU=YU|~dS-?lg8)@eh1{l()_GdR4Q&rlfYTy_y3j{f5O;{ZwCJzz7(sVEm3Zh>X+bT6Tcw6!E zOg8+YG3uL!B!etXL5rO4RFWgy<2V|~^xusC9Hhi!^YtZrYDCt-dw^*Qm3-U_rQ6Y4J*+s0m{{Y?pF8X0Jv`Rd~2z!edE!pg{_z-oHE zcn-2pNGbJX&E@Y6DPlKPF-KQ;b(-nImZqU%r$N<4U%Jl2RQ`uO?MzgY15s(?r-r z+vQC=C8P~0hGS7;k9-b5JouxpQ!RhQ80ewhr^^_k8HD4VQ0<`_w2=Yf0_yfAD#O19 z9q%2_6$Z_I`jJ+)zxDu1lm$+8*<(Oz2ZMc#8)GO6^#vO8j!V|{tul%GD$Q=R<-3{~ zK7GJk-zJXKx4|ziqwkZ`&LFKvE$C(46RK0lGm`T$_bV3b4+H?o-&oRf-g=RJGyp;2 zCwH(gLGPFbs4@I&ov&OH7fm`J2wGrn8SB8>vU8d`@ezkV8k@X2QcsKShBPx=iLBAKpkPtF@fHKA>{2R)0Mo#s8s=^iJH=lOXo zB0z*V`(5$9p~EVdCmD<2khc9zxX9rcR?WZ1KfxMRR|8GOAIW80xFCwR2-KWKi-zJ! z-ZR;+bbiIm6#OYw$(f30g=5=eB)ma0Ay&Xg-UTu~fl2`>WL=ed@2~PSTmd)F@!xtFMWo2hF3C)?S(YvTSp+pB ze^1ZY+qN-ZK4DT<-{1c!2u12hR)t-@d^DP4br0jV3EoX7CDX{D2mDxGeRW2Ndto|VpZL6XL-Ec$MBlGkzftwbga-l^%cmD=HIPW#OGj{i|3onT-gGEG;4t8TKScuN{N?%kep9KJCC;|cgGiSZbIH!ObFbcN-C-q6s>ph$G^c; zq;le{38Gz#t_^vI4HN*L*LvI|DwD7K|3N6-EZhP|4h;yceNLc(Y)#t_uLgWQn>XxT z^6?~OKi*XIAdJyj0xPX(?i_KKJ2SeKsf8WAS+f z!k^Lq@L;nx*ayRV&3G_)Xk+N5TZm;xpUZOP!U|x}F~y`xLNF0vxe(un`*sUy28O<; zD~M&O|60bNg9_?=1as$PM0%($Hp$Fh+T~m-S$RH@)+NE~iKL1Og6R4#M*HhH?s}(> zcLKrS@ggka$-DwIGL3BaqH%v|M!=@clbTGLx{u5o$oTc5-<}}+k$|p%V|}O9RCT%F z)g`OSp_JQUuS|JR3Pf2OIX>Qiuim6K{lgz+RkFP zf$FLiaaPS5jSD)Jmg#w|V-p&xX!iOzzMPo!ad=e?lYvDQLyeo%m7;^rLutp{;nIym z0|if3g}_Z835lAU8vueqj1njBy-SMEbZ&@}0@&ZXT^PY3jg%^`R752~bf;0lJN0kX z*tAleFyL{9yIV7TzppiYpIKAOt6gcH!%Tr6DZ?YLh<2aK{(`Y4+7TT1cy_3r>EYE6XfRV1h#4^B zc2P@wKSiAvxUnqSo-!MoxB_s;k7zV_ZPzdPCce>qV0f3U41U{aa>8J^+Ii)JC(Lq} z%?2>Ads)&al<<(`P%TG-mFLdq$#vN+_sywUzslsI11C?~t9OGFc6FXf#67ZRnmN_K z4DQ+ncEx(37{q}hn4h)lR%(E2@sq5`JRY;}8;R8I9{}@<3fxU-RXma_1{wnIz&OMT)KPWpv^SM@ z8mT6RA50H$%H^tNqVveW9js|Fn&m97#1Kx(rpY}~y9m8(XM`_OtK8jwu%bI$AfOp3 z-&f=u-mM2!Uo25SDcjLSIEDA(v`k)su5?ImbpFwAP4n{01s%&*@f_69vWpGr&@7|; zbNLPFImaj6D>}X%Al-tm+z8}q_&mJW-S}cyjXi=Vgl5o&{VEg&O$&ryvb_OH@@FQU zDk=}iPZY~CW z%%SwMik6Na=ml=3J3lRjFyjSblB9(U3dj%bXjhip+AzKBQY&OWLp>wWDb`X~lC*#yIlQP%s3M9WoKc3==DDQE=^u!G?lcqW&!(-?WB zXBYPmRH+F|56Ltc{C2^|H1b+vtmQV1XT2AV|CECk_CAj(!hb!s_=%||lpV42w`)m$ z?ww_ekQbG?gBta}2dZ@g31guJ<%esJI7!T#52{)s{Y8-72#WGTp{r|HWALGM-24@{ zY?|Z)V>FdcA{fKdUU~Ms6}qpm!IB7k3QkpjV>bu;AnEh;&=MVwE?mGYs4bW@Z1YbE z>O)x35X48YGb!>Dh?t9nR<(J%xWesJn`48$*+dr756UtHb++u(UUE&}uMchRa~{Y7 z(5SxPdE+-zA^;=6q{y=kfqw=2hRl4a0%C2j4w!UnLAk9|uLdsw&5gMb)GyC9_-hW4 ztfUs4PV4}5XViNZp0X_CnVVMfoPj|&3fW(`2Z1JoHDjdjoZNCjBNg`(2H#`79bbl3IvjBu+e_6=KMPo1t~ zOd!2NnzJJ6b3k~Y;`nF)Uk=+6#n$ZeJ-Qa?)esCdX?D!{77*7w2PVYoJ`tM}o}Cr0 z@B&mj?7XuOROQ!g^z8T4`mf|L0kuJx^eICnEk1<-3sl=!dT87Cv`8eX_f z|6B@}1*l|dy^&EWbqdf@yANdcRK~>_%=HTYpsWQ0Kn{=CE#Z3)h(T9k9!MtvD0d=h z7_(#;0a9$HYXsqNpX1yC`t}j{S}4gcy;@8Fz^%e!xqg`)iY||xX}Zr<7*x(5vc@Bz z3=UeQTxHEFd+f_tNFwCyfC-*jY3_cQVHylWe^Swit#%(*>;@q<7zJ|@+D8JqLCMg0 z(T&yH>&&mFh=*1P+KyidH-BUYj(`SR6RMVa(m4XXQ#8K>JgamS42u@aTOQZ*i`ytw zQh?WWfi4MKMg-ZTk2{!1q^7HoO~KAQ2;vdY3y^Pma^2Q%h`rQ2v#w2H^CJ}JT$|J6 zrH{AJZ1#Z{?_161TnS;Li>6MJp0N;@7J?87>X5h0rLlsIwamoiM20HY);zi<<*pA@VH8qdh7>B_Jj%iE6m=qq2v+>vpCmm z+B8dmf3#BY2Bw1EPq##>(hBJFN{;2jQmVo}$K#w8A= z!?Maccbb60d)ncl5C~Q|WbF0B`Wa#KG-nBLQ!WAR1)@VUMTtpu9xjiS6s0K%7Sprb zb$q7i>0&LVE{Dmi@l~^L2Sk0bPFxHS)dOZ+dL@xbsrpAd*a(Fw7eZWpBqxDf%Hp52 z@~j(XHsr!Nb#Q|zxsuk>OGaMpa%)`Ou{U3knQ0iBsL_(^GghJnKZWikpq5|p4pt%sv>;$Zr(GM8 zJgtkLhp>=O-0M#@|O>6(>&6IF@NBP&kZ`N*FST$kGJX$x|~P63fHZqIQ1p7XGu@*!1awiHIte=)2Z zG@pt6lUnd%F2$#o@2Wf8t9Ut16m2&&1-Ha+y!P)Nsrbc%=tefWDAM1uQoL!Uk5)3w zAi@Oc2F(FNdht*`NcH+Y?vxCP0x6TUciW_NZ{V_`Ufe6mNv|W&99<%784(E?rO7l zSa||F896UwwaCGn>`j$VeHQ_TCYRNHeZAxmECsmeJrpb`lhL`^9sku{xU-DS=e{LI z-B$0EdJtpG{xS(>`kTh$=+XGUcTe#}RhHS9dtj&$T#Yy|m@ z<@NqOZY~oxb-cdSGBXDELY-k2l))Sbyd*C*Co*YfxB)0SG+>QDjmR8St}+Cak){n* zbGdPq6RFu(bm(peh6Qt+%Fqr)d>Krax9PVVwfawSD9{50LeH9=Acb9;xXMc~+K7+? zZfHw?K1k3?&Qe;9^6duSJ$VxUUP1-cFg1^!8a9h^f3OQ>iM#X6EU?& zy(%1z*!dQ8hZehJcvyY(=fZ6{_n5V@m{bDxX=`YDzo_kJVic*A+gi#=eN41Y zg5Nl46F5Rd0pVRK24i%lw8U=}5(N`-_zds>V9)$Y%`;QV=t?5Qc_J zYn(qz0cSf1QAgY>R3|eTnzg=&Utr|83OW{9vYmukT zgq81_h!ZOJ(WUV5EW61#Tztv2yDk4xk-0)u(b?9~M3MH6Y5tPZ3|i7JGg!TLP#8+c z-p~Qm%+Oepq`5l61g&atOpVMfa(7#rj;>7+#|8h4RQzJZ+O08IOkdS^Hr-u7pt(!? zhh<@XvP3O~o4+)RF$Pbrda7Z-fJz{BA|m1Z+8squfG&Qd3vLaBY%{KQ1n&$+j*kn; z;$I_Tv8Njcc*Xy!kees|70g!e~IpHY(=b^=F+%VXeRMHng$~VubLeq21UE;`)U@hemBcbeh9Iqn&bCz*63f zN?&h&6$y1c>I9>-K}f~j`Odv!Oy@Uw&TvmQ6?bTeq#fFc>!pg`uUOrc9*c^rB~F3x zXfmSGPUX$Uv=6ILlGCB(kIY#8p6$m29Y_z}CGC`bW8j8Z@^Ti(KZdlQGNwafk=!JA7Bcqt)97oRY_B@*KD$eIhqZV?Wsi++Y&DSY$o(yh6=cRiVWG zgyFFhs(LbLR~&lBn=dH^PvPKUnK z3^E3d)GcAXwC@Q#(&CsWIkbwkT_v;guM$QKGw%X$x=#M-vpG;i6QW0TY?kDh&Ik)& zy7koQl{)2t=EUmo7zbyHgizG-rizS&05K8^Qnp_Bh_gB)2V)<&jfDMY$2*%Q`74HD-AH+xArO#p zf~xnV299X9sq(=vSlkrvGob6@4<(SRzeetrWVUdWeGV0{LJRj_1u zE{7;|n2{Ts-lq&lTdM58_9|F@L7KXkyf|=qSe*;Ko!%uk7SSujpfx57k&w$ol&E!V z(8x-{Y(UsDOg(UN|M#yxvYY)(^0Ucqy*RIBT1X@loAcpG0(9vJRIXLjxW&GE; zHt-eeso-z&_Qe{PxOutC3(_B-#Y0b*amIUW8SR;FGyHE;iakt&u+;%~@w30QvOjC* zcvrC(7_gyJ7tc3-wW_DVxj@>m-*DYw8)zQW(Zs7Ppnr<*UW4Ls&oL=lbmiM!eDM0N zqj!*H_YhiaMz@bdm~25Lxk)22yX1lXDfLQtx@XO)`AaKiKFFl;2nZ(eO^^}#cfXPB z$U2*|8cOVggz;y7!f?Fz-L<;MfbQkHU0c?Cu+y*kcDBQ8-8 z*y#?vkr`Ky3fkfMJ@|$k#r}C&-yj&plbW<%N#OiqtEiitp0Mz>aBev zikEbTi-ByFnH2WlMArJf_$iG_>2(7E9?_!nf?{->y0iOY*Wk-i=MYXQvZMc_ZluF2 zXjj{vQ7+jMnQN_dG8zKM~7?0Qc2I`pEg(v`^jl4m!lI z6`B;{88PC~@!Wh~kI25syLSr|Sl@9AijbTFBb*Ws8b{9$Rz+fe&n-O-w;qHf=bsrc zJOUz+-`Fg%9c5cJQ$6_C{_-fbt{wz22j6tpl6fGt)&2%nKKNA^q@W7Ukdbq z0BiXMubyiQzQP<{^u%6v#F&S)wSVa~`xC{2bcdq>6rLi%fWJ+sHB!=Yh;>K+D0O2dcpfb!IsI`L-&CTgXTDS->NFU3 zv)xPpaFRly4jxjV0c|c~vO2N16#R+ldtl1v`Pm=n@~mCb<%H)rz${m?SC@~DdeT;~fAQCQx|`lA zXA@NoX%9e(77zA&J1~G5ZAssAB(pd_CkB6IIcb|&>bAZ5m6}7osiZhPJa=|oHRwPQ zlO+qT7lHS(rPoi(6?xT2BKrAQP`%IgNidqw067dQUR2Gs6X;oMCg#CIzP7Yt2)8ON z-0Ip`eNUtO9IZWYWpTTjkQ34m`DK9rz@`PZdu!UVk$#h8PWIixgFjmOBToFN(}nUe z_y^~_{2Jt3G~D_n%SS76J&yijqzPQ^AA#{+VHq#e>zu>xa)~50tKa?>;9_g5jV>GL zx%mr6qxx!2MfVvd0$-od$ayOhmz7QWm`A~sXhw&$yo|M@m#Rm!43EK4tw~+y+R{-8 z?rPuIePO9MEW(hJ2PCn9l~<;*_nkc_%p1$qZNALRqzfJiKwW#EF1lB$ zY(!f*6T33%j09Fy+83*u$CjQ>XdmoI-t+W8JT3bN^E8G?niIHXz5lJo%m zV=mA162q*pVHABu(HMN7tbp3E8~xk@hkc6DADCkI z)%4hUH`I%VJ8{}B8xx*Ww4W&-JS?(^-b_*4cI{aIl)cRBqed^-)+1>HeS(Ey=8=5_6LH?~R+D*uwNA8qDy!YCUd|FGbU&u-%y}{uOPp)^ z&!HPi?QRm~P=b%DWy4+tLHcxnDo-tu1ge4ykt88CwhdO#-FfKmf9JAOmY_!UV{wk^ zRj|(91aSl2XPARLg zF=fcaew(yCJf^;u9L1_;>C+DO$8|!*M6~)ndM7ExhaPfmY3t-wLI$2q<8+}#*IDJ! zN({`SIa-=Ts+Jtcjn06{{3E4H7tV4XM%75yQH)%&7NnwUH@7yxf6-!p%ujAkbb#76 zp^f|k^Y-$5^0xJ;aT94JW(Vw^AD8+;;?dDVR9}bMSpS;q77Gk)WcA#t)oSz{`6s$R z%#n|7kBD@piL2;MXBLsHm@jIMG9-d?+sLiCUS1d2OsIiBk8z^LPneP>q47*EbC~Dj zw+C>yH$V?&zJ)nzF6i%VjP0-$cF($ZSX`v^NiT_Q6-vI0v{lO_I*1iYp}}0vC2x^A z*74sKvNoD7(>Jp=5g}e?QQzwHK%A4iUkNIh1>a)dg~w4s!?*$K4?T!~o#lppI>`k*-xolbfTO9gvFeBdZM& zpDC(@WHKYDf^9z<*cVWf4FAa5q8Su`Xn23a2GO(BcUnc=YZrcO&IHaP4Z5s`Bz@^Sy2IvVN6{6DRchW$ExkfP)tX?Ae^W^(JOS$K$21&d<~P~_`ju-b1*9u!#AWVF{@T#?Vhk58tqNlm8v;bDqz(u zK(4fQyPypZ$jGtO7bkZ3E9g`o3Q}R~DU9_-Y45@~)pVb|67+#v@(uI`0{rGwx6=oC zjA)%w?N0@{Vqld4M!2jY56KKN7n}%HE2$^t{A8tToVjk5t( zMuc@DeMMHoM6QMq5Q-x*b;yg>p;J=(P1}tx5s(!Bx6{p)dxMWZXvT%!`~V9Hw9dCfz^umfp@_YC*OLJgxE8CjX{CR?%z4K$#y+K>!&sV-mqP%ko9@RC0whTI1 z{8*)kCSO)bs(4M66qEN$k=z5>1{QDL0(vBB zKYZPTQmmXYNLkD4H*vW>q`SJeVM6*K@+qVbcG9_x34@xMur~j|PYHE4Grc}UrbT?` zSq#=6FCqu9r<$#Zn%SYnT{s!f8!~>qJWgE1|6eDgbD0IEh=r#1>eu>km3U)@U$Z)5 zjzgUunhNVYcp$e{NToUO&+@t~e&HU`Cq$i`BSE~;(Me3?tTTFGpTj#2io~))NRb_F zro{G}>26%?o5?lixW75Ws6_1#r6h1M)kpY8D{=&slv50!PBfj_F0H4W97T&Si}y)h^rc$UzU1~447qG0@<9RyA+6&yB$!Fzp#Xnq^Onc$l_4As-{UwD zJIOG^(`q*4##^3Ps#>F2L2s}XeFUNTmsLF6=-nk3chS=H5t}|her=teN)9*L1wz`j z`<72Dc#5NtD3*CVYz!&FGIgcA?*C*jKN)g{CS9hF&wN(~_`#kDKIQVNKbisC{q7B2 z6h$fz)EAgapgr`;A(^sgHNy$ct@cwO3uEZq8{n=n?OvV|BVL+zP^DX!Pl#Ct2lx-D zx8CeqBHiXH_7&|Ur{=nOGHEd+^Qlf8o;UpAk<>lF{{bU8nzB@vl|8(#b_5jxf9(k= zv=R=oZ51%W{9aed8lRsOcnfm;j}GlHx&`Qdnbz7?tAx~v<-tjDETY0Qr4!EEk7(Xn zF%RdA9L~nmUn=$&6X)nObOoMRzLvZ41z!mkMn$fBWStd2_%i)9C@3paoD*Y^_jgrZ zu(p2;iOoorpy~6VKP%+r^bvqPVjc*-Cw2HRqNNhF5zOl2Tl-*uBXEu(?iAMBMGZxb zd6Y(?;msWM0UCSw;vO8dhhIA0X`Pti{JED9xX*dpVGX2; zvc!3AWszPz6*&#@83sY~+~C@8&419}>Pi}hqm@D;LR7v|suzs{yFX=xR;z#urvWE} z#{_bq*8*v)^cpINXEMn^v8{)8hDY}*&M+wL0GRAO=Q4rB2R<3{OI->Ey#pFk4qmVI zkG4#J_6AVT^@Qaz#If2jdcx7c6*hCn#NMs>O)b-bC*JZn4exf%jL2XuJd%Cx+Rk!x+B*KLa z0bhqnMbttgNm%D0TF}6{e;~^1H}j2ZXIfvYce1^(8x* z{Dm9U*aI{|xy=a?zNAhP5RCqCLuY^_PC|r*oRhF8(^T|P&^QND8Rp77n{xwa!lc2UBnaPZOg^jm@|>_fmbUnI-pAV~r2c+&Ay*yN;I z$Z6xJfz#&HpID$4-U4fb7+(D8=^9{@Rj$S_3|R-*+J<@ZDNh}M(m#C2QX{Jp=<(5? zEl|rA_oGm`Jd$wAvU+haTi)`fwNwFVp@z0=24{GGb|S>!fu67`Fk#D-rc1bd8@zzk zfPRHd;$EAwNT%R#UPGZFZNduQc_j@4+ojte5CmzfZyjHVl6P3^aB-@Jv`?+yOV=TA zT%=F($ff+i7_QLdBt*AU&joh(GTc)=Jj*@g<+;3xm9^`7eeDYjq!9P8eVV~+)e~yY zFL)%fXB^~Kxl?UlR=&USjoW5|#x));v?nAtqe}_pr)=cTy?z9&%!m+QO^LmUZo$#h z>A?!Z{;1IZ)QI@I9GpVmzR-HBMG%s*3UK9uQxS-ho~#4K9B#>$22UXoFKJHpH6)^p zeE)o%(p*zySFW`}r%FKV3Mpd=j^%yA`EI!3pTmz}msIocYfOQf4PPh6)Z^uwiTEie(;9H*CE$sq(V$_aRs~(eR#_G{b@F6QTo68 zDh1~jzvDuMV0yU9EYP=U#)hb5T`P|KZ?5LP6mMX0qOZdcYH3`FiZ=`IG0X|3QuOF{tnda=NZ?5BNcZ z0vr7x1<`wHf#>;l6#}qOd8ZrsRU^%5--yJNOUd8RI_GTJXCf{3E!!zDiX3_(CP~^> zcTJa#n$C^NN`UF5q>~xdKb3hLSh=oe--}M`#`MnGqH@^Xu6`;j;G-tHhmFZ}1OU#* z;4RI(p<+j!6e2S9hV@9T{r+B%>Ea^7nYXTHZl1j|iiMBX8ru5J{RCscaMbt(IQRHQ z#t3RKAru?E7l~P09O=gyXW6T-ePj%ps0QVGE&PxWdIChwYU^%!M%7(SpqN&2na_d| z%rQL640maY1a8t6ZyB1bcAqCs9kYlAlF@CtfexgK176JjcX};ac|e$Yu89N~*gt6W z3yLEGon)KoPRxa|13+b=Q&i>l19zrmW~fNtX)r4$U^>4$bH`GmFhKx6SKDw7Km@q{ z2ju%UBPhs~P`hfj7Z^VL?S^|Vx5Be^FTcN*D@IK>fxUgk;oSM}YCvWW5{(Wof{&$H zY`2aoD^p2CTR^d$hF(=xH@T1_ww)QAx{b}0tn?WUnXEOWv zH+J*XrZQj=ifa-0FY<8)WOIYiyCa>i>bl99F}Ar7s6jvm1|(TiSpJKkeRI|YeD#(8+8$78l{*^Vsz4szI~(k z8lQ`brrOBPkh30je1!bTW7k(9tlHE#0;_iIIxWh)%bS^MoG=>LC#M&IxrljF-8lP+ zP{z#PlqBpdYDw3DLw_U55xisyv}W*-IEZ-Yf~rCAZETNr7?*I0!h>?$)LFb}5ckB6 zGJc5i1w9-`%A8e@7j!nPI{TyNzEqAQN4wMoKIEnBo4_f)V~% zrxir+1;UTD%zZw&NLJ%CUW?x^OFP%@%n~oD^Z8Xnm_bs9se)x%hAM@zq1CTvJOr=X zj;ac4@y$>M{=RUg(S8RypUsT%q!S5!;N0fa@4ue*jdo=3_%eCS_H(h8hq7Dh5v@oj-b9vBLdc zq?7the4bdPOcU_QWHwV)l9P#@sEcS*;Vr z)a1K*Rrb&gagl7>It&Lt-0bqnll2>Ls?0=uG;Tl9>bkzb+RPHsACjg~iqTtF+DcWi zfnU)h<$~`e&23VAQWrL$L^feTM+$}OVh*c+wZqf428%%ec0)=h3PkVt%S8sVN?n~% z=!b^DJp?=G?mstJ&ECpp|9PTg^3LY-ezMOj99c;UQbvt%%#KDhzX3Jm*8|@KKsvXkr-4q=e>Y57-YAI+LVt?*1 zeWTqqb}Tw7(RR;y&|Ao`7{D7I@hz8HK!4U^_RIe?g)2zmvLy=;{e0Tk(^kJw(U>-{ zT#NqHMv;6#u6$Jm1i&!fs$>2nHn8KowD!(K>!TrgJ9=;9x1&y7Yk<8E(X-$SR^I~7 zSe=66^oMwZ+e)_w!xuz`qF%((HO&EkBkvq{RJY9>cw2IUMJ&^i5X!~>IgAS8l1o5t*hGVS5tJG7VZYhoGhJdFoU09bF1&Ajow}xBrR{* zFwx&~M|Pn>LrfrD$NJJuzk-S9QGKh*fCZ2DE9(9hm_Cw@ZC7VScYJoPU?)WNWqXLt z9#TTWr$ZJ-$rdJJ9R@!_0t&{Jq41hDGf~g2xH}WMF^*iQ9u-|U0g!igzXK3>Oq%_v z5(02R(owXLXiM~+6+Us7CBxpMxAJ(MmesTBWLx5|%e9*NYXzj`tP-KVsA)@Vv-&mC zx!xh@I0z?PI~fAaAB2M$?VNJmwQ!-`crRkydaLQ%t=EG=*q`R=DcXLgZSF)Wl`ef; z_kO&);DW2%yg4m`F5%#dZyjprtK%`IQe45cx-W|e&ZE~@{cSY720^2>INV}_N~+pZ zkdH+<<3?_=NObV=(ealT>duB`gyKr8=%{&qeADueWM%X6CR*57m_--#qi%&zUQqn zQ=SsctsB6=bMG1^Fne}ANV|e(0tUnU+;fJcUte&Nj^F;*$CTjHU z4ZID~?NWuXj`F&0Ck8nMhL_f9GM<)AHD!f@n+n{+d;w!Xm@{bBtfaz0>x>8=@ukZ# z?e%|^Zxg0fvN>dM2rRHu4K!EbsAN~i0cdlwFt!=2!lU%RLxV277zraWtb`b7C5(X- z^cf{?0)^GPsV7+ z?kM`5(#j=NPf@3wwJ)Q(3vGdo)T@XIY61DsZk=(-RoE;8J zCSU}Wq=$*F*qY?@%}lPq8YTd==nri!!2c|t*^K|Yk>M|4L|BjomKKzXryucN%(|s={mZG0 zY>Je;knxPhb2wI+uwMb~hJ zryMj{Uy0Ce*eU~KzkmSP>i#E*x4&!rAauvV@D7*0ZIf|k1<<{Y~Ad_RTU)W^j ze?CMOB^vtu<-L3~*aA12e+y?XaO?`CQVcn#^U7XWZbH?NHd}$jVzl9!K|BgH8t`LS z{!yX8v-AjQ`eUjKtiL4LnGon##+|T=qnHgKLJTs-+xGX^F`UyeXb)~Ph#vjIFHy@W zNE{!)*uW12d!p9zc8AyQyv`sAvd6_Ii#X3%ibDX?ZUe=@_J8ildX$GQn?bT&L^B?B z!0p%z;yRmHjXc#EUL~@q2he~Bz-q%ARenFP@-{NA-lh%EF! z&NjJu1g`7KGbYSr*}%Br4*W5e5N6zwR12(5(k#RKo&4TcV^XIH=hx@-@uImO2Y!?p zGyBu4OPo99`-Y29^gq!i`y94`HVDh9j%cBk1EtD<>+DCteUS-+y0Ubh$R?5( zSDmu$Mp5Ib>pAqE9HkBJH}Jq^pU0=?!%Mz$m&(cM&~z>5*UBRQ)Pd+6%|iscfrhqs zor&Btz6s96o%;Ak>YLHoMHMt4>Un&hbX47dT+_;h)|zo!r#U4v4r}xOWq+K(|8gV@ zOSw_^#2@KM!sKYwM!2+dgY8i%!+rdaMN>@3$iVlm@+B-Rs9LR<<~^x}@ZpKJ+)PIT zIdbA0l*ycGp$bh7luduX{gT;^Bk0lps9)3nVH5Y#nc<-7RLgtcZHJde3|7WJWrBMo z@3Uf8^!O)MqWz@@Ry&3If;Uy!|5XF~ry?=qRs8fcqU~hZ!NYuLgZPEvEHD$HC zq;_1)(Wf}@2UB_t&Mr`g0uvTZ2|>j$e|aDg+Xv+~qcNkxjlFxqe?Vp_3*YfQUzbg3 z6`De?Ts(?c5v-xC)&Fw?Whg>LfVDVucY+L3G9AnFHwnfq8DZvWIGwHDv=ebdtOPb@ zlPN4N%r(EBY%6IYojxHjb#1$zbFNT0GUzmG3B4M#46L!`;=Z=|Plv2S+--`|$Aq3& z#^C5nDMyFP6n^qXo&ae-&S1TZe>EMPikcqRViJSbmO0#$*l44ZvF@&r29>ODw$=>n z8^U3^;P(ZUwJy9pi2Erb?dGq0vxV(6R)m4$EmFSUVPi#PLaiPFY#cBc)6SGoq#8MQ zjkTgfxHT3Z5VN6*oo_z1yZiJTEh4l{rAwWzFUt-x?+udkLiJQiV>d{1!H zm>c~@BzoDh0EgugBIjD^a=I+nPhgjjkHCVL==$;!gtG)mrSk5CFXNjo3Df@O40D+2 zb}qm^KcxNq>15uik@N;p{VGJ10&=f52&#D#FT(njn(3wQA!S0JX1g!NBVyo{CW-`< z;2s~B$cHJlx?*ckGTf5nzw@$m$QxcO$lrgsc5IJ|=K)=C0RkHl3z{8b?ry_-K~?}|Ei^7`)-_4%i`Ry0jXdw^0R7w5%9x7s2m|O z>(hc_Hmnl!45E}MV!}f|L0n%rB|YXG>K90$w?3$o&>ci!WZrr&rPm?}Zl@p{q@ii_ zNmr>N^*I5c2`4?Lexz#inDpnY1;#m&JBsA1@VJr5$Es6wRU2b~Y};`-2oFb|wlbXv zq?ZA+KbUJrgZ}#XUMqZ-9b}1QNr>(#5nTtxNVXlUxr#I2i}gaxsj@rNjvJz261%`9 z9T%EYd*((tLKO+Ip^&a{4NEx7-9bBEFIz`)C!fWwg;F2V(f}GnvMD+7`V#?#1?lzs zS}$RnWfz6!H=$iIa|_S4r!N)es!1hEEOxxm7(#UInGi3FvnGHq#(E8?EF)$6ik?lj zLmm)z?|)lDXtu7OW`fXQB_(z*Vj$$T;C_0nL;_#G;xw;QYTmsG*HUM<{s5;` zu=J>aMIt`~2k~Gq@6$CkP$iq-Lwrn-^K`_X98XjnV)!)^kzcCkTSjagrdK-Xa&kyT z;vTgb;P>=G`JXx95RVgh}TJ4!HhEmdru41;RU zfZ{*WA%3Q594Lb7P%Tyy3eWIliCYQe@*~w*$VqMls?qpdku~fLv#_<@Eir=nwDAarykS5tas28wyB4 zrH-SDQTEhIh-|5PUzmCaGID*XKBbE&!mm@~MHBDRW)($lqo^EU?ONr{uKlTX&8o@{ zEXi3G_De2QDDFmC9aMy5++bpgvJ6BBcZ2mn_%nSJV31S?W-p4cQkCz9737sDjN(+b zW2Y4#T4Glh@c;Z_te(6pBgQd8Kya5R*aHmqO&_?MQ=wcRJm!RzJFot1^AZw7>?QT8 z=A_~2RR~Y!X^U;Xm2(Rc$SC|IorHB1YIsM|h$#YHa8rcZ2)uBKl_`=LbGhJwppXuUx z=F_Iyjq#`H&uBRFo$Yn*`$TKD8=T%nK{!xJGWHWfp_CiGQI>3;F_uD3cdWcI@@aFX z5SW6o>!vgGMQRn)-Bt^6p`JFv&Rf22hcH#^vTGcxAEJB7`1Wxpb(f1e!Vu!oAYKAH z8w;>2j4NC#ImAHndQ&Bt(=7yL@Gw;WPUs4OvB z5_6rC_I5}LO|0|5=0%wa>t|7Ky+AgjL73xDzQA(!c`z5SHx(402@1lXGRcCP!fVWY z`c~$d>{9Ev;5}b;rY#a^Lu;iDqWAFY6f+GcGFH{^V((orHR@4UxUF1k0%_o?#u!Hu z`+fAwX2zCzO9pGT&9K&YdgLHl7Em9mw5H1@=oQU?7-BvXZ0O>EysiqEJA~!Z^DPzy z1W>?X`)#LR0DEKAsW{&CCJzNexmPX1)b4LFMKg~~s`uxTn+ku@QgTxV_YorZnr1OU qL|QQUijrfivFYD7&=4(-Z`0?q_`0)WGxC;aT literal 0 HcmV?d00001 From 3d0449fa93624e95e22da035ff6019abca7f3aa8 Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 18:13:32 -0600 Subject: [PATCH 205/214] updateStats: barrier the roundoff reduction by hardware, not by size The LDS reduction in updateStats guards its barriers with if (num_threads > WAVEFRONT) bar(); which is the assumption 0e67b41 removed from bar(WG) and barsync(): that a group no wider than a wavefront advances in lock-step and so needs neither an execution barrier nor a fence. True on AMD GCN; not true on nVidia Volta and later; not true at all on an Intel GPU or a CPU device. With G_W == 64 and a 32-lane wavefront the loop runs at num_threads = 64, 32 and 16, so two of its three steps write and read LDS with no barrier and no fence between them. Threads then miss values written by the upper half and the reduction returns something smaller than the real maximum. Use bar(num_threads), which now decides that by what the hardware guarantees: AMD returns immediately exactly as before, sm_60 and later reconverge the warp and fence LDS, anything else takes a real barrier. num_threads is a compile-time workgroup size (G_W and friends), so every thread makes the same number of passes and reaches both calls. Measured, -fft 256:2:256 -block 200 -iters 1200: POCL CPU device master ROEmax 0.117, ROEavg 0.091 patched ROEmax 0.127, ROEavg 0.095 Intel iGPU master ROEmax 0.127, ROEavg 0.094 patched ROEmax 0.127, ROEavg 0.095 0.127 is the true figure: the iGPU reaches it either way, and the patched CPU run now agrees with it, while master on the CPU device loses the maximum entirely. Residues are identical throughout (0d42972a9d02005c at 400, b39c99fbd03b3454 at 1200) -- only the statistic was wrong. That statistic is not only cosmetic: the comments in fftbpw.h say the bits-per-word tables were computed "by targeting maxROE of ~0.35 over 1000 iterations", and one entry records lowering a limit after a run failed at ROEmax 0.294. A maximum that reads low makes an FFT size look safer than it is. Co-Authored-By: Claude Opus 5 (1M context) --- src/cl/carryutil.cl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/cl/carryutil.cl b/src/cl/carryutil.cl index 9d8dda6b..c9c88a76 100644 --- a/src/cl/carryutil.cl +++ b/src/cl/carryutil.cl @@ -145,9 +145,14 @@ void updateStats(local u32 *lds, u32 num_threads, u32 num_blocks, global uint *b // (see https://github.com/mahmoudmaftah/MaxReduction-Cuda/blob/main/code/reduction_benchmarks.cu) while (num_threads > 8) { // Write roundMax for high half of threads to local memory. Ignore threads not participating in the reduction. - if (num_threads > WAVEFRONT) bar(); + // bar(num_threads) rather than a hand-rolled "only if it is wider than a wavefront": that test assumes a + // wavefront advances in lock-step, which holds on AMD but not on nVidia Volta and later, and nowhere else + // at all. bar() decides that by what the hardware guarantees, and with G_W == 64 and a 32-lane wavefront + // two of the three reduction steps here were running with no barrier and no fence. num_threads is a + // compile-time workgroup size, so every thread makes the same number of passes and reaches both calls. + bar(num_threads); if (me >= num_threads / 2 && me < num_threads) lds[me - num_threads / 2] = u32RoundMax; - if (num_threads > WAVEFRONT) bar(); + bar(num_threads); // Low half of threads do a max if (me < num_threads / 2) { u32 highHalfMax = lds[me]; From 9d8f3935ccc274c8690dc8db706bf2361a39730b Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 18:24:46 -0600 Subject: [PATCH 206/214] LDS sharing: fix the region and semaphore arithmetic, and refuse what cannot work Four problems in the LDSMUL > 1 path, all reachable only by opting in, and none of them reported by anything today. Every change here is the identity when SBMUL == 1, which is every configuration that does not opt in -- checked by enumerating 180 combinations of numWG, SHUFL_BYTES, WG, RADIX and LDSPAD. 1. SBMUL need not divide numWG. The workgroups are partitioned into groups of SBMUL sharing one region and one semaphore, so the partition needs ceil(numWG/SBMUL) groups spanning ceil(numWG/SBMUL)*SBMUL regions, but LDS_BYTES allocated numWG of them and LDSinit initialised numWG/SBMUL semaphores while the highest index in use is (numWG-1)/SBMUL. With WG=64, RADIX=4, SHUFL_BYTES=8 and -use WMUL=3,LDSMUL_W=2 that is 2144 bytes of LDS written past the end of the array, and semaphore 1 read uninitialised -- which either hangs on a stale 1, or slips through the spin loop without the lock. 2. The spin loop tested the wrong thing. atomic_cmpxchg returns the old value, so only 0 means the lock was taken; "== 1" let any other value out of the loop unlocked, after which LDStx_end cleared a semaphore this workgroup never held. Now "!= 0". 3. The "4 semaphores" assert was the wrong invariant and compiled to nothing without -use DEBUG. The space is now sized from the group count instead of hardwired, so there is nothing left to assert: -use WMUL=10,LDSMUL_W=2 wrote 5 semaphores into room for 4. 4. Two combinations cannot work and now fail to build rather than quietly misbehave: - variant 2 with LDSMUL > 1. partial_tabMul4/8 read and write the shared region outside the LDStx lock with only a bar(WG), which does not cover the other workgroups sharing that memory. This is what the "partitioned_LDS is a nightmare" note was about; refusing it also puts LDSptr's truncating divide out of reach, since variant 2 is its only caller and the truncation only occurs when sharing leaves LDS_SHUFL_BYTES off a 16-byte boundary. - LDSMUL >= 3 with SHUFL_BYTES == 4, where SBMUL*SHUFL_BYTES is 12 and shufl's >= 16 / == 8 / == 4 chain has no branch: it would return with the data unexchanged and no diagnostic. I have no nVidia hardware, and LDS sharing calls barsync(), which exists only where HAS_PTX >= 200, so none of this can run on an AMD, Intel or CPU device here. What I could check: the arithmetic, enumerated above; that the sharing path still compiles, with clang -fsyntax-only for an nvptx64 target across LDSMUL 1/2/4 and SHUFL_BYTES 4/8/16; that the two guards fire for exactly the two rejected combinations and nothing else; and that residues are unchanged on an Intel iGPU and POCL, with defaults and with -use TAIL_KERNELS=2,WMUL=4. Co-Authored-By: Claude Opus 5 (1M context) --- src/cl/fftbase.cl | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/src/cl/fftbase.cl b/src/cl/fftbase.cl index 2c55164a..708c343d 100644 --- a/src/cl/fftbase.cl +++ b/src/cl/fftbase.cl @@ -51,21 +51,45 @@ void OVERLOAD LDStx_end(local void *lds, const u32 numWG) { #define LDSPAD_COUNT(numWG) (!LDSPAD ? 0 : RADIX == 4 ? 12 : SBMUL(numWG) * SHUFL_BYTES >= 16 ? 7 : 56) // LDS_SHUFL_BYTES is the number of LDS bytes *allocated* for each workgroup (SBMUL > 1 means the workgroup can *access* some multiple of LDS_SHUFL_BYTES) #define LDS_SHUFL_BYTES(numWG) ((WG * RADIX + LDSPAD_COUNT(numWG)) * SHUFL_BYTES) -// The total number of LDS_BYTES allocated by a kernel includes space for 4 semaphores to control workgroup access -#define LDS_BYTES(numWG) (numWG * LDS_SHUFL_BYTES(numWG) + (SHARING_LDS(numWG) ? 16 : 0)) +// The workgroups are partitioned into groups of SBMUL that share one LDS region and one semaphore. +// SBMUL need not divide numWG, so round up: the last group is short but still spans SBMUL regions and +// owns a semaphore of its own. Allocating only numWG regions, and only ever four semaphores, is not +// enough then -- LDSsharing_ptr hands out a region past the end of the array and LDSinit leaves the +// last semaphore uninitialised. With SBMUL == 1, which is every configuration that does not opt into +// sharing, all of this is numWG regions and no semaphores, exactly as before. +#define LDS_GROUPS(numWG) ((numWG + SBMUL(numWG) - 1) / SBMUL(numWG)) +#define LDS_REGIONS(numWG) (LDS_GROUPS(numWG) * SBMUL(numWG)) +#define LDS_SEM_OFFSET(numWG) (LDS_REGIONS(numWG) * LDS_SHUFL_BYTES(numWG)) +#define LDS_BYTES(numWG) (LDS_SEM_OFFSET(numWG) + (SHARING_LDS(numWG) ? LDS_GROUPS(numWG) * 4 : 0)) + +// Variant 2 keeps its own pointer into the shared region (partitioned_lds) and both reads and writes it +// in partial_tabMul4/8 outside the LDStx lock, with only a bar(WG) that does not cover the other +// workgroups sharing that memory. That is what the "partitioned_LDS is a nightmare" note was about, so +// refuse the combination rather than corrupt quietly. This also keeps LDSptr's truncating divide out of +// reach: it is only inexact when sharing rounds LDS_SHUFL_BYTES off a 16-byte boundary, and variant 2 is +// its only caller. +#if VARIANT == 2 +#error LDSMUL > 1 is not supported with FFT variant 2 (partial_tabMul touches the shared LDS region outside the lock) +#endif -//// BUG BUG BUG -///variant 2 partitioned_LDS is a nightmare -- require variant 2 to be LDSMUL=1 until we can figure it out? +// shufl dispatches on SBMUL * SHUFL_BYTES with branches for >= 16, == 8 and == 4 and no fallback, so a +// product of 12 would return with the data unexchanged and no diagnostic. Only SBMUL == 3 can produce +// it. Rejecting on LDSMUL is slightly stronger than necessary -- a call with numWG < 3 would have come +// out at SBMUL < 3 -- but numWG is a runtime argument, and a build error beats silently wrong results. +#if LDSMUL >= 3 && SHUFL_BYTES == 4 +#error LDSMUL >= 3 with SHUFL_BYTES == 4 gives SBMUL * SHUFL_BYTES == 12, which no shufl branch handles +#endif // Initialize access to LDS memory. It may be advantageous to have independent workgroups share access to LDS memory via a lock controlling a critical section. // This may let a kernel use less LDS memory, or have each workgroup use more LDS memory to perform fewer passes of writing and reading LDS memory. void OVERLOAD LDSinit(void local *lds, const u32 numWG) { // Init semaphores to unlocked state if (SHARING_LDS(numWG)) { - assert(numWG / SBMUL(numWG) <= 4); // LDS_BYTES is hardwired to allocate 4 semaphores if (get_local_id(0) == 0) { - volatile local int *semaphores = (volatile local int *)(((local char *) lds) + numWG * LDS_SHUFL_BYTES(numWG)); - for (u32 i = 0; i < numWG / SBMUL(numWG); i++) semaphores[i] = 0; + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + LDS_SEM_OFFSET(numWG)); + // One per sharing group, and every group that exists: the highest index in use is + // (numWG - 1) / SBMUL, which the old numWG / SBMUL bound missed whenever SBMUL did not divide numWG. + for (u32 i = 0; i < LDS_GROUPS(numWG); i++) semaphores[i] = 0; } bar(); } @@ -117,10 +141,13 @@ void OVERLOAD LDStx_start(local void *lds, const u32 numWG) { } // Have first thread in a workgroup lock the semaphore controlling access to LDS memory if (get_local_id(0) % WG == 0) { - volatile local int *semaphores = (volatile local int *)(((local char *) lds) + numWG * LDS_SHUFL_BYTES(numWG)); + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + LDS_SEM_OFFSET(numWG)); // Lock semaphore (set to one) to gain access to critical section - while (atomic_cmpxchg(&semaphores[get_local_id(0) / WG / SBMUL(numWG)], 0, 1) == 1); + // Spin until the semaphore was observed unlocked: cmpxchg returns the old value, so anything other + // than 0 means the lock was not taken. Testing for 1 alone would let any other value through here + // without the lock, and LDStx_end would then clear a semaphore this workgroup never owned. + while (atomic_cmpxchg(&semaphores[get_local_id(0) / WG / SBMUL(numWG)], 0, 1) != 0); } LDSbar(numWG); } @@ -135,7 +162,7 @@ void OVERLOAD LDStx_end(local void *lds, const u32 numWG) { LDSbar(numWG); // Unlock the semaphore if (get_local_id(0) % WG == 0) { - volatile local int *semaphores = (volatile local int *)(((local char *) lds) + numWG * LDS_SHUFL_BYTES(numWG)); + volatile local int *semaphores = (volatile local int *)(((local char *) lds) + LDS_SEM_OFFSET(numWG)); semaphores[get_local_id(0) / WG / SBMUL(numWG)] = 0; } } From 128c0aa95879f6a18901c40f5f74e12d64f97b4b Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 18:35:35 -0600 Subject: [PATCH 207/214] shufl: make the FP32/GF31 "second RADIX == 8" index workgroup-general The LDSPAD "second RADIX == 8" case of the 32-bit shufl, in both the 8-byte and the 4-byte paths, picks between two index expressions on WG == 64. The else arm lays the data out as exactly eight padded rows, which only inverts when WG/64 == 8, and reads a row of 64 regardless of WG. Enumerating the resulting permutation against the generic shufl: WG = 64 0 of 512 wrong WG = 128 896 of 1024 wrong (48 reads of slots never written) WG = 256 1792 of 2048 wrong (32 reads of slots never written) WG = 512 0 of 4096 wrong So it happens to be right at exactly the two sizes that reach it today -- 512 (WG 64) and 4096 (WG 512) -- and wrong at everything between. Not a bug now: FP32 and GF31 route width/height 1024 through shufl_and_fft2 rather than plain shufl(f=8), so WG 128 never gets here, and no RADIX-8 shape produces WG 256. It goes live the moment a new shape or a different 1K dispatch does. The 64-bit sibling a few hundred lines above solves the same problem with an expression that is general in WG, and at WG == 64 its two arms are the same expression, so one form serves every size: write lds[i * (WG + 8) + lowMe] read lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)] Enumerated at WG 64, 128, 256 and 512: a bijection every time, agreeing with the generic shufl element for element, no unwritten slot read, and the highest index one below the allocation ((WG*RADIX + 56), which these paths use) in all four cases. The WG == 64 branch goes away with it -- that branch is what the whole family of these bugs is made of. The same shape is in the LDSPAD RADIX == 4 reads, whose else arm balances only at WG == 256. Those are correct at both workgroup sizes RADIX 4 can have today (WG 64 from a 256-wide shape, WG 256 from a 1024 one should the commented-out clause in nW()/nH() be re-enabled), and generalising them measured as no faster and no slower while adding 16 instructions to fftw.cl, so they are left as they are and guarded instead: a RADIX-4 workgroup of any other size now fails to build, with a message saying what to do about it, rather than silently reading slots nothing wrote. The guard fires for exactly that case -- WG 64 and 256 at RADIX 4 still build, as does every RADIX 8 size. The LDSSWIZ masks have the same character one size lower: enumerating all eight swizzle cases, five of them fold rows together at WG 32 and return 64 to 192 wrong elements of 256, while every one is exact from WG 64 up. Nothing reaches them at WG 32 either (that branch of fft_common asks for f=1,r=4 at RADIX 8, and f=4,r=8, which no swizzle case matches), so they get a guard too rather than a rewrite: LDSSWIZ now requires WG >= 64. Residues unchanged on an Intel iGPU and on POCL at 256:2:256, 512:2:256 and 1K:2:256, and with -use SHUFL_BYTES_W=4 and SHUFL_BYTES_H=4. Co-Authored-By: Claude Opus 5 (1M context) --- src/cl/shufl.cl | 52 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/src/cl/shufl.cl b/src/cl/shufl.cl index 0f16c2f7..ca81c57a 100644 --- a/src/cl/shufl.cl +++ b/src/cl/shufl.cl @@ -1,6 +1,28 @@ // Copyright (C) Mihai Preda +// The LDSSWIZ swizzle masks below are sized for a workgroup of at least 64: at WG 32 the XOR patterns +// (lowMe & 7), (lowMe & 15), ((lowMe / 8) & 15) and friends fold several rows onto each other, and five +// of the cases then return the wrong data -- 64 to 192 elements of 256, depending on the case. No +// dispatch reaches them there today (the WG == 32 branch of fft_common asks for f=1,r=4 at RADIX 8 and +// f=4,r=8, and no swizzle case matches either), so like the padded cases above this is a constraint to +// record rather than code to rewrite. +#if LDSSWIZ && WG < 64 +#error LDSSWIZ needs a workgroup of at least 64: its swizzle masks fold rows together below that +#endif + +// The LDSPAD RADIX == 4 reads in this file choose between a WG == 64 form and an else arm whose +// i * 64 and (lowMe / 64) * 16 terms only balance at WG == 256. Both workgroup sizes RADIX 4 can +// have today are therefore handled -- a 256-wide/high shape gives WG 64, and a 1024 one would give +// WG 256 if the commented-out clause in FFTConfig::nW()/nH() were re-enabled -- so the code is +// correct as it stands, and generalising it would add index arithmetic for no present benefit. +// Any other WG would read slots that were never written, silently and with the wrong residue as the +// only symptom, so refuse to build it instead. Generalising is easy when it is needed: i * (WG / 4) +// in place of i * 64 is index-identical at both 64 and 256. +#if LDSPAD && RADIX == 4 && WG != 64 && WG != 256 +#error RADIX == 4 with this workgroup size needs the LDSPAD reads in shufl.cl generalised first (they assume WG is 64 or 256) +#endif + // Strongly typed versions of LDSptr and LDSsharing_ptr. On TitanV, CUDA 12.9, this is 1% faster. local T_F_Z31_Z61 * OVERLOAD LDSptr(local T_F_Z31_Z61 *lds, const u32 numWG) { return lds + ((u32)get_local_id(0) / WG) * LDS_SHUFL_BYTES(numWG) / sizeof(T_F_Z31_Z61); @@ -196,6 +218,10 @@ void OVERLOAD shufl(local T2_GF61 *lds2, T2_GF61 *u, u32 f, u32 r, u32 numWG, u3 // Read from LDS in output order. In the example: u[0] = 0, 64, ... 448, 8, 72... u[1] = +1 if (f == 8 && r == 8 && RADIX == 8) { LDStx_start(lds2, numWG); + // One expression for every WG. The old pair of arms laid the data out as exactly eight padded rows, + // which only inverts when WG/64 == 8, and read a row of 64 regardless of WG: correct at WG 64 and 512, + // wrong everywhere between (896 of 1024 elements at WG 128, 1792 of 2048 at WG 256, some of them + // reading slots nothing had written). This is the form the 64-bit sibling above already uses. for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } LDSbar(numWG); if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 8) * (WG + 8) + (lowMe & 7)]; } @@ -620,11 +646,13 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Pad 8 values after every 64 values to eliminate bank conflicts. if (1 && f == 8 && r == 8 && RADIX == 8) { LDStx_start(lds2, numWG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i]; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i]; } + // One expression for every WG. The old pair of arms laid the data out as exactly eight padded rows, + // which only inverts when WG/64 == 8, and read a row of 64 regardless of WG: correct at WG 64 and 512, + // wrong everywhere between (896 of 1024 elements at WG 128, 1792 of 2048 at WG 256, some of them + // reading slots nothing had written). This is the form the 64-bit sibling above already uses. + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i]; } LDSbar(numWG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i] = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } LDStx_end(lds2, numWG); return; } @@ -785,17 +813,17 @@ void OVERLOAD shufl(local F2_GF31 *lds2, F2_GF31 *u, u32 f, u32 r, u32 numWG, u3 // Pad 8 values after every 64 values to eliminate bank conflicts. if (f == 8 && r == 8 && RADIX == 8) { LDStx_start(lds2, numWG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].x; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].x; } + // One expression for every WG. The old pair of arms laid the data out as exactly eight padded rows, + // which only inverts when WG/64 == 8, and read a row of 64 regardless of WG: correct at WG 64 and 512, + // wrong everywhere between (896 of 1024 elements at WG 128, 1792 of 2048 at WG 256, some of them + // reading slots nothing had written). This is the form the 64-bit sibling above already uses. + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].x; } LDSbar(numWG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].x = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } LDSbar(numWG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { lds[ (lowMe / 8) * (WG + 8) + i * 8 + (lowMe & 7)] = u[i].y; } - else for (u32 i = 0; i < RADIX; ++i) { lds[((lowMe / 8) & 7) * (WG + 8) + (lowMe / 64) * 64 + i * 8 + (lowMe & 7)] = u[i].y; } + for (u32 i = 0; i < RADIX; ++i) { lds[i * (WG + 8) + lowMe] = u[i].y; } LDSbar(numWG); - if (WG == 64) for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG + 8) + lowMe]; } - else for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * 64 + lowMe / 64 * (WG + 8) + (lowMe & 63)]; } + for (u32 i = 0; i < RADIX; ++i) { u[i].y = lds[i * (WG / 64) * 8 + (lowMe / 64) * 8 + ((lowMe / 8) & 7) * (WG + 8) + (lowMe & 7)]; } LDStx_end(lds2, numWG); return; } From 76863a33a7e2cc8ea696bd4efdaf0eef0d7fd59c Mon Sep 17 00:00:00 2001 From: Mark Rose Date: Sun, 20 Sep 2026 18:58:28 -0600 Subject: [PATCH 208/214] -use TAIL_KERNELS governs tailMul as well now Since ae8aa0c added the double-wide and split-kernel variants to tailMul, TAIL_KERNELS selects the shape of both tail kernels, not just tailSquare. The help text still described it as tailSquare only. The TAIL_TRIGS line just below is left as it is: it says tailSquare, and for FP64 and FP32 that remains exactly right, because tailMul has no TAIL_TRIGS branches -- it always computes the trig values from scratch. Co-Authored-By: Claude Opus 5 (1M context) --- src/Args.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Args.cpp b/src/Args.cpp index cec0ef76..bb6bbe54 100644 --- a/src/Args.cpp +++ b/src/Args.cpp @@ -174,7 +174,7 @@ named "config.txt" in the prpll run directory. may not work on Nvidia GPUs or on RDNA AMD GPUs where it produces errors (which are nevertheless detected). -use NO_ASM : do not use __asm() blocks (inline assembly) - -use TAIL_KERNELS= : change how tailSquare operates according to : + -use TAIL_KERNELS= : change how tailSquare and tailMul operate according to : 0 = single wide, single kernel 1 = single wide, two kernels 2 = double wide, single kernel From 784e712e80c21605b82218d580b100634672c472 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 21 Sep 2026 01:57:25 +0000 Subject: [PATCH 209/214] Make restrict on the trig/weight table pointers optional: -use ENABLE_RESTRICT=1 Commit 65d5f08 marked the Trig* and BigTab* typedefs restrict. On nVidia that turns the table loads into ld.global.nc and lets the compiler batch them early, which raised the register need (FP64 carryFused 72 -> 96, tailSquare 80 -> 118). The default carryFused register cap (80) then spilled to local memory, about 12% slower on a TITAN V. Restrict is now off by default and enabled with the (undocumented) -use ENABLE_RESTRICT=1. Measured: default carryFused fits with no spill and matches the Sep 1 build; restrict plus REGCF64=96 is about 1% faster on a TITAN V; no measurable effect on Radeon VII / MI50 or on the 1:512:8:512:202 NTT. Co-Authored-By: Claude Sonnet 5 --- src/Gpu.cpp | 1 + src/cl/base.cl | 47 +++++++++++++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 2a9e5c36..7b6bc2dd 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -295,6 +295,7 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector Date: Mon, 21 Sep 2026 02:02:24 +0000 Subject: [PATCH 210/214] Don't list ENABLE_RESTRICT among the supported -use keys The known-keys list in Gpu.cpp is the set of officially supported -use settings, i.e. the ones users are encouraged to try. ENABLE_RESTRICT is dangerous without intimate knowledge of its register implications, so like REGCF64 it stays an undocumented setting. It still works; it just draws the "unrecognized -use key" warning. Co-Authored-By: Claude Sonnet 5 --- src/Gpu.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 7b6bc2dd..2a9e5c36 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -295,7 +295,6 @@ string clDefines(Args& args, cl_device_id id, FFTConfig fft, const vector Date: Mon, 21 Sep 2026 17:23:24 +0000 Subject: [PATCH 211/214] AMD: optional per-kernel waves-per-SIMD cap; default 2 for the in-place middle kernels on Vega The in-place fftMiddleIn / fftMiddleOut kernels compile to 133 / 134 VGPRs on gfx906. Above 128 VGPRs a SIMD holds only one wave, and with 256-thread workgroups that leaves one workgroup per CU, so every barrier of the 16x16 LDS transpose stalls the CU. This, not the memory layout, is why INPLACE=1 was slower on AMD. Add an AMD analog of the CUDA register cap: Gpu::amdWavesPerEu() passes -DAMD_WAVES_PER_EU=n to a kernel and the new KERNEL_CAP macro turns it into amdgpu_waves_per_eu(n). It is applied to the fftMiddleIn/Out, tailSquare/Mul and carryFused kernels. The default is 2 waves per SIMD for the middle kernels when the transform is in place on Vega class GPUs (gfx900/902/904/906/909/90c); everything else is unchanged. Overrides (undocumented, like REGCF64): -use WPE_MIDIN=n, WPE_MIDOUT=n, WPE_TAIL=n, WPE_CARRY=n, where 0 means the compiler default. Measured at 512:16:512:202, workers=1, INPLACE=1: Radeon VII 1232 -> 1080 us/iter, MI50 1020 -> 924 us/iter (out of place: 994 and 906, unchanged). The cap costs 2 spilled VGPRs in fftMiddleIn. Capping to 3 or more waves, or capping tailSquare or carryFused, was slower, so those are not defaulted. CUDA PTX is identical for all kernels. Co-Authored-By: Claude Sonnet 5 --- src/Gpu.cpp | 30 ++++++++++++++++++++++++++++-- src/Gpu.h | 1 + src/cl/base.cl | 8 ++++++++ src/cl/carryfused.cl | 18 +++++++++--------- src/cl/fftmiddlein.cl | 16 ++++++++-------- src/cl/fftmiddleout.cl | 16 ++++++++-------- src/cl/tailmul.cl | 24 ++++++++++++------------ src/cl/tailsquare.cl | 24 ++++++++++++------------ 8 files changed, 86 insertions(+), 51 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 2a9e5c36..c7d724b4 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -643,7 +643,7 @@ Gpu::~Gpu() { // Part of GPU initialization is to compute the default number of registers each kernel should target during compilation. // Kernel register usage is critical for maximizing GPU occupancy. The default values can be overrriden with command line arguments. -// This feature currently only works for the CUDA compiler. +// This feature currently only works for the CUDA compiler. AMD GPUs have a similar (waves-per-SIMD) control, see amdWavesPerEu below. // Most kernels have occupancy limited by register usage. For reference, the following guidelines dictate where an "uptick" in occupancy occurs. // If kernel threads=256, register crossovers are at 128, 80, 64, 48, 40 // If kernel threads=128, register crossovers are at 128, 96, 80, 72, 64, 56, 48, 40 @@ -793,10 +793,36 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { // Format an explicit register count setting return string("--maxrregcount=") + to_string(regs) + " "; #else - return string(""); + return amdWavesPerEu(which_kernel); #endif } +// AMD analog of the CUDA register cap: an optional minimum number of waves per SIMD for a kernel, which caps its VGPR usage. +// On gfx9 (256 VGPRs per lane, allocated in units of 4) the occupancy crossovers are: 128 VGPRs for 2 waves, 84 for 3, 64 for 4, 48 for 5. +// A kernel a few VGPRs above the 128 boundary runs with one wave per SIMD; capping it costs a few spills but doubles occupancy. +// Only that one-wave cliff is worth a default: on a Radeon VII / MI50 the in-place fftMiddleIn / fftMiddleOut kernels use 133 VGPRs, and requiring +// 2 waves per SIMD recovers most of their slowdown. Capping to reach 3 or more waves, or capping tailSquare / carryFused, was measured slower. +// The defaults can be overridden with -use WPE_MIDIN=n, WPE_MIDOUT=n, WPE_TAIL=n, WPE_CARRY=n (0 = compiler default). +string Gpu::amdWavesPerEu([[maybe_unused]] enum WHICH_KERNEL which_kernel) { + cl_device_id const id = shared.context->deviceId(); + if (!isAmdGpu(id)) return string(""); + const char *use_override = ""; + bool is_middle = false; + switch (which_kernel) { + case CARRYFUSED: use_override = "WPE_CARRY"; break; + case MIDIN: case MIDIN31: case MIDIN61: use_override = "WPE_MIDIN"; is_middle = true; break; + case MIDOUT: case MIDOUT31: case MIDOUT61: use_override = "WPE_MIDOUT"; is_middle = true; break; + case TAIL: case TAIL31: case TAIL61: use_override = "WPE_TAIL"; break; + } + // Default: in-place middle kernels on Vega class GPUs (gfx900/902/904/906/909/90c: 256 VGPRs per lane, 64 KB LDS). Other architectures are untested. + string const name = getDeviceName(id); + bool const vega = name.rfind("gfx90", 0) == 0 && name.size() > 5 && string("02469c").find(name[5]) != string::npos; + int waves = (in_place && is_middle && vega) ? 2 : 0; + int const override_waves = args.value(use_override, -1); // -1 = not specified + if (override_waves >= 0) waves = override_waves; + return waves > 0 ? string("-DAMD_WAVES_PER_EU=") + to_string(waves) + " " : string(""); +} + // Kernels are compiled one at a time, but OpenCL source files contain multiple kernels. This routine set the #defines necessary so that only one kernel is compiled. // While not strictly necessary, startup speed will be a bit faster if we do less compilations. string Gpu::kernelDefines(enum WHICH_KERNEL_TYPE which_kernel) { diff --git a/src/Gpu.h b/src/Gpu.h index 15a44c85..c7f24dfd 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -359,6 +359,7 @@ class Gpu { void doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors); enum WHICH_KERNEL {CARRYFUSED=0, MIDIN=1, MIDIN31=2, MIDIN61=3, TAIL=4, TAIL31=5, TAIL61=6, MIDOUT=7, MIDOUT31=8, MIDOUT61=9}; string numCudaRegisters(enum WHICH_KERNEL which_kernel); + string amdWavesPerEu(enum WHICH_KERNEL which_kernel); enum WHICH_KERNEL_TYPE {KFP=0, K31=1, K61=2, KALL=3}; string kernelDefines(enum WHICH_KERNEL_TYPE which_kernel); }; diff --git a/src/cl/base.cl b/src/cl/base.cl index 5fb5ac8c..105303cd 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -300,6 +300,14 @@ ulong2 OVERLOAD U2(unsigned long long a, unsigned long long b) { return (ulong2) #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void +// AMD only: Gpu.cpp can pass -DAMD_WAVES_PER_EU=n to ask for at least n waves per SIMD, which caps the kernel's VGPR usage. +// This avoids the one-wave-per-SIMD occupancy cliff (more than 128 VGPRs on gfx9). See Gpu::amdWavesPerEu. +#if AMDGPU && defined(AMD_WAVES_PER_EU) +#define KERNEL_CAP(x) kernel __attribute__((reqd_work_group_size(x, 1, 1), amdgpu_waves_per_eu(AMD_WAVES_PER_EU))) void +#else +#define KERNEL_CAP(x) KERNEL(x) +#endif + // ENABLE_RESTRICT=1 marks the trig and weight table pointers below as restrict. That lets the compiler hoist their loads (on nVidia they become ld.global.nc), // which is sometimes faster but can cost many more registers. Off by default. #ifndef ENABLE_RESTRICT diff --git a/src/cl/carryfused.cl b/src/cl/carryfused.cl index 8d857586..1f949333 100644 --- a/src/cl/carryfused.cl +++ b/src/cl/carryfused.cl @@ -122,7 +122,7 @@ void OVERLOAD shufl_carries_up(local void *lds2, i32 *carry, u32 me, u32 lowMe) // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { local T2 lds[LDS_BYTES(WMUL) / sizeof(T2)]; LDSinit(lds, WMUL); @@ -348,7 +348,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, +KERNEL_CAP(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigFP32 smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { local F2 lds[LDS_BYTES(WMUL) / sizeof(F2)]; LDSinit(lds, WMUL); @@ -577,7 +577,7 @@ KERNEL(G_W * WMUL) carryFused(P(F2) out, CP(F2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { +KERNEL_CAP(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF31 smallTrig, P(uint) bufROE) { local GF31 lds[LDS_BYTES(WMUL) / sizeof(GF31)]; LDSinit(lds, WMUL); @@ -813,7 +813,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF31) out, CP(GF31) in, u32 posROE, P(i64) carry // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { +KERNEL_CAP(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, TrigGF61 smallTrig, P(uint) bufROE) { local GF61 lds[LDS_BYTES(WMUL) / sizeof(GF61)]; LDSinit(lds, WMUL); @@ -1055,7 +1055,7 @@ KERNEL(G_W * WMUL) carryFused(P(GF61) out, CP(GF61) in, u32 posROE, P(i64) carry // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTab CONST_THREAD_WEIGHTS, BigTab THREAD_WEIGHTS, P(uint) bufROE) { local T2 lds[LDS_BYTES(WMUL) / sizeof(T2)]; local GF31 *lds31 = (local GF31 *) lds; @@ -1324,7 +1324,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { local F2 ldsF2[LDS_BYTES(WMUL) / sizeof(F2)]; local GF31 *lds31 = (local GF31 *) ldsF2; @@ -1602,7 +1602,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; @@ -1882,7 +1882,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, P(uint) bufROE) { local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local GF31 *lds31 = (local GF31 *) lds61; LDSinit(lds61, WMUL); @@ -2158,7 +2158,7 @@ KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShut // The "carryFused" is equivalent to the sequence: fftW, carryA, carryB, fftPremul. // It uses "stairway forwarding" (forwarding carry data from one workgroup to the next) -KERNEL(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, +KERNEL_CAP(G_W * WMUL) carryFused(P(T2) out, CP(T2) in, u32 posROE, P(i64) carryShuttle, P(u32) ready, Trig smallTrig, ConstBigTabFP32 CONST_THREAD_WEIGHTS, BigTabFP32 THREAD_WEIGHTS, P(uint) bufROE) { local GF61 lds61[LDS_BYTES(WMUL) / sizeof(GF61)]; local F2 *ldsF2 = (local F2 *) lds61; diff --git a/src/cl/fftmiddlein.cl b/src/cl/fftmiddlein.cl index 5c416917..aa5d28fe 100644 --- a/src/cl/fftmiddlein.cl +++ b/src/cl/fftmiddlein.cl @@ -10,7 +10,7 @@ #if FFT_FP64 -KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { T2 u[MIDDLE]; u32 SIZEY = IN_WG / IN_SIZEX; @@ -67,7 +67,7 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { #if FFT_FP32 -KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { F2 u[MIDDLE]; CP(F2) inF2 = (CP(F2)) in; @@ -128,7 +128,7 @@ KERNEL(IN_WG) fftMiddleIn(P(T2) out, CP(T2) in, u32 base, Trig trig) { #if NTT_GF31 -KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF31 u[MIDDLE]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -189,7 +189,7 @@ KERNEL(IN_WG) fftMiddleInGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { #if NTT_GF61 -KERNEL(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(IN_WG) fftMiddleInGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF61 u[MIDDLE]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -298,7 +298,7 @@ void map_striping_group_id(u32 base_lo, u32 g, u32 *startx, u32 *starty) { #if FFT_FP64 -KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); T2 u[MIDDLE]; @@ -353,7 +353,7 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { #if FFT_FP32 -KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); F2 u[MIDDLE]; @@ -412,7 +412,7 @@ KERNEL(256) fftMiddleIn(P(T2) out, P(T2) in, u32 base, Trig trig) { #if NTT_GF31 -KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleInGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF31 u[MIDDLE]; @@ -471,7 +471,7 @@ KERNEL(256) fftMiddleInGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { #if NTT_GF61 -KERNEL(256) fftMiddleInGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleInGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF61 u[MIDDLE]; diff --git a/src/cl/fftmiddleout.cl b/src/cl/fftmiddleout.cl index 234dadd5..78b589a3 100644 --- a/src/cl/fftmiddleout.cl +++ b/src/cl/fftmiddleout.cl @@ -10,7 +10,7 @@ #if FFT_FP64 -KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { T2 u[MIDDLE]; u32 SIZEY = OUT_WG / OUT_SIZEX; @@ -75,7 +75,7 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { #if FFT_FP32 -KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { F2 u[MIDDLE]; CP(F2) inF2 = (CP(F2)) in; @@ -141,7 +141,7 @@ KERNEL(OUT_WG) fftMiddleOut(P(T2) out, CP(T2) in, u32 base, Trig trig) { #if NTT_GF31 -KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF31 u[MIDDLE]; CP(GF31) in31 = (CP(GF31)) (in + DISTGF31); @@ -204,7 +204,7 @@ KERNEL(OUT_WG) fftMiddleOutGF31(P(T2) out, CP(T2) in, u32 base, Trig trig) { #if NTT_GF61 -KERNEL(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { +KERNEL_CAP(OUT_WG) fftMiddleOutGF61(P(T2) out, CP(T2) in, u32 base, Trig trig) { GF61 u[MIDDLE]; CP(GF61) in61 = (CP(GF61)) (in + DISTGF61); @@ -305,7 +305,7 @@ void map_striping_group_id(u32 base_lo, u32 g, u32 *startx, u32 *starty) { #if FFT_FP64 -KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); T2 u[MIDDLE]; @@ -361,7 +361,7 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { #if FFT_FP32 -KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); F2 u[MIDDLE]; @@ -418,7 +418,7 @@ KERNEL(256) fftMiddleOut(P(T2) out, P(T2) in, u32 base, Trig trig) { #if NTT_GF31 -KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleOutGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF31 u[MIDDLE]; @@ -472,7 +472,7 @@ KERNEL(256) fftMiddleOutGF31(P(T2) out, P(T2) in, u32 base, Trig trig) { #if NTT_GF61 -KERNEL(256) fftMiddleOutGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { +KERNEL_CAP(256) fftMiddleOutGF61(P(T2) out, P(T2) in, u32 base, Trig trig) { assert(out == in); GF61 u[MIDDLE]; diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 761c9046..035f80b8 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -88,7 +88,7 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s #if !SINGLE_KERNEL // The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL_CAP(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local T2 lds[LDS_BYTES(1) / sizeof(T2)]; LDSinit(lds, 1); @@ -139,7 +139,7 @@ KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES(1) / sizeof(T2)]; LDSinit(lds, 1); @@ -237,7 +237,7 @@ void OVERLOAD pairMul2_special(T2 *u, T2 *p, T2 base_squared) { } } -KERNEL(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES(2) / sizeof(T2)]; LDSinit(lds, 2); @@ -357,7 +357,7 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s #if !SINGLE_KERNEL // The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL_CAP(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local F2 lds[LDS_BYTES(1) / sizeof(F2)]; LDSinit(lds, 1); @@ -405,7 +405,7 @@ KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES(1) / sizeof(F2)]; LDSinit(lds, 1); @@ -500,7 +500,7 @@ void OVERLOAD pairMul2_special(F2 *u, F2 *p, F2 base_squared) { } } -KERNEL(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailMul(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES(2) / sizeof(F2)]; LDSinit(lds, 2); @@ -616,7 +616,7 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar #if !SINGLE_KERNEL // The kernel tailMulZeroGF31 handles the special cases in tailMulGF31, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailMulZeroGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL_CAP(G_H) tailMulZeroGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; LDSinit(lds, 1); @@ -681,7 +681,7 @@ KERNEL(G_H) tailMulZeroGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; LDSinit(lds, 1); @@ -784,7 +784,7 @@ void OVERLOAD pairMul2_special(GF31 *u, GF31 *p, GF31 base_squared) { } } -KERNEL(G_H * 2) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailMulGF31(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES(2) / sizeof(GF31)]; LDSinit(lds, 2); @@ -911,7 +911,7 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar #if !SINGLE_KERNEL // The kernel tailMulZeroGF61 handles the special cases in tailMulGF61, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailMulZeroGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL_CAP(G_H) tailMulZeroGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; LDSinit(lds, 1); @@ -976,7 +976,7 @@ KERNEL(G_H) tailMulZeroGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; LDSinit(lds, 1); @@ -1079,7 +1079,7 @@ void OVERLOAD pairMul2_special(GF61 *u, GF61 *p, GF61 base_squared) { } } -KERNEL(G_H * 2) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailMulGF61(P(T2) out, CP(T2) in, CP(T2) a, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES(2) / sizeof(GF61)]; LDSinit(lds, 2); diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index 4f1e396d..f9c73ce7 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -92,7 +92,7 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { local T2 lds[LDS_BYTES(1) / sizeof(T2)]; LDSinit(lds, 1); @@ -133,7 +133,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES(1) / sizeof(T2)]; LDSinit(lds, 1); @@ -242,7 +242,7 @@ void OVERLOAD pairSq2_special(T2 *u, T2 base_squared) { } } -KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local T2 lds[LDS_BYTES(2) / sizeof(T2)]; LDSinit(lds, 2); @@ -375,7 +375,7 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { local F2 lds[LDS_BYTES(1) / sizeof(F2)]; LDSinit(lds, 1); @@ -412,7 +412,7 @@ KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES(1) / sizeof(F2)]; LDSinit(lds, 1); @@ -517,7 +517,7 @@ void OVERLOAD pairSq2_special(F2 *u, F2 base_squared) { } } -KERNEL(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailSquare(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local F2 lds[LDS_BYTES(2) / sizeof(F2)]; LDSinit(lds, 2); @@ -650,7 +650,7 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; LDSinit(lds, 1); @@ -705,7 +705,7 @@ KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; LDSinit(lds, 1); @@ -805,7 +805,7 @@ void OVERLOAD pairSq2_special(GF31 *u, GF31 base_squared) { } } -KERNEL(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailSquareGF31(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF31 lds[LDS_BYTES(2) / sizeof(GF31)]; LDSinit(lds, 2); @@ -992,7 +992,7 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; LDSinit(lds, 1); @@ -1047,7 +1047,7 @@ KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { #if SINGLE_WIDE -KERNEL(G_H) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; LDSinit(lds, 1); @@ -1147,7 +1147,7 @@ void OVERLOAD pairSq2_special(GF61 *u, GF61 base_squared) { } } -KERNEL(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { +KERNEL_CAP(G_H * 2) tailSquareGF61(P(T2) out, CP(T2) in, u32 base, Trig smallTrig) { local GF61 lds[LDS_BYTES(2) / sizeof(GF61)]; LDSinit(lds, 2); From 3121d9854849636d8899bf4c2c4762e400a76d3d Mon Sep 17 00:00:00 2001 From: george Date: Mon, 21 Sep 2026 18:07:26 +0000 Subject: [PATCH 212/214] AMD register cap: use the existing REGMI64/REGMO64/REGTS64/REGCF64 (etc.) options instead of new WPE_* keys numCudaRegisters() already selects the per-kernel, per-FFT-type option key (REGMI64, REGMO31, REGTS61, REGCF64, ...). Let that selection run on every backend and hand the option value to the new Gpu::amdRegisterOption(), which follows the CUDA conventions: 0 = not specified (default), -1 = compiler default (no cap), 1..10 = minimum waves per SIMD (the AMD counterpart of CUDA launch bounds; 10 is the GCN maximum), more than 10 = explicit VGPR count (amdgpu_num_vgpr; rocm generates its usual code and then spills to fit). The default is unchanged: 2 waves per SIMD for the in-place middle kernels on Vega class GPUs. The WPE_* keys are gone. KERNEL_CAP now understands both -DAMD_WAVES_PER_EU=n and -DAMD_NUM_VGPR=n. The CUDA path is unchanged (identical -v register reports). Radeon VII, INPLACE=1: default 1079 us/iter (128/127 VGPRs), REGMI64=-1,REGMO64=-1 1231 (133/134), REGMI64=3,REGMO64=3 1076 (84/84), REGMI64=128,REGMO64=128 1080 (128/127). MI50 default 922. All residues match. Co-Authored-By: Claude Sonnet 5 --- src/Gpu.cpp | 47 ++++++++++++++++++++++------------------------- src/Gpu.h | 2 +- src/cl/base.cl | 8 +++++--- 3 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index c7d724b4..778b01fc 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -643,14 +643,13 @@ Gpu::~Gpu() { // Part of GPU initialization is to compute the default number of registers each kernel should target during compilation. // Kernel register usage is critical for maximizing GPU occupancy. The default values can be overrriden with command line arguments. -// This feature currently only works for the CUDA compiler. AMD GPUs have a similar (waves-per-SIMD) control, see amdWavesPerEu below. +// On CUDA this sets --maxrregcount (or launch bounds). On AMD the same REGxxxx options select waves per SIMD or a VGPR count, see amdRegisterOption below. // Most kernels have occupancy limited by register usage. For reference, the following guidelines dictate where an "uptick" in occupancy occurs. // If kernel threads=256, register crossovers are at 128, 80, 64, 48, 40 // If kernel threads=128, register crossovers are at 128, 96, 80, 72, 64, 56, 48, 40 // If kernel threads=64, register crossovers are at 128, 112, 96, 88, 80, 72, 64, 56, 48, 40 -string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { -#if CUDA_BACKEND - int regs = 0; +string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { + [[maybe_unused]] int regs = 0; // Default CUDA maximum register count (the AMD path only uses the override value) const char *use_override = ""; // Allow command line to prefer the CUDA compiler's default number of registers if (args.value("NOREG", 0)) return string(""); @@ -784,6 +783,7 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { } // Get the optional override register count int const override_regs = args.value(use_override, 0); +#if CUDA_BACKEND // If a specified override is small, use the count as a CUDA launch_bounds rather than a maximum register count if (override_regs && (override_regs > 0 && override_regs <= 16)) return string("-DCUDA_MIN_BLOCKS=") + to_string(override_regs) + " "; // If specified, override the default maximum register count @@ -793,34 +793,31 @@ string Gpu::numCudaRegisters([[maybe_unused]] enum WHICH_KERNEL which_kernel) { // Format an explicit register count setting return string("--maxrregcount=") + to_string(regs) + " "; #else - return amdWavesPerEu(which_kernel); + return amdRegisterOption(which_kernel, override_regs); #endif } -// AMD analog of the CUDA register cap: an optional minimum number of waves per SIMD for a kernel, which caps its VGPR usage. -// On gfx9 (256 VGPRs per lane, allocated in units of 4) the occupancy crossovers are: 128 VGPRs for 2 waves, 84 for 3, 64 for 4, 48 for 5. -// A kernel a few VGPRs above the 128 boundary runs with one wave per SIMD; capping it costs a few spills but doubles occupancy. -// Only that one-wave cliff is worth a default: on a Radeon VII / MI50 the in-place fftMiddleIn / fftMiddleOut kernels use 133 VGPRs, and requiring -// 2 waves per SIMD recovers most of their slowdown. Capping to reach 3 or more waves, or capping tailSquare / carryFused, was measured slower. -// The defaults can be overridden with -use WPE_MIDIN=n, WPE_MIDOUT=n, WPE_TAIL=n, WPE_CARRY=n (0 = compiler default). -string Gpu::amdWavesPerEu([[maybe_unused]] enum WHICH_KERNEL which_kernel) { +// AMD analog of the CUDA register cap, driven by the same REGxxxx options. override_regs is the value of the kernel's option: +// 0 = not specified (use the default below), -1 = compiler default (no cap), +// 1..10 = minimum waves per SIMD (like CUDA's launch bounds; 10 is the GCN maximum), more than 10 = explicit VGPR count. +// A minimum-waves request caps VGPR usage. On gfx9 (256 VGPRs per lane, allocated in units of 4) the occupancy crossovers are: +// 128 VGPRs for 2 waves, 84 for 3, 64 for 4, 48 for 5. A kernel a few VGPRs above the 128 boundary runs with one wave per SIMD; +// capping it costs a few spills but doubles occupancy. Only that one-wave cliff is worth a default: on a Radeon VII / MI50 the in-place +// fftMiddleIn / fftMiddleOut kernels use 133 VGPRs, and requiring 2 waves per SIMD recovers most of their slowdown. Capping to reach 3 or more +// waves, or capping tailSquare / carryFused, was measured slower. An explicit VGPR count makes rocm generate its usual code and then spill to fit. +string Gpu::amdRegisterOption([[maybe_unused]] enum WHICH_KERNEL which_kernel, int override_regs) { cl_device_id const id = shared.context->deviceId(); if (!isAmdGpu(id)) return string(""); - const char *use_override = ""; - bool is_middle = false; - switch (which_kernel) { - case CARRYFUSED: use_override = "WPE_CARRY"; break; - case MIDIN: case MIDIN31: case MIDIN61: use_override = "WPE_MIDIN"; is_middle = true; break; - case MIDOUT: case MIDOUT31: case MIDOUT61: use_override = "WPE_MIDOUT"; is_middle = true; break; - case TAIL: case TAIL31: case TAIL61: use_override = "WPE_TAIL"; break; - } - // Default: in-place middle kernels on Vega class GPUs (gfx900/902/904/906/909/90c: 256 VGPRs per lane, 64 KB LDS). Other architectures are untested. + if (override_regs < 0) return string(""); + if (override_regs > 10) return string("-DAMD_NUM_VGPR=") + to_string(override_regs) + " "; + if (override_regs > 0) return string("-DAMD_WAVES_PER_EU=") + to_string(override_regs) + " "; + // Default: 2 waves per SIMD for the in-place middle kernels on Vega class GPUs (gfx900/902/904/906/909/90c: 256 VGPRs per lane). + // Other architectures are untested. + bool const is_middle = which_kernel == MIDIN || which_kernel == MIDIN31 || which_kernel == MIDIN61 || + which_kernel == MIDOUT || which_kernel == MIDOUT31 || which_kernel == MIDOUT61; string const name = getDeviceName(id); bool const vega = name.rfind("gfx90", 0) == 0 && name.size() > 5 && string("02469c").find(name[5]) != string::npos; - int waves = (in_place && is_middle && vega) ? 2 : 0; - int const override_waves = args.value(use_override, -1); // -1 = not specified - if (override_waves >= 0) waves = override_waves; - return waves > 0 ? string("-DAMD_WAVES_PER_EU=") + to_string(waves) + " " : string(""); + return (in_place && is_middle && vega) ? string("-DAMD_WAVES_PER_EU=2 ") : string(""); } // Kernels are compiled one at a time, but OpenCL source files contain multiple kernels. This routine set the #defines necessary so that only one kernel is compiled. diff --git a/src/Gpu.h b/src/Gpu.h index c7f24dfd..06ae27d3 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -359,7 +359,7 @@ class Gpu { void doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors); enum WHICH_KERNEL {CARRYFUSED=0, MIDIN=1, MIDIN31=2, MIDIN61=3, TAIL=4, TAIL31=5, TAIL61=6, MIDOUT=7, MIDOUT31=8, MIDOUT61=9}; string numCudaRegisters(enum WHICH_KERNEL which_kernel); - string amdWavesPerEu(enum WHICH_KERNEL which_kernel); + string amdRegisterOption(enum WHICH_KERNEL which_kernel, int override_regs); enum WHICH_KERNEL_TYPE {KFP=0, K31=1, K61=2, KALL=3}; string kernelDefines(enum WHICH_KERNEL_TYPE which_kernel); }; diff --git a/src/cl/base.cl b/src/cl/base.cl index 105303cd..02ffceeb 100644 --- a/src/cl/base.cl +++ b/src/cl/base.cl @@ -300,9 +300,11 @@ ulong2 OVERLOAD U2(unsigned long long a, unsigned long long b) { return (ulong2) #define KERNEL(x) kernel __attribute__((reqd_work_group_size(x, 1, 1))) void -// AMD only: Gpu.cpp can pass -DAMD_WAVES_PER_EU=n to ask for at least n waves per SIMD, which caps the kernel's VGPR usage. -// This avoids the one-wave-per-SIMD occupancy cliff (more than 128 VGPRs on gfx9). See Gpu::amdWavesPerEu. -#if AMDGPU && defined(AMD_WAVES_PER_EU) +// AMD only: Gpu.cpp can pass -DAMD_WAVES_PER_EU=n (ask for at least n waves per SIMD) or -DAMD_NUM_VGPR=n (explicit VGPR count), either of which caps +// the kernel's VGPR usage. Used to avoid the one-wave-per-SIMD occupancy cliff (more than 128 VGPRs on gfx9). See Gpu::amdRegisterOption. +#if AMDGPU && defined(AMD_NUM_VGPR) +#define KERNEL_CAP(x) kernel __attribute__((reqd_work_group_size(x, 1, 1), amdgpu_num_vgpr(AMD_NUM_VGPR))) void +#elif AMDGPU && defined(AMD_WAVES_PER_EU) #define KERNEL_CAP(x) kernel __attribute__((reqd_work_group_size(x, 1, 1), amdgpu_waves_per_eu(AMD_WAVES_PER_EU))) void #else #define KERNEL_CAP(x) KERNEL(x) From ef3014a04b13f6c793f0a5159e77006376753571 Mon Sep 17 00:00:00 2001 From: george Date: Mon, 21 Sep 2026 18:52:51 +0000 Subject: [PATCH 213/214] Rename numCudaRegisters to numRegisters (it now also serves AMD) The function selects the per-kernel REGxxxx option for every backend: CUDA turns the value into --maxrregcount / launch bounds, AMD (amdRegisterOption) into a waves-per-SIMD request or a VGPR count. Rename it and update the comments accordingly. The per-kernel switch has to stay compiled on non-CUDA backends: besides the CUDA default register counts it selects which REGxxxx option key applies to the kernel. Guarding it with #if CUDA_BACKEND left the key empty on AMD and silently disabled all the REG* overrides there (REGMI64=-1,REGMO64=-1 still compiled the in-place middle kernels at 128/127 VGPRs instead of 133/134). Only the CUDA formatting tail is guarded. Verified: AMD default 128/127, REGMI64=-1 133/134, =3 84/84, =128 128/127; CUDA -v register reports identical to the previous build. Co-Authored-By: Claude Sonnet 5 --- src/Gpu.cpp | 36 +++++++++++++++++++----------------- src/Gpu.h | 2 +- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/Gpu.cpp b/src/Gpu.cpp index 778b01fc..62e01dbf 100644 --- a/src/Gpu.cpp +++ b/src/Gpu.cpp @@ -648,12 +648,14 @@ Gpu::~Gpu() { // If kernel threads=256, register crossovers are at 128, 80, 64, 48, 40 // If kernel threads=128, register crossovers are at 128, 96, 80, 72, 64, 56, 48, 40 // If kernel threads=64, register crossovers are at 128, 112, 96, 88, 80, 72, 64, 56, 48, 40 -string Gpu::numCudaRegisters(enum WHICH_KERNEL which_kernel) { +string Gpu::numRegisters(enum WHICH_KERNEL which_kernel) { [[maybe_unused]] int regs = 0; // Default CUDA maximum register count (the AMD path only uses the override value) const char *use_override = ""; - // Allow command line to prefer the CUDA compiler's default number of registers + // Allow command line to prefer the compiler's default number of registers if (args.value("NOREG", 0)) return string(""); - // Determine a kernel specific default maximum number of GPU registers (values set to -1 have not been tuned for best default value) + // Determine a CUDA kernel specific default maximum number of GPU registers (values set to -1 have not been tuned for best default value). + // This switch also selects which REGxxxx option applies to the kernel, and that selection is needed on AMD too (see amdRegisterOption), + // so it must not be compiled out for non-CUDA backends. The default register counts below are only used by CUDA. switch (which_kernel) { case CARRYFUSED: // Register usage depends on NW, the FFT/NTT type, and perhaps the long carry setting switch (fft.shape.fft_type) { @@ -875,14 +877,14 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo #define K(name, ...) name(#name, &compiler, profile.make(#name), &queue, __VA_ARGS__) - K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numCudaRegisters(MIDIN)), + K(kfftMidIn, "fftmiddlein.cl", "fftMiddleIn", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numRegisters(MIDIN)), K(kfftHin, "ffthin.cl", "fftHin", hN / nH, kernelDefines(KFP)), K(ktailSquareZero, "tailsquare.cl", "tailSquareZero", SMALL_H / nH * 2, kernelDefines(KFP)), K(ktailSquare, "tailsquare.cl", "tailSquare", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, kernelDefines(KFP) + numCudaRegisters(TAIL)), // Single-wide tailSquare with one kernel + hN / nH / 2, kernelDefines(KFP) + numRegisters(TAIL)), // Single-wide tailSquare with one kernel K(ktailMulZero, "tailmul.cl", "tailMulZero", SMALL_H / nH * 2, kernelDefines(KFP)), K(ktailMulLowZero, "tailmul.cl", "tailMulZero", SMALL_H / nH * 2, kernelDefines(KFP) + "-DMUL_LOW=1"), K(ktailMul, "tailmul.cl", "tailMul", @@ -895,17 +897,17 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels hN / nH / 2, kernelDefines(KFP) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel - K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numCudaRegisters(MIDOUT)), + K(kfftMidOut, "fftmiddleout.cl", "fftMiddleOut", hN / (BIG_H / SMALL_H), kernelDefines(KFP) + numRegisters(MIDOUT)), K(kfftW, "fftw.cl", "fftW", hN / nW, kernelDefines(KFP)), - K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numCudaRegisters(MIDIN31)), + K(kfftMidInGF31, "fftmiddlein.cl", "fftMiddleInGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numRegisters(MIDIN31)), K(kfftHinGF31, "ffthin.cl", "fftHinGF31", hN / nH, kernelDefines(K31)), K(ktailSquareZeroGF31, "tailsquare.cl", "tailSquareZeroGF31", SMALL_H / nH * 2, kernelDefines(K31)), K(ktailSquareGF31, "tailsquare.cl", "tailSquareGF31", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, kernelDefines(K31) + numCudaRegisters(TAIL31)), // Single-wide tailSquare with one kernel + hN / nH / 2, kernelDefines(K31) + numRegisters(TAIL31)), // Single-wide tailSquare with one kernel K(ktailMulZeroGF31, "tailmul.cl", "tailMulZeroGF31", SMALL_H / nH * 2, kernelDefines(K31)), K(ktailMulLowZeroGF31, "tailmul.cl", "tailMulZeroGF31", SMALL_H / nH * 2, kernelDefines(K31) + "-DMUL_LOW=1"), K(ktailMulGF31, "tailmul.cl", "tailMulGF31", @@ -918,17 +920,17 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels hN / nH / 2, kernelDefines(K31) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel - K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numCudaRegisters(MIDOUT31)), + K(kfftMidOutGF31, "fftmiddleout.cl", "fftMiddleOutGF31", hN / (BIG_H / SMALL_H), kernelDefines(K31) + numRegisters(MIDOUT31)), K(kfftWGF31, "fftw.cl", "fftWGF31", hN / nW, kernelDefines(K31)), - K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numCudaRegisters(MIDIN61)), + K(kfftMidInGF61, "fftmiddlein.cl", "fftMiddleInGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numRegisters(MIDIN61)), K(kfftHinGF61, "ffthin.cl", "fftHinGF61", hN / nH, kernelDefines(K61)), K(ktailSquareZeroGF61, "tailsquare.cl", "tailSquareZeroGF61", SMALL_H / nH * 2, kernelDefines(K61)), K(ktailSquareGF61, "tailsquare.cl", "tailSquareGF61", !tail_single_wide && !tail_single_kernel ? hN / nH - SMALL_H / nH * 2 : // Double-wide tailSquare with two kernels !tail_single_wide ? hN / nH : // Double-wide tailSquare with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailSquare with two kernels - hN / nH / 2, kernelDefines(K61) + numCudaRegisters(TAIL61)), // Single-wide tailSquare with one kernel + hN / nH / 2, kernelDefines(K61) + numRegisters(TAIL61)), // Single-wide tailSquare with one kernel K(ktailMulZeroGF61, "tailmul.cl", "tailMulZeroGF61", SMALL_H / nH * 2, kernelDefines(K61)), K(ktailMulLowZeroGF61, "tailmul.cl", "tailMulZeroGF61", SMALL_H / nH * 2, kernelDefines(K61) + "-DMUL_LOW=1"), K(ktailMulGF61, "tailmul.cl", "tailMulGF61", @@ -941,7 +943,7 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo !tail_single_wide ? hN / nH : // Double-wide tailMul with one kernel !tail_single_kernel ? hN / nH / 2 - SMALL_H / nH : // Single-wide tailMul with two kernels hN / nH / 2, kernelDefines(K61) + "-DMUL_LOW=1"), // Single-wide tailMul with one kernel - K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numCudaRegisters(MIDOUT61)), + K(kfftMidOutGF61, "fftmiddleout.cl", "fftMiddleOutGF61", hN / (BIG_H / SMALL_H), kernelDefines(K61) + numRegisters(MIDOUT61)), K(kfftWGF61, "fftw.cl", "fftWGF61", hN / nW, kernelDefines(K61)), K(kfftP, "fftp.cl", "fftP", hN / nW, kernelDefines(KALL)), @@ -950,11 +952,11 @@ Gpu::Gpu(GpuCommon s, FFTConfig fft, u64 E, const vector& extraConf, boo K(kCarryM, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DMUL3=1"), K(kCarryMROE, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DMUL3=1 -DROE=1"), K(kCarryLL, "carry.cl", "carry", hN / CARRY_LEN, kernelDefines(KALL) + "-DLL=1"), - K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED)), - K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DROE=1"), - K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1"), - K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1"), - K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numCudaRegisters(CARRYFUSED) + "-DLL=1"), + K(kCarryFused, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED)), + K(kCarryFusedROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DROE=1"), + K(kCarryFusedMul, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DMUL3=1"), + K(kCarryFusedMulROE, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DMUL3=1 -DROE=1"), + K(kCarryFusedLL, "carryfused.cl", "carryFused", WIDTH * (BIG_H + wmul) / nW, kernelDefines(KALL) + numRegisters(CARRYFUSED) + "-DLL=1"), K(carryB, "carryb.cl", "carryB", hN / CARRY_LEN, kernelDefines(KALL)), diff --git a/src/Gpu.h b/src/Gpu.h index 06ae27d3..0427d871 100644 --- a/src/Gpu.h +++ b/src/Gpu.h @@ -358,7 +358,7 @@ class Gpu { u32 getProofPower(u64 k); void doBigLog(u64 k, u64 res, bool checkOK, float secsPerIt, u64 nIters, u32 nErrors); enum WHICH_KERNEL {CARRYFUSED=0, MIDIN=1, MIDIN31=2, MIDIN61=3, TAIL=4, TAIL31=5, TAIL61=6, MIDOUT=7, MIDOUT31=8, MIDOUT61=9}; - string numCudaRegisters(enum WHICH_KERNEL which_kernel); + string numRegisters(enum WHICH_KERNEL which_kernel); string amdRegisterOption(enum WHICH_KERNEL which_kernel, int override_regs); enum WHICH_KERNEL_TYPE {KFP=0, K31=1, K61=2, KALL=3}; string kernelDefines(enum WHICH_KERNEL_TYPE which_kernel); From 3fb186b78f7c08d303f08c9913602a4e6240634c Mon Sep 17 00:00:00 2001 From: george Date: Mon, 21 Sep 2026 19:08:01 +0000 Subject: [PATCH 214/214] AMD register cap: don't apply it to the tailSquareZero / tailMulZero kernels The tailSquare (and tailMul) .cl file is compiled once per K() instance and its code object holds both the main kernel and the Zero kernel that handles lines 0 and H/2. A REGTS64 request therefore also capped the Zero kernel, which naturally needs 126 VGPRs (the main kernel needs 84-90), so a 3-wave request forced 72 spilled VGPRs into a kernel run by two workgroups. Keep the Zero kernels on plain KERNEL(); the cap applies to the main kernels only. With the cap confined to the main kernel, -use REGTS64=3 (3 waves per SIMD) is a small win where tailSquare sits just over the 84-VGPR boundary: MI50 out of place 85 -> 84 VGPRs (5 spilled): 912 -> 891 us/iter (p6) and 998 -> 982 (p4); Radeon VII in place 90 -> 80: 1078 -> 1057. Not made a default: it depends on the kernel being within a few VGPRs of the boundary. CUDA register reports are identical. Co-Authored-By: Claude Sonnet 5 --- src/cl/tailmul.cl | 8 ++++---- src/cl/tailsquare.cl | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cl/tailmul.cl b/src/cl/tailmul.cl index 035f80b8..12826691 100644 --- a/src/cl/tailmul.cl +++ b/src/cl/tailmul.cl @@ -88,7 +88,7 @@ void OVERLOAD pairMul(u32 N, T2 *u, T2 *v, T2 *p, T2 *q, T2 base_squared, bool s #if !SINGLE_KERNEL // The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local T2 lds[LDS_BYTES(1) / sizeof(T2)]; LDSinit(lds, 1); @@ -357,7 +357,7 @@ void OVERLOAD pairMul(u32 N, F2 *u, F2 *v, F2 *p, F2 *q, F2 base_squared, bool s #if !SINGLE_KERNEL // The kernel tailMulZero handles the special cases in tailMul, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMulZero(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local F2 lds[LDS_BYTES(1) / sizeof(F2)]; LDSinit(lds, 1); @@ -616,7 +616,7 @@ void OVERLOAD pairMul(u32 N, GF31 *u, GF31 *v, GF31 *p, GF31 *q, GF31 base_squar #if !SINGLE_KERNEL // The kernel tailMulZeroGF31 handles the special cases in tailMulGF31, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailMulZeroGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMulZeroGF31(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; LDSinit(lds, 1); @@ -911,7 +911,7 @@ void OVERLOAD pairMul(u32 N, GF61 *u, GF61 *v, GF61 *p, GF61 *q, GF61 base_squar #if !SINGLE_KERNEL // The kernel tailMulZeroGF61 handles the special cases in tailMulGF61, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailMulZeroGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { +KERNEL(G_H) tailMulZeroGF61(P(T2) out, CP(T2) in, CP(T2) a, Trig smallTrig) { local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; LDSinit(lds, 1); diff --git a/src/cl/tailsquare.cl b/src/cl/tailsquare.cl index f9c73ce7..09ec76cf 100644 --- a/src/cl/tailsquare.cl +++ b/src/cl/tailsquare.cl @@ -92,7 +92,7 @@ void OVERLOAD pairSq(u32 N, T2 *u, T2 *v, T2 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { local T2 lds[LDS_BYTES(1) / sizeof(T2)]; LDSinit(lds, 1); @@ -375,7 +375,7 @@ void OVERLOAD pairSq(u32 N, F2 *u, F2 *v, F2 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquareZero(P(T2) out, CP(T2) in, Trig smallTrig) { local F2 lds[LDS_BYTES(1) / sizeof(F2)]; LDSinit(lds, 1); @@ -650,7 +650,7 @@ void OVERLOAD pairSq(u32 N, GF31 *u, GF31 *v, GF31 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquareZeroGF31(P(T2) out, CP(T2) in, Trig smallTrig) { local GF31 lds[LDS_BYTES(1) / sizeof(GF31)]; LDSinit(lds, 1); @@ -992,7 +992,7 @@ void OVERLOAD pairSq(u32 N, GF61 *u, GF61 *v, GF61 base_squared, bool special) { #if !SINGLE_KERNEL // The kernel tailSquareZero handles the special cases in tailSquare, i.e. the lines 0 and H/2 // This kernel is launched with 2 workgroups (handling line 0, resp. H/2) -KERNEL_CAP(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { +KERNEL(G_H) tailSquareZeroGF61(P(T2) out, CP(T2) in, Trig smallTrig) { local GF61 lds[LDS_BYTES(1) / sizeof(GF61)]; LDSinit(lds, 1);

0h7R$T2 zfPmMo3{=A!RCXgd)Al*NNW{#LMe8qvBQi7FubD2_T<-PdQTTM_(ic{e!mehV%sjlI zh@h=|ATMUaSlchO7zxN(S9f8DoD@yJdmw0C_dd^DTsp)Jg@5i0)0qm66oth3d3?lc zY17ntJziE0NqNz?s8))TR70#O5&#|2V`*|r3eXjz?FS7INod zE=c6{Aub7ct~ER=FGSb^(=*OtJ&CXo1KG?K8!rEQ=)Ul5!{d_QS_NK$pC$MIKliyR z-B1p8B;LO%I(tpU3B^*0K{r{>I0Ab=EE0WUO(6dXbdxWzdYT0nLgP-+r7yZm(y``p zm=oj@{Fji`NUGtaq=2JwZT}7P>fHH4*i+79@Z&q24;FS^Y6Jjx8?5?f`S6qh`Qinx zC#I|Et9$vYtX@YXz%9lq^0{qxqLr|GZ;_wph*!nia1bJZU%R9}15ry)!YLyvo;{#3 zZrfZLvPZ5K>X^nitcZW{F+u;_ARo}3DaD{{^nX-i>vUY_$I42Q>Oe%-7x8Rf>BN9IKK_gxiw38 zUB>EaVB}>NH7~i(pXdTPuwK@jkda~7@XmLAf&CMP=C5`-yD#vMtr@>=wU{z~X!OWU z48Du6C(42g{8n(e*3AZpSoHRwfU(2>pN1)7yXzo)3>I15mqBhj0lF;?)0}UwF~r7EIXY#u|Ppmb-gVvV$NY>IuO<@F`@nlza?=y#&$6q zEGA(aycD2p)LAUV8h6un37Fjd@YGnu26fi#LKo;mw*&E9b{hcKPQ2$|TUwQ9&y|~k z8b`xLhTJ*=-3n2i za(CvP{iVhJu`D~7aHA^X<%fCGi22GC3_T_ZU}OyFWT`Nv8vSe=ZL&u4$ZnsnGx%INTG~pH&?y=BLz-FRE?6m z4i&Id33&9+m;!Q^cPa^OBl6+5Nd-PkquhwD29i@R_=2x8jkmRP9`@S){PcpWHP^E$!CN?g33l?4q%By~NzSOlA zz?A32opw)f_;}@pXE=8<8ljBG>t4j3;+p%uRJ&I!LXh2E9P^Wgu*Rd4<-w8}ZSQrm znQ0TcI?x{kzsE40_K;q}eN+d~`dDM&#QdB`rw&07<~9y6zPl0Kf*2Fv2PR6aiydfU zK!v^;=#4LVX=XzEbC7nUt%(h1sTVAHIAs-$*TVYNj^mVU?~~{07%uJMZdO43=y2g+ z62Cd8rM@D)R9Toq2ehKn(pB0fd-*4IV?f^8JD|2L-+T62LbcA+elB?U&=^OH--Kl% z4baTQTx*DJHhR%f6|;(7;DH@7nBA9cYUKDzXdl|v4kZO%@%rOZ^pa9Vlra7MrWGoI zLEt^?BW9{J#lNCGXIH#{Z6P!iu?KJV9c(VGfQcs41CSK|j7jMap*{IWl7DZ2RU8g& z{^5rz>}z-w%*IiDXztIza}8ApZJ;gTCZaqFA?*o|+k@?`*m?gid#sFI8B+pHxg>SV zHCK!_3}F8fQ{bF%qgfxrCR_?_lmj0lPrq%>eMw|>l%%7ln(_{RGa-?M17N}&AM`#Rl z{mm&grCc@?t~O^W?(ZF+Jo5o)uKuBd{#4>M;;?r~`Yg>3BI{!LG7K+lXAdE6MtM+R zLinKu^vJCqRYSh6^i(=AG=S)EImv)Z)!(Vc8G}G?22``^*}xvSglLhY05HQNw3wK6 z-(DPX;AGC>tXDorLFj(Y=BvH8mSb}PeuFdRkpo@9>#GNby~>XUo#(1DHu08kA@()b z?9wO#bN()&pb7~!Qqxl#0afQ8RErfu2(2-oPO(! z_J?8O0gc49qe0p19r4(qSLWojeuH-&^GnDJI=--!W34JUR`cUD!4XsRsk!S*tj$e3!O`b^72I=L0%y%DIRY zz%1Zd1WLf#S~tKV1?gA)JJ*}5T|2ZwND)4~PP!9$z@;aCcZy9)532G?DVawVuUFr`EiWFQJnzZdQ%3|-eu4!2IR zQPhC~a_=fBj71y@{Qs!fV>%X*3A`oRYV#J)lcOyF?k1!<^*RG+`;`vOJ9|rZHYWAW zWUc3B$h=6Jek=0MYg>)H?vHjDkeP;y8Qh3a{5+gg|M}N-D?7nWFzqsg7@%jH8?4lW zp{Ph{&AjE$GVZriQj$3lK7ehK6o=V!0-Q^}%lPY3$ttvZ;xrJnyIoGZ?xcuKq`#l z#w{b!x)1>MUDa=PV}BPsp-solfkg*>s`r>qJdDw(<)PFHS9c1u;H(cKUIp2In#a`m zBdFWRFrCq9l@?2w=pd7SOTE?da#Q-|ta-YstQo?w=Trf6dGFco9KF}>Qt*rEjfOmA z{VysHSZ*b~qEiy#AtGJ9wM?B{X`n^Hvm|oiqH#01XZEsOn z@A){rpp;<+(c7VzLHQJ4&{=p8lDM8CS{P=~J!~vEY!zFhTINjxn7^MUwj}^e?m3d6 zP|w;!kLh>arfcv+uE>P{E6#8jS8+~r zvubw~DeM;emyEM>{JQ4pp`~RCsmP5!={-z@kBP%WvCHIGCw! zEpDGcMxY3x6~~4k?s1-SvV~d7zpZ% zM#;Y3)4!L7P&rFL-qkcUn( zl!5pZr_YE71(@x6x}_y&u&*KMKfZ=%DHY?=N-Caq&T~o4i6~qi$vpu4-fNE!qsjO% zYH#(p=-jf^#cc`lQ-tW!#gdbm-Jk8&TB_acT@yi2ObD4wZ~sXNQK;RQ#KzC0wY9E| z3$^VtXk-6^+Xd|~5a>wGPVstNil};o^Zm&PwqMqs6W`k8*wiyc#6IB6+)I`s7Q%xm zX6C!(BFa4xRYd08euJ8K0_fr{PKj9*#_g#uXg!|wNxG`WnXrGy>dHJp#W`-56Pokv z5>@IibUgG25v*ElQPo}vK}?8m)ZLn;19{LNbS(st8>OoFirh$rC1xth1P6VpcOf$1 z+fW8Zm5xh=>i)(X^`Fu2llLe!Gi)^@etd!EBn%w?sjCxk+g8XIt6fyZG7!I`x39F3EK}aT;#QdB)&JL!+3S&#FHVuhZ@8De+IEHTbg#sGQ`%_r%zK(o^myKm9H{RroZg8s?);8nt*%a6L1|sM6MvbSyRh<{0s(!k$11us9{3jSO+dHP8aD!duIPTs z-R3adspZ-}{ej|05@8G<4z5e7cBEW7_86@_wRuCNa7gXLGJ27lV|d6hvkXd#l>~d; z=AFHS0kiRyN%z>R6Y%Yd0Ua;310ti4u+kE!p-BeIZqj{jC;@f%sjDY5fthHH?K2Nl z9mUf8>hZzv5FQ-WOiw?6dF7;Q5^QC4ff!`Lj!`}y`aEKH{0AZhM;(nz#T^Ro?&uFSf$GcyuSEly%0TeK6}d#0-n@8PEyIqv3u`86e&zE zEdc6ur)2cHhN2n*9`z`EiGD<^5&kw)xlw}UK!ZcianHNZhvS@Wgzn>MW#;KpTzWfy zZN{((#H59spCrAcCt>iB!szAAt=#BU@HwExa--pDdPye^$m!9biL25F=F4fI{_nA1 z?r3!IILL8L28AYM)F>4cyNH~vx~XqIvvd$DU{-{ly4Tv3i+A_kmO12JM^`OiYJ^~H z+F#Q%UAqTtLKu_O){+2{36D2wmBTw10(L6M+2|az+>o-%Ktez$@&H9Zy1&_GO@w~s z!`H(2j^1qfE5j2Ua2DupyZYRh@A}kIbj&)~ssVk|*$xy_UShSfASapCauMSG)F@I0 zbEd2OpNqO5pQgq6XgVTwsrfGWs5pY(EWj|JmLOZI;T&r#<7LKr$FY0>YewK)SI$-n zgw!HcB0BfnW;7EfB-K4Xa>I_3jE?3p^+wNsKTrTw#$!ZjmTqV`9f1PoafSu1nwR06 zvRtPN$mrlcD0TAX`9|r~6%5U7IbZJh%g4ZgSC{14E@g>-nt-!!Zxy*DYlQ8+k5)6S z{{^mdVjX&j5swgZg6pWqp`&|l9c8bvo#rh`$|1bgxv;I!xDrt(F?W6*`!-X`3N(7* zL^gF}^KU~EE7}R4nZ(CVzl$7gZ9l7czw4)qoCqOU{yX+qtV-AT-iGAakz)5gI9QnC)`D-XzMyl7O%O-dTMP4lKss({k=q6;!El zN@N8LC@rK{;=7dr+0Fs)MCV1c1kM4n8z*B!g-# z_bCsv5>?4xzC-q>HA&Cr#uScm@M$Yg5Yl<%%ZV&zZ}8{K{C1#i*jz~B#e%vI+{u2G zxtAJb;Oks;vq+PRRlmGrmUDKePg?kYh#gw(++45Iz;3L>;wbX^PUi?&k{3JP$q96N z&;mQ{mQA?>OGg>4!1krN;?XSi@2A*{5onZO%bEZ-H3sns*p#B#1C{$c9yji@?5jkT zOBZB>X9@i=wBcZ6+>@NcQtd>oqC_%Lbz>72_M!I^O2}x)Xa_77X3^wVFW)}BeY_+`}wRb3qum)JBNI3OvKe?L&nqLKYh$4GEAIW$SRzhJV zX)V61Rxmix{B%db_M4_*gefJV0!9HyZNxN&)?$^5;V+tb8JHD4gFF0N_AL#UVVkf1 zpwH85=-7TH6?G+aWyqW zgW`4Q<BDN-XUc^H5Aap zG@%S8a1)>!oOfU!&)cFMDjFq5;U{l!OeSrKhwQW8P9W4vr&={)MI4a1FKSXwrWj+; zz`ywm6u2skI-E>_vkTm=fW}VgtRoU8H;PGXD~IjS%8bcZ9p20zK9y4~rml%0-0^rq~rZ!EGZVpExOpB z(o(|%uG={fSzol%USdJR>~NHkyIHO&ut9pc5@3Y9 zztd<5O3J+@4p|9$9um}?A3WYhiAd4c)PAHpwG4=ma%G&94fiH7D_bA1_XXm*63Ujh zulsh1U#(h9GitV!c)FrbJ6qJhvwXo1^x7%tM6V*N^Z-8)QNew1mi${h+wen( z(Kz?u+{c2mHmtScsxN!~KD$_iQ#;4F2>Y&8kz5bV3Woe_%7Wh$;M7nYIhx^VAltJV-V;{L!j1cI-v2d3?PG*IyBu@nvGX;6UDlJPm|LrNe)?}XGMAs2pstxq9L*b+y zb3!*yXq>C(-ONyUf({Ic7N6RX+0_y^35&%w9aSszKd6BTGdkgXPGC+K&w^C=SOHmC z3}caSS}L3jlG_AS%h)_WZ*SW9O%1WxKh`~Yyc{FpPTjI*9syg682C4%(6(}Qh?I4ohNvTQx3X^tMsSz5l0 ze|eEhLL1uL!)^QFX}g%RH}KNmyyKs^)7KQ~pCmGbl7&wcFy#gH5hv!|%=7QW|Dc;W zEA%5l(aW-VK%xECAWIi^Z=0`Lyu*w~R{^Cg1aiE3cmzfuC6ie-6BP?8nR@tx?lO*) zc)gjqU4B)qxGalBb_47G-f4o$vdl>cy-^1Yj^&v}t$%UBQWS7u&A>Nw{FP=srm1)| zcsktE#C5v%NR$~P7|&q2&;O96R?v2$Usufouafgh z@iDhey9&XW%qf~NZd6fbK}v>$O|l7NW!9bw#x_;pCz^*{!naCR6aZ*j{=@arRFYJ% zBcNZRTTH@6IW_?PeZSoPJND-2Ge7ux$TrT6 zbZ~?USbZWJmRtp9T~5Q#13Nk}`|8OOS;eu_u;M7$3W}y~o?S3}maf3DKGNIe^JGb9 z`m`X1(e!BB)`Pl_xRSq8k0$`p$V58J{)zj(#T2DqkP$n8MB3_ znLEFR(t^8;I&PWmmNaYB8E)I|PIC+|K7~iz85y&MJ^!gO0;(wEe6$&7Jde#_QHg5=9ZYcEUVa0AbT(K z{8lI+d8tL-~+ldDWmSx?KX5cKugUn%!+>L+{SoeE5Fe-`|*OQG%z?8gMRRU zA6oXZO)$vvOU@A%g7&xRVT-q?3e%3`Qd}2`r_=N{G^$vEQCXtSiCXg34!CWrr7L>| zt1dY7?=@C8U3U_Nqhaih)wZ88VhcT-ZM-}nodK>|lZfhW`(=@ToKa^Ti~Xu|Wjcyv z$bWi>Xhz0SC>NNx>wBiAcEq(}hI#&OYlpET#M`n=S@W475Ad8K2s|CgscN8*yCg{+ zMF4u*laj=B27Mf1NVJ+Eo3^Uwe+ zd&i?V|J;8|j%Ya&DkzNwS0#%+%#T%7jA#d7GTTRh-pSc)JS%IQNkpBYbMk$G>O z5)(03!4>6Y(IjFPD!N^`eA(`I-1)3tXz>_NYhJYpQK-ZRd} zC&JOuOj_?>xz~uJb`# z@F}$3TQTq?CLLZ%$#;nr)Kpzmzf`nK{B!5ypX>Tx>1`curPtF{4LyF;N4aU3MXF@dQGP7pibhmE?`7!g+jL3R0YORbw3Q&H}xOyJRn(O#^-CF<*fH( z{=#i=RhL zdZ^QQ0&BdMn%6s&Y85ilDXX4?j zfR;j8Rv@fNW3=1Czd%iwBcyJjUU%n7KH%3^{3r)JDR7y*w3G6-1BXaHFPb0MVn3iFzwT45yzhS_$#)2hogOs`qK+R1P z4U3jqBO7tv>RO-D;z*eW*1dF@(fmSI1A&@AjJ?o@gV7&C`boy-G)a%O)}UL5;qcaL zIdvlqtaYb-6uZ6yB!-J=ga?ozfWFv9OwDD$**M?ZC8#Q(hc@piGq^nkui$4jc9Q*d zHg7^Qbt1#l95GxjF{x0i1qrl8q4l4aU}p)rtt!#00s)!{h<*1knhTC@Ctq zb%3SSVp|k%fwFT!rVZpygXrnUFCf|WVwvqT<;|a~g+N20PNcsF5C-DfB<7;j&ua_y z54n2&4`hF9&H_N*qk|D9Pcd#JrG1F23ls;$CKS=EARMBR92Swy2DnG+XYUF5L9&5r)%`03) ztjkn6e2A_Lp~*VPc(Jz;BwRvSCDay;KW@a~yJ(T0m&FoxgzFje>kJx^Nv@dhow#*I zQ;(-_a^fleJ$z21Kv0XLVivk*tPppB!0a&=9b=j$Zx5+i{&WPsufI>~iGk%syN}cZ zU+&I?ynr_E;b8#y-4ewR%xfLJNU(+ydTO$oDH2wE#l{f|bE_*4SL1EDaq(OBiWnPXvqof;i=0FP@ zp4~ci(0}M@TpYZezGozSA7ZBS9vy)1h?dnMgi?+rIsD`NE{J87WQwC$l z(rn_ah6Kc%A?tDsM9AH%j3qLo4o$-y8Y(%4F)YNpuyi39pG1mw(5Il1k@rL7-Xl^3 zuVaGI=zKmv{b)>Xy_$;B-R=->9dDGNYOQ45#YWGeyGD^)GRx7bP|hA*Q}>!lOnvz( z5>rZg-3u)H37R>47C!(%3o?5Ol1B&MF9F_sR-w3K2Ykc z1pIjf#{~Eg_Yo5`5zN;<6m|K8^i9!1hGz8X(oJjRn5Vnc;7N^O$%>ak*uNnYAm4AE z8K>#AJq zb*rHGfRSP-*Cz9Wc}0zOP$-*})8>;-AHb;02HO;=?yejRqQGtYhOqdN(b0A{j@CC^ zxM~s5J+0D?BqyWOWEi_tl%NjnszosvDRQlgw=nC1Uv)6J*Y{TjKPm>0X>PnQ^RM36 zeiciXgH{PN9O94P#@w9mE+kI)1GNSPp@8Mi*MQPC!k1ER^&?!rO!Q%4vUn&TKR+Vo z?({HEXz6AboBx`L!X;1|5G^YRfz#d%kqm=!^VN@);>Y*c&~l+%yn%c#5~mu5^um+K zV43Gfcsy3Zos~(64MeEyoQ=d74CGNBkG>LItyhBSdJ6$kZ2y#-8ni75(lDpfWF1GM zMfy`-t)0Y4;xN|=!Q}vP)(mM-3ZM5D`xn z9w_MaRTgW^H=y^CIIO8HefS=V)_OHJbe*@}yCF(2La|~yiPX!!t(AMyl3#wpjsONc z2Q{KUe!NbRhOkJ*)=Y8o%AA&YxXqUNbYMn90budF7lUstZ~~m>gjFOOlMQ zfD5B{-R->IU3+bd8&bF9pS?n=)?8=O&dD;#EKEwR&&%V58&I370;1>AY4V$iXevy$ z6Sk0JXM}nA!>P`DxqN4ji()$MxFFa1*)k&mQ+73$A{Vgn)oqIfI_cNO)t@!K7uAz` z-6@Mgc;!X^2m-yWgu}gZPzMvn{oJeqB$4H?->--Sdlbk>rC&OiWj7qV8r1YU)WMdutlCLME#m+UELt z2!U|uAr30R#ec?Zx^;h#_OV4Fs$Yv_j$*zOLNpgn>0aY?B0WvZJjDHNQo#nWLW1%G z{q83px=qw~QFlkb`{Qr0u($6q?@#2V5+t}vcg?~W^@=JMA-Yjfpi>j9gKov1iv7vF z3my~xyYsNIqsLu0dvh_Sn__XAt)FufppV#yUK`m-`mZFY=fpKX!KoiSm@=KvF~ZZ; zKE!g9CF<1V+h4n*D$6frWcl4vO2RJtYBsd(QO9-Z8{vu~g&p?9iabH#x$GGbuWg+C z3SKF@)lsyI`YZE|X+8X3zLX9=*pNxS5NuCC^@5Z(JK?y*M>%A9u1U|cchN1so&%{o%#RgWT14G4b3Ql^3ZyQ!I+7Gj zUUBe-DLih~!m1qNv7T91kk!X+>Q#k11|K@zHw+0ZJ~1(V?Z-|`2H;^Ala)%wF=s);^U4-&43A=@*2>&`d%~@;*+neO=q5E@wZj`_7H0>67OaifK`wUc3%|5C`)A`RKN7?<{LjLCC4mNG01qFR z^ypxJaxrMtayjmpZ4w$$CLZq5GqU(2xu!u%(IH&5C|bh+&ZYKF>*Dd|IIz)0+Xyi< zie;YFD#OhGd%KEAaJ|O?q@>VU@+IvZlr`3`5RL0;ISiLgf44U;h3Z(&YhRWvf>_{9 zr3hUXr2~3H!aJ&*G9tcx)R8KuAsuK^9Tvhu2$xfK{HZby!i3@G6m!g6XC4ouhUJjc zAWnf3%IvZ+^LcxRSID;ANCVr@z-RjKvJF0eC9y~}3ql0-kx2!9mm@D3$j}lo8#H%S zR_$QAY&)%2!C>9kU7fZ-1<~K5Vdi>i=icB<0psx~tl}Sp9MRh|QOfHWW}&iOy1*$m zhpA5I(4(EPnI{mSs`4BY(N`Qk#gL|4|&y8D_U#UE)o%+2~p8fpig9{Tj zS1yIv6)G+TR7MB>uoy)P7`PHm^5meR*4?NPzZ`~jY~d*BuxOAev7AHZY~$7BJEDz{ z=gNe(upugDA}@jn;Sndu4AnYI{!0;3sEoxkObh~2En`ZK&_kpH3S&=Rf7MLtP0aV~ z`I>}f0eLv5Z0+r(;#9!ZP+|*=1pjLeQ`vW9R)~%*u8^DWqyYaetr~BW)xyU?*y+8$ zC{DLsUEJ1f8bD$y&N8kO##-GlPQdN`HlZ+wf&BL5WK(p~P~{d$M9PM^C!I>9f~N$z zT>7seK4M$QC}yJPaaSp#=**&`lz!w;PDs^6;#O)|qLQq$lr8}m*u^e(%TJ51@FQyF zE-qi^eBNYOA|kn-yU@>umO$IltE^OqOwMmhpr4CYHyS;vA?kqP^a(Hv$b1SNxwPHx z$nbn)N@JMMn&~Nu?$(_J4hO@tzX)aP%_*Cc`jK7Zqvn`#36MOf;GtH0ZIyMo?axG}BDiSa#s&RTd1HV|$w_ zvJ1uJH@LTrhpl~u;;`W-7?MOA-W~&!*AKN5@qO2WOY_W8#K6qr{qO0GI-}gqT+be5 z5GVdWBpt%GxJR?DQU{m^u^4Z=o`HALo8+NhZA3~0uvBbEvAyOgczMeIen)FMUuG9S-Q^Go1WB;xfP^`L*J!_x8Dr^9W0(g|N_;63U?WB<+m-G~T!`dyGnlu#Vpy zTs%PHh5Hh+b&E5s3bUG|9owb!u^T)@BJ3tf%Oq^&iM15eq`LYUJKpwd0_Fga3o#DpS~7QNR;Bj2iAw*n|n?${Y{ z0@lYDW_G6NO(r-IZDzQQGy)Dd+SmXU?BRuC9~U!wxC?Ob;3zJ zel+0QG6tBD)^e$fvfXu3}^nZusXExV=`-y&}M8yrl7k~urCasfbm znd&5ydhAOV{1~mVgtU5_vi+6A5VwbvEkxr0OMJc?v7!0HTdb}!zKHLJA(uxXVC~*$ zRLvs~#Eo*ycz^|~Aor>i%F?_ZS;K?+e=rWE*$8| z{{xwe%#=dMtoJQZHpj4);laoJ`4qPX!`7ts8}D` zWJBl&w8!1gL~oW2SKB`Ki}a7wY~LcPiUr6uJl*;%RK!nxZcl*ekx%Y&9v0RYwgu1r z8DNrd#jxub-m3GX~@HCdZ-dQfO&%j97AMNRkB zL_VZL795md?K;1`rTCFmbI6lMRhZ}KnF%42wQHR~|~L#K8Co9m4^cgy98*|;-TugHu+8{QV9%(0uF z-#vfizmxgb{npbXd(rgr7g+%to~JG*GP5VwK}=27ne4c( zxiSGEm2TQeZsdQp$vos=lI6h^Tw@=HPm_HR&w(D|0ito>XPCp_3X`SY?NUkAP zw&o`Kw4V+hqMx4;c^(l@(e&RfXRPB`&1rN)AT5H*xN62-alr!lDntKiliKGs=|z9o z*n)KU)a%(^eOpObVUqv-Lk#~Uv||C9(%H;SXJgFoMsD{y1cZSmP$oIDJZr$z9dj_B#)xWwS`5>Qfy zGo4qvy1$opCT6<_)_>iWku4#rk){2!x?jMb?k5E*OlwN7f$xU9PF$uosWu?I0}N0T zaJp>9LCQ}v@XWG;ADXfLbglcy8&Zkq`^!v^fVBOZi9+rr@li$Q z6?sjC)Vrje3T`ShpK?wZR5S3LU7_Nx`-}@0@h!u|hY~{}^}(5zE}5wt zTL=9t_dSI2>6TO7aFcKla;WzYiATe zRomdETeqbU(8&wkH3GS3F)DWNITB;(t-FUV`84`!c0A11@LAn^BZ?-Knw-*cCx zS&$ZDkt}|q8gO$2g4$m#L=x!V)5IcmI??0J=6pu_IQAB){|Fk9XgqdGaZJ7X;7G@b zxg*wF~uT4MXakaJQV-Vg+bP}womgA0%6MWbmAXo1MckV&`cxx9Kr@8rIEpMz2E%{9|ock9Jp5)mPw9(Wj=0p{W zhz00cghOXr#q+^s$89H%=1>h4M7te0?_hi{jsF zlL(o<1ItOokwl>SPRYt8`{h#0WRxhf0r}45{Yd3LX*rQx;H-S5qfpM-yd8Sa?rhTG zOz|ICy3~8lU`W6#56Su#GQYB$hVLCDji1eHuZh=bGg%6L0~c;R_NJ7isQ_h?ch;m#2%oJpy4LX{*fffu@(x}951#VfFe=={$Ch6zGW$dG)(&2TTX>xAv) z(&lGPvd?y@?Wxl9c|YhreJE|Pr+CRV#;fG-UzgUc@lt?ML?8v=G)z_adG4iT<)@pD zq9@OflzDk|r<*nFD9~jDchq`rOhg>Aw#JPJxeG|nZ0Xbj+&Zy3h-waL@gPm>83=RG zi5$&DlvbMT?lW?gLdaZoidao>YBCxq)Dj%Z!^h|AZ9oTX- zZ+FTOJu8>J)B%LfH{AQr65|7Vr}1Fx(RCn=)W9c60%fXxX*M%uQy-uF$DzS?^vpW) zE*i)~RHaxk_hQ7g;6>E-0)|4`P79B9Qd`b|467|1e5N0sYW<*bs6Uh$uYdWY%QJ3g zCePuCAjw%60*mwOlZC1doyl35gzAEiy=>Wnr!l4MPuchNe)%)O0eap$CF6e+tfG{x z;=1&{SG%V!@3@(@kHr81H=x$TZ~up{)Ntr_688*99!A)-L1$R(>9e@|#-6HNwcm&U z@}V+skV-3uZ_C6{v?wR`!Lf$HmG?iPhXU2Oj!uFb(wv2jc4kgZ{w*l)6|W@}-9pH| zedNw?3s~b|k%}^*ZlbL!K@G~{(d^g*-ujdpq}rs+55ec^SjIbNg2y=H58})X%@B1H zVTptpO8IEj8qfMR540fpP0R^L{CcT?p`S>?S#?zy=!p~qSyqX5hpClNLg@|ehxh)3 zRS3KE(Wxo~lZ>>#`HtjXt&0aGJq1|Ws%xOxYuLW(@3inco^mygupCfR+KpSf} zd5A-g6wl6CJeA%}{>VRxBWk3hNwC~-yN~-CLvmM%<|zn>qjnno1_?R!qwF#>ef80I z+15!8Y}F0%PWHOlwNnLCBR*rZrnXdw=1_0AAdx5y%6{VRjwA_yL4x(c>z#3@YHn~I zH#huajsNxa2ZB3H%o{97rx3UnBOa7Qm-E1?H^;yMq!?SypdR>-Yme7!C7NLTwZb~l zMDujHKMZjRv$jw|t7F=->D-*OjRA7dj=4r&ZLwFbMdN~mi$WNH>!x+V){L^b@Y;m@ zVH@U>bqC2V2Nnqp3irOS`m%1q-UucWDlgqd!5B>TjjK?99krW! zgh(cLt~a4kWDejt{zN|A&kdN-q}MJQ61`o=if;soa$?TyW|xTzjcM5x7aj49=R=Gs zQ(RUPIJ+!KN~rUz)k&~2sVA-aO4#45e-KLq+890~LQulZRnB%BTr#a~QDx;=|FE`# zvbsBl=>(<^iO!DRO0gjGFi<{K?j>L)!5T`*7TuJTaVKCjm<($E0vdb4v;w*m=+TAq zlWJ}g63>+kgOpzo&aCeiv}+mmKn5`PY8h;%;Ajj# zWMx5Hvq+qle>6wQS09621tg??h+3D`X!A^m;G0bijEB4F^vYP#8A6V=L>~@eS*RwJ zk?U#%YecQUQAZ}r>(T!;KNPIiH#@cjnPJm&xi(*lx|Cmdz1-~MptI@=q(<6RL>rmf zw%>XLKo6z%Z(EG8Tp<}A!8Uo%T( z5)t+%!%dxB62^Yv2|K(vD0{YG-Ue!y*_%-0_KKcP{(y!vXx7Fh0c!wmk+hej9z- z0$BO%Y5ZeHH8>I~x?X;+#lQjC??v$h{WI1Inmn~qHDGCOm@rPgn`HiCIfq|Ip~LZD z$;a}D9MUf|*|gv)XQwK+`C$h4h*Euc372_fhPOVmoe)AyTvJAgzqz0d+9u9DvxeIB!F8i#381 z;|&)Xz;1m6*9Mghc$~*N{;BflW}X7;-^uA{zr-7r>-DpdTlTW=p0xW^3!T;Qi6lEDhk9Vd!CHS~LdhC6>jBAya>=GlAWkuH%6xVFU)R z`ggbl77^}&jx|_3qndu>|o-^8cn&W+JCCiz1=nIVh) zFDalype{hu^>Yo0V@Rc5NH_9l&pXP4sR#C2bk;7IR*^Hz=S7f!dpls7#tsmSo3Z5! zr-j7qkCFZ;&yOZrXie*=b$jOU5xuEJ zHm#(d8}8j(g+jEC_D=QRL8aXe3io8pDmscCrf5P4+|j47iKv+f zPW{4`6X~C*Za2svoMuG+1cy+esZil>jp_*GG zARx)(qAqBj`#9VH`oPFR8xLne20+aIV1dSYr=jnlfnGGb?pFoUsdj9rku>oBy=?z^ z^m<1N45_-@|VzNIUKz4Ny6TVV>@v!`1J7u)-((5+?A zmTsJG3gb2OE=dCsd1RvHM4-A=)YzUy%T8PQ z2|N3Ey`thqy&{DMep_#c+XXf}dIx&3SefBSWl$)Ovb>sHb48dmHQX9OcbdHZ;iVp* zk=7@F45|*^``v*2`SA>3iuKFivc47i@AOnJ27Ek(H1NWdz z#!kqlWT-t=io5g~ri^@pHx#!;LJv$Xh86_w6}JTqzg-2%aEVc45k%)Bc5-o^YCimT zCb<*rPwWWWWur<%XC%1_tym}o0yiZWs~@}@qU?HpSIn;+aW*kxOc{gmcj`G?2Pn_p zFl59>V0!_?`R5t%)Xe9iwe30<9f(?ibcO#Il(7$S%7}>G&HF!e;hes2WkeB%Q(*Ks zdRV%g4h<=j7(Gd#CH5mWih8M?S<*4S3Tob!eyYF_*uZ{#vPqC&PU}wpAU$5Xho2ga9&w$2zTWpycN>faGxYOX0nMVKR#pN(bo0T_>E(2y2Sz zCrofvP|!VaUIP+bC|_mkWMIPx|6m|ad%re*Z?6gPHs41k_o)2J3z^5VqlTKa-&$|( z7Qm#!ugW(_JiDQ$ppyJPZj15t82)CzZMN}%ze540CJ5aaq99y=sGBbN_vs$zsQSbm zevuMPm}Zk>uT9_0IQ!)8-_fVi;BEd*J*%<_W(2Uqxeut9RA%ZTBAh4GPH&1@t1D^3 z+5)2rJ4FDRY%$Vc_$i=mkR6%CxoEaC$t0KO`AV&Ru0xaV#tC0S&_Fla7^E+l7Nvn; zdlFrRw0zPf%#mx~wHj1Ep$(|gIY*Ge>&yvWwyun}dfP>Py5CcQLMnKilI{PT|41Ur zPc3ru8lCF4yxPt=&Zu)*760?}T_}t927RJn(v|;cz3UQ*ayb09Gx>Worg9~o!8jY` zLqP?Y?X_L5ybS`1RNP8@B(g*sYuk&!=kR^#|M%&I9gN!`a*S|1H>2MIwBU@o722AE6VAnhN|Aa=`sFd84i&AI49 z$n;olWpX6of6JZH907jp4u1Z<`o*zXk%n+rF5`^#1Sm>@ebPxQ1KTE*3{^Bt zn)-N8g^oOL4=^?tiY=D?x|r@Pg}Vm83Z6zz7Kr!aENo2XlTZ?vrPLa}?pSJS zN)L9RcG0rqRkOZha699MbY6~%EsgPg?3?Pvg-(LOJE*`b7_V?N{j`iXKj5PoYtc8@ z=P3EKW0`wV5CXN*2=JH5Tyy4I_;5);wq8k@OQq16epH&zDC?Ettw8V{z}#xms90Pn z9Yl`T%Y~(cd;_D#KP#N6myb$Dfo)3;mzy})xa!f|j-qd}gifT8 zMiB$j#AKDO79ih61nTNb`%lD4;wUpRw0*h8DJro~>`*55DBo73zC!N(V=!uI-0=EY zc-0`rI)wW%MEscp=S{6H6YerP4#$6cG{u$rj&6M4?`lg>f_6P$K1w14H>`wMmhza7 zByCok^Zvrf@>7AKB!Y7qO@)D;xV@-fo>rF@KeuQ!T$lMtx)iCM9zXSY?h6LN%k{|- zr1f)0Nz8>|FI>wfoCNEgIQ=y7Z^{a>^)cvCe#eC=9bFG0T%OmmzULO7jt_V2oL{G0T z1QSj%Vy+Fcz(+liQs9y^#Us-+RX5Fgy`*^8W{c+5ZTY?pT7@6CV!eVho6#kB`o&!p~#A^EB(`=oS3#2cJLB$qtrDOL0( zXB3ko=hB^pmfC+BPrikeb850h4zR>df}*l|la|ICzd0#-?UrJb094R-uETy-^?c$lpQDf5W z;vhMfF#C;v2W3>~nt&6s;c z!zL<-el9P;bYp9Ip&-oSZ5&EZN!Ge0s8g1Ila6>c$vx@Ya31sC)0vot5)|FPeB`9K66H(V7K1CYY^?cXqAVvp(QdFRx;(qSeA2&=iPgA=G#JvBQ6|t^(spQNS5t!&g-wct*H(@)Cr&n$e6gD! zS{NfRr)!^o!20^x(mv}l>}7F~JRybITwe{G3UIpb(d3x7?Z=hSgbH(51L4LwbWg1Zza-3Pxz(^cc)!q@MGLy6gnUf|Etf{`DT zMg-PK`hCBC>+#D!T|VX+rTsRxxC{q#4WdhHMUI|lrLRb3TP;25ckLrJdO=snw9QO< zhIGLJ=l2S^Dt3bsy2KD(?*q87TlUKk;s)OZjjD}imBvmn5YgW*!`g-goLFD!D4?qt zc$NvJpCItR`~7V}p0L6s@7Bf9@h)EhlAIEcB&IM5X{?cfI$WgWtLstD3pu`Ln_BHe zDUrq9P6N7T&mVZl3?6CY_cjh7w50dqX`4}G&@pO_Xcwbl)iU5(iLvmHm#WlO^kTOQ z^!N88E2yIZ8^+I!PH1?0G~jDz#{vLSY4^mpglBZ7Sqjasp;k7hpWi;Nw+cQS$$)_} zY@$Gra=1q~aIaXrQ}zV(V4vF$9K1mlcW0Ksm3`(J6;02k5@Z-3Xa^=vm0nK4GA0%fH6q zZ7kiurk*&Rl9$qnIN^8xHdY&4WfDI9YG=1bigaoWa<1nQV<(}VkQ1Pm!$iQ_N{1N5 z5-FNPR1tdWq8dUbhXMGbx*=D)llw_}zJ)yiKi{p09?~R+J63!KfE|NgMeGi7zr2=x zJHNtyA^fwj1$Wx_<2zGz4XMP!G^G=WQHvon8b|s$MTJp%;|Ib3?si7qxs?)J zVRA4WmV(M@{~?cEqMV2)+t|li9}D^1ljk79I2-8duMoE@Qf3Tqqb6fq5ltPKpVzK?AlNsHqd!}hb}Kv(XOC8!r)7~E1zosjW*;>au&guPi;*j z2DP%KvP?M1=C#`02_>cPjSFh(1J4;WHexCi^C93n^ALw}-Q> zgyZd!IzaLXh_KF+B`kjjYV0Ck3!2KOGl8{FA(Q$4t_E^M+NiI%flZ`R`eX=Q#edp0 z94(fC%g?)E?w9X{ZA|S~YRkF7?Rp3dgZw!Io=}(I){dy4Rq3}4|Ep4?)})7IrSHJs zt5^K91K*+lbkkFt?lpZz=jh>!wcxn2pc@Bpq986n?)TrD6`J+wJ14~T9jdYsvMsp9 z&vm7i?>HTO6*Dyd!29O_F+k40pfbwd2+5W=4T+a@nj=DXvi6$)I4a_4jGdxfo5JLf zZ-#}rZP2!XTtf6ba}k8s3o#=n^VbiemCnHZp$x}q1mlplJIO7mdbNfLbPRQ*<`j<~ zL#gb!z?L{|gp>rQmHAJY|DVB6_>dXtm6VtG^O7Hd*m-qHn!SC`EWo0oAm9`UT(TpK z-e8tlHFL-01#E|};T_DfaO3+q%FJyhXU?aJdlSGI3u6TLj65f^(hr3Pwpg$-%WK5n|mvWbk7Ttnn5d0iMinFOXcqpouD*x!^UGnYAPdao|u6pL*-P=O~Tio%G*f5B`aW#Jez@ zgMjjNN$#ELS%0{0(Ox`I)h#p)&20G^knT{YM)nzNeghRJrai)6-!6g-55sdXC!1E& zSPs$2To#4X97#O~Wxo0!d=;V^{Z^`E6y~Gqd!04a;BUlHI#OMxs)N?0Nc@1%V!4qN zq=!9foh_Odz+2J}6L;qgIK- z%C`I~unsF6fqI$kP3~lp7}<8M)v0%CFfo&pl-|v~!E1TUcU%_>RXn|9ZpDn6v~014 z|A!EX4F)qY;$y}NVz}z+_jH7c1pf)!8z@t?ml#h4k_XLVhXd{8Dg4NkNW<}D?2gHkW zM0MKJYnNFQ@|K{9zRMj;oIW@19m${^ADT?}=T zCP6j53p{(@VfYrDne`Q_<#>Yn$R zI!ZV_ige0xK;4u zzkKPxP{m%$z2Z?hIQW6=RNuVhuVa5z{Pf}iLJK^D>=vX9G|-hLB#8lQJJf|;ec&!-3LgjWD=UoW8BKNDtB6_&AU*p%u_I2IUMJWSie_ArC|ryO?eYOb)Tj%l9PIhDv2 zxuD++Ku<~ycc$M)LqK+5p!qxnW_|No$4QRNnbq*>+DEThy)J#LmH9M6+36**syR6=}88 z-a3v_hYJw!r2Jz9disGkppo0^jJ7gp zUF%7IJy7IbQ2(QP#MWgZ?TObt3aPC~qXdHICLJ<&6_JEehLAc-by#PUVg)V%yc~R65))I)LzBwlF?~K@rKEm9 z4Jwn4Sa@_V3CW7&u+c=h4%iWRB-GalLw%8c8tn(9fTjx0NuH9bT8V!T$ny7KTMRyqD)5Hx`R4moC@vK7J2skO*i{^R&05W`TTERrWR0Ke_^JVs*GVAvL7zzXIFI?J zj)jk~j#3UAOb@|oQxMQTQt;sU2C!{#Og@V$)zIqv&ORl z-PXf@Un&-1NC1y3jerTreY|guVqCq+Ht|CfKd2zzE&`Czz$Q7SuJVBB)+XLhsDyB_ zx5e@!?Z5@^*6`bExaWe11AgDEb7Wbn#+J!0A_k*z@`?qRlr$rnv>nU`g0A^9vqaaa zn&@x77pwey5%K0<<3DKc?Q9`(bdeVF#^gbV+qsARx!ZCix;Vja#1cv!Ox_Si&czvHQ7eZX0_$I8RIADKHZqOJ9q1q&MVj5DPAR_%Fj zxKP4z*6(&NqmYMeC(iVA7taa+2K+%W2b9V1-$l_G+F*-bhpG{Nd(<}kQD=fa> zVc4Yc!x-K%y{WIlfKO_re2@iN?o;qm0dK9Xit~qNO?K=^;B@*PG=M}lu!e)VD5Yq_ zPfJ2k!uaUJf&BqVQ31dwE0b-$Lm9Wc?&=RpH^*+=5lSY!fF z{SVx@*c`K0Jk8Fu0OpE;ft@dphB8uxJT}Xqb+-cY&mOZNb5%dg0|Wy4r4wM8C4%*_AP9Du(CWRYqnuphkr{o9 zmGe_;pWuWAK-U2K{kk+uFdK~x4?yYZYX%N9Tcr|Rsk2bg2Hm>9`z@vPHnDRn#^UcMzn~0$>m-tReAB#)*xym?0#`sR#rYs1|gGY8?8Khjput!|HyPC}oa9Wmsuco8%42)`oQF@FD}OnQoFGnJ zOoe+WNU1EG#L6lAed4$4<3`S}ax&)L>It6rHqw~=Pt2d~ksKD@Fh%UcS-eS=QFF8H zA{O1E=S%yk`SLm^z0gt;ou5ig>A$YPo5)i zJ)B>jp2fo{w6&|b+XSNvO)`Y4&3^q0EeieewMwN+Y)WZQeWbl^^$@tGfKnO+IvG_m zP-CRTRY=cj2BI_B?w5|)@t0cLuu0$0>}zl`&5HqE1C$#Xp^nBCL1_;Uahj9GJ~y7; zXGiXb1z)W1f3?+tl|Bi9Q4mYAg8A%H*picK0nr&A zGr%12*4KpeEtObH$ap@0_U}ydtkATk(+w73hij@FJ$8yR=4&{eA;W_FIAAC)VRm!vFVPqZ zSC3;j-&DWGNO(a*?CYZ(f#^JAwpF8P-K->0CW$vSq>E#-!5iL<`5P8kPdof(zR?K0FLyHJrVel&V6VM*)T=>Z|*w`P@lXR7Q)KFQYdVyeJhgaFj zVLY8Nm^w7bNv8=((K-}Y>sqZ@K{-`Ncx2E014yj8jBXA6ea5%+n2}N_&xy7Kr^Yw~ zo*qE3)uJH4AMZEJABOgdF6YRad16}ve2R;a$f_#Jr%by$Xqm=e z8h>**1&*JV_+=OP(Z2>t4!o7Pv)U;7sBBOfiy_rSWj9j`C}pr)UidcLyKucVc1_Cz zMvnvx4-(1I=0t*rsP*Ll5FlD=o^8>fJhpHzz%SKK%%(&xiKNDPWk)$$Na?{{v6E0T zTZf`cEIu}~aEeWiq`YLS{@;|nxhC=km*0JzPI}FwxpJrp>})hEC#3d|Y_`wn_&=+x zazuNbiydMns5i;KDy>~%R}w;F@7`I#ti@B??R0L|8CsJhP%>Sjq7?gpuddAME55~O zM(hZ~%IcDdW-V+GPZ3H|ah@4Joxv)6OMRrVxiJCdYI%pmKSZVO2vLzi7Knf8!<4! zQ=f8bF<(wvl_OGaX8?2Bgq(1JTzl0;OSy#Dkgo!W29Qe$rP!yyet^iX!A6d3HY6SI zd8ZBw4xy-U%5#4Mko-NN!$`ce)B2XObZ3thP8hRgaaI%8`z_^8vDi(9v2_-57s}sr%WAG9 zhln3p&0WEaM|bJfYG9dm)ihkkobQ0f$pY(^DI=sAIm=A7JB%4HS>4zhnYY?%C`1i~ zL5L>~+HeMBV^svK_fB=n2xLM=43BZ_(B&a$eYIsNGBVl<2b|tWSFF_u1(`ShJOnPH zEHZS(Lry8WPE%g>#ctRkCWuyTJrU8q{y8EmnYEpYY=GbWzRfU$>9qYj@kcPkBbjHK zkAD$4XmFq72GU30utFU(Hq{~Aq#RNm%Rp|}vDIh)x6EaJ7Q)_pmY81mVFEkiRy}Bs zO0GwY3|uAi+qo?wy6PrxEeq?j_d~r*Qh}7w!04oh@M}ofFaM6^7_8JIfM?5fE_{&2Z)=E@y1_J9HzMd` z^+N#?4z5>-AH;_WaUhPQ6J^O3$a!)8Rbo0#vZ!TKb4r7fUITurty-H-D&yOXTISI< z=N*S1AB=kEiT7Z1GGw5M6p)`HYP-u!p!0RoTO$n!I|vkHJ8gQNkk{6A0<99+JDbF} zub;dd3cQG0(JhhwsM@ml3J4IkehWaEbQ9d#BNjqeA)jF(_vW%gEy)>S-H$DG*lkmN zw=LRkq$F|ma1%3kTxp>sPH%4`*YMIyhPMJkjT?{(F~mN(90eQ>Kii7ZS9js^0(g|D zaNb_$mAt@6a0&7yNBl0V@$OKly*YC0Utv$Nm_#&~j-n^|a=BRTDpa6y0}peOLKhcuD!*c&@Ht6f@+dJ*c~y_15oA-sR1^}fP?!G zm(iGjx_;<&=TBug@LLTLwjwz{4ZD0?C~=0@3lEe@ zorv2sfj;C07>z<;-<;)Ojr*&d-H?7mdo|VC|Igk*OIma^1we|MyDDK<2q%|7NNMAb zd4yS%1@TK?yfk*iw#inE6I=GU!3dY*7SL4KM;k9r!S$52|Bs$=4rFw4k>QKJ(6TWG zpW{wf%p_}ojo9N&YbXs=H2lmFYystoN(9#N;%^cOd{v>WK;9H2*8@j?;C8C!LX9zK zt;+sO+EhohPed!8F`=9tbJ3ME$XPHLOj8kV6FJjJ-f3crBhDN76gF;y-ZWYxCdFVlYcuM+lpH7(WcWVSPY&I^ z_4U!dg5T+Gfo_h{I5&^(@qr^vj0U3x|Mcm~mP-|_$!Oa_*`TMEwlo+(HUZCGiKkbB zPQA^4x^sLNbD(RqKHdnbNWag@25=UssBFh~w&CO-4joZV89@}`Z^r?u)NkLA0{jq% z&xxV>j<+jmmdx+{@*-CymFR3I22WX_Q==K;jyg5R_ovyVP&0qW$_?28#!2WMW8v)` zmEO;ewSz}MdiHF(#QKB$NSl5PgyhcMcHqJGQht}IWZ`8vZxJPg^Pm7qEMU7bNvfkq zw@Zw`1j85=iO=SCa;D&9p4#JAB-w!;r8{;<0Ey#8q0`4j6gNqueZG3@R!B*mM2hJt zWb+}RReMq>3?v~UcK0&eahNH5x2kf>2BF^IB=0tX!y3q-a%bxa{iLBU;2OXHV9}|R zhhU@>nHV@W2T**6r1R|Zx%MdyT@yG+B335Eq~n(r-31BmX|*5tU|!$ZsqOmiO9p9VZl|q895<$SlugtJe(0< z3}Cd>R~wzqf(fHgiAzX_j$E*Xp6QpWIY6-~l{D-8emZudz7aS<0f-n&aR$=j*Z4E* zg2MZ$M-)Qpc6<`RRa#B4QQ0J1IjmnF@yq48=BV$GEXz{4TF-T!gh9T4-D;@%*Se!b zCYU5lgV3nB5{dYz#{ANmSgDKTzFQMEtG?%=_MWjLe_2e-Gc;t=GPLEGZc#UCkKTi! z71tS2PsHE)RTtv}6%_ioa+k|)4Y%!?5s%La$KYxM5_juBh5N+YGls1=d%$_*2>e=&Zq@#DSfXZ2l+&#?N#1?(UJ23_c(gQ8DUQUHkq5T+-7A;k4w58_L`0US-42X4mgw5lW1tWpm!>@vvPh*tWT0Zv&k zz=tZ9z!r+6W@vO1-1B)fnyXs?wjllJ99*5}1 ztiN_*@H0^40LkKbapiWDlO8;}jK{}8Js+CJBp1sdYGypS!|jMWlS>83N*Pz`1+5E$ z)$OeDIx|9+}>J-sV>tuHByRXgIPL@*5uN1gz z+d(eun|w#q;ZKGn8UL0BuB!bN2m?SJo&-A1#802cu;g-dJ=#(#Lm7mQ&AK~ZRFK>O z?C|I)0^j(M`H&%d*;(86-dHv_cY8+spJRp5{lyb(|{a1`NwSa{#J2-=FN*-(qBTdRm ze7~GbE`{{+%89D|SZa<35MV8L>cL@=neX}=^ADAuSe2<_ZPbSTeqEq$?3V*169f^V zXdLGdYPhjwc}Z=+oFCQ-Vk?xoA5z;ZaZ**hAP;v3yRhDWM!abl`8G3FzqglR(YBaw z3N=;AU3~Xo(eLZep#4L9WQDAlykKUq6hKWBshyi}_oDJtqe7?YR1PLNLzb#EqH0=?Y;J)V!=*3^(F7m;K zA9s>8B(ANUQH+39d9cQQ55H%L|SJ_w(hCk)Y3jvxi|jE*&nFs4gwa&jFyQ2 zb)KxH@aA-|IC)Qh%zSnZpd16<<-b5-86ai^hQMn_8w=|TY9Fwp-OFsMKcZb)!rUnp@|)jvWL!aXRo}NRL$(@LZ;+Pi-~OziqD;&c^7R*T*7|rOeJ`WTl~%D3%Q*iW?MddzGuag z>^L;*)62t|KK1Z+@>(#*K+DpNba|@Vz3O%r!BCp~KCsE%39+6DA_6`8vJh7rx zg0JYR4CFWf>uW>ZSRqcJRljv2+c4OHT1XT%6GjP&097<2S2YX^Ir$B`)I&Z;HM`@} zq`V(L%5=79kgozUV zFo&si2TkJTy5rdhNvz*~!sS_O7-{*c2d_@>d_eD^Tdl~>(p;kHlWbCJN~gLqvB!Rm^4n=KoUZhJ`e1MQ3KxDQZ)|(Sc7vIgQ#z_*BHLuG==E^#Knn5ZfAgDC%`?rdgzc8#BRTSe?{sEU4E*EAe;G>;wGti)ad zP^`*2^1t#@s|a!0AFB{&@nRrXrqpL_5yLpIiPckA>xFM9i=p2;nxVygD{5H>QtmLu z7WG}>k4a8$d@`4!S}uNU)`k&n z`b*^qs`UOq)U?K9Z4P`RLFzS2*f^xpz(nChhCwUvn@+vD(9S?4@S})N2J!ENZifrc z=`;B$QGay7k)A$zji9xCB%D`WJkPuOxIe=*5yp0;!Wf98D&2aRa5&piTT9n~x|XqXP>2&^Cf7{l0s%@gr@ zU1a)Q9_p>#2RsZ(hWCo8f4`nd_>-&@w?=Z72pV~8e}N;wO&BgYQ%z)lQD5bYM0t+` zfh?NCdP5o2Q2`Ke!#em8E%?6JfA_ff1P1dHOYWlBAA&TA^&F1bGg=RCyL8vmxdNWuT`S;UWX$wfu+nSqKD5hFDmxoa$9g4+F=<0{&QJMJcfH$k87XjzXDed7hAe zC>I-&p2Vjw5Ct?^vbllknjQB5)Bo=`MjW5Q=urHq%%PNg$`2f=h*NX@26Gn1SEi@pS zrjND3;Ph;zo_|CuS=Qa@_{4-(=ZdIRH8gllPv8wE@+%s~@8J8n;J1WyCll|XIld6M zfu4Az_9cTZS?If?SD@acwahn?UP#;Qq&J`O+`HhO%s@LIEy$KGG4_>~tw2{^?-)CW z2|JD(z1Z6y*sX*k@I*;xlLT3GTjG@R0O%#)7^Qi@bOQ)ypYAf;u_;HeO zi1e0N_A`eAm)8c2%eGTzNNgBF;vRg3G`pJ3ZudQbbvW*~yFcOEV+OI8v$8jx*BcGx zM}|@v)Qk+6_h#c=4M;k0N+Z1quibJXYmK9i3wyma=w&15UGK=(9s_FP7j=<8)Cv4J z4jDQ8jaK(`BP79*5e8#nO_M*7UkpZlk71cMufm}OF4%mZ5@OalvCdv(5=;NzA$@YV zd&;!U^fIt+($}MpYcph7dzsA?irtwPh(4FyZKodPJuXAtleU9Jjt)}WmY&?S#`58L z|NS<)>Nyb#H?X?ED%gUtI0XVS8Gg#6ZqG_gxpxj1k7}(ISD2b+LWlO{cw%%XSgd`+ zq5+UsV0s7HY7mq09rxRqGvWo{0#7B3k8F2yfEff{dwMYC02>aCx5PO|PO5REmNEu` z6yVDk8cjLQML_XFx3^$!gUZvpfdEtMjYaF7Lxv+1f(WwVKB_6Kb@nZaQxpEnioLVi z)fM-dNwBF!;b}*x0(XW}3Cd`f1!C>s3sZg6%h*)(R_;X2gD+%9B4_ampCRic6>nlJmo&p4zUq%BtN7$am1w|tw|-s- zBSJAOQ&j;8L$GS^?%U-p_uS&_?9**9lsOI%({9YZsV}(N^;5IDsy6h`s3I7g*bRQj zfPvDzs@^}S?S|^5&V>1{5uN0{lr}0vLzwhFa=Tp|N()ycZFoaDXD$6SUQQ3KRnfPJ zv)8ZEDLm}gAq_%`%o4?w;y_@7b#syQgJbcNnCq+pu`;@!oLa}U-B$oV#v$)h%N~S& zB_nlyZ=ZV>dvqE+Gttxm+~48%il$bs9wrD-0n$<~KCe2X+R0_hTzitAHG_?S@E9x6 z5>7;LAuvzOycHGEYdQ!PhPKu-<(1-#&VZ*DrQ6Cn>TS8r)FxFR30oatBVQRhv_g%3 z3ynz;>&rkmtFxlh6Zqfk${|)4K!&8W6Bh|zXF?3jNqB?UC~#lr8o z5HXcho(6kTCoEP*&QEfV3;rXZt74j5UM8xgPhR*3tHKV=Z5!<-036qL+<+Jc>2v=Rl%`dN)%$rTN1Kb z1wN7MhxM(C#%N53YTDI;>oUGIG=B&gfjDFQ%l6nI#OX+OfeuqM4{7Fk!e77GP;mci zlS@@uLIZkJ(GfN#|3T%d96d(emN&Eon`6Vf)9#}D$*g=7k&~aXcN`GrDfk~8G@HeS ztV#Kr5~eYgXUIBlBVe3H);(?cu9`DNpcMt_J!w@r1OnhDuoS!A_Hwz@hvCcJi?HFc$=#Cp*RHvUQb z^8AeKntA@pts1cO$!l+&b@>4JS4%KWEBhWtu6w)7Tl#UbTL9!=ep8#Td&A%^&##^x zLsHzPLj#$ykD+&VWzaN=HQbLU+3wHWC_b$=#RF4O`M0*T$eoF({U$vp0KdZc_hV(b z1859>8+UeTYxRXN{eYijPo7+NL{;oGwt|hoUD@>qAtAY*LO6nFK~E2@jVZi4q6?iYvl zbr~`h8R99YQXS00$kbyuU?XaS3{U4OGFy8mrU$Yw3-+4@(!h~)*Ewfov_yXOL-<49+Kkv!k}pvoYG4F`>%i0xHm&hA#;{ z*Sr4a#U_;5Ep)GuSP9RC`3m0wodtDCw<;2wMW~5toav#~xMNOxDnxGI(NFyCC)gkw z9i*2jpUh+p5Ul(lEMG$aL3cp0B#_Z-b>P)>)g6U*ut!dM*B{(`+l;ercz@*7KRUn(&h&Y z^OyX0f`e!Z!A^!lIX3--)x_WjP=a_L75%2H$Ph?R5ta23SqGgQX3@Yj)+@vAytwe!f!xr0$#gXVw@ zXYLc8Tde|Y2zr6Di(k=)vPKBK7NRHt4Y6P84Os_@Iza!fUTnM&__IEw*hR7i{_AHd zpM8+04$7O98)ILE4t3xLl0zxmdFYyozTnVG4X~y2Esr1r(nUup`Oy8Ed73WXb#t$N-rO`n3a%y zzhkA=wDbqnp}3cXN#vu;N5q1Wh&I3k&@JI}57_i;6o$S%&uA_9-E6VYp5qx;Dw|Qn z4@pkQzJr&1e`q;7&!Ow?m-2){SxbWR9s(-jZ})+{w_=~LNBo(9&GdE+;gg*wq8{E1 zboNPCWG;N6tNOyNG0ZPit<2w)>$njZDn27ET%*+PY9Sgo3rrpFHTo_#1CeR7%$!cf zCTyj=8)KePJ8FG?6FuKk?(WCw5&c*Ehe0Cc`5=%bWG$uv!KRZ$G>copf8he+c}sA_ zaL$8d32Bcek*8*YA(wF-gL*4IbI*s^j)8^hX#jPckp}HdVYh(!rnr?Dz?PqtK8guV zRXo((vF1q#H?ozS6mRP0>0X26G{-iYr_9#px{F8{QLWBRGBfu3xLqc~3ol>4armUP zFNLR&n+oK8FJn?_p*g7kYM`TWzdqtsp-3mT%K;gF$@ zYmOmpOT13 zzwG0M@^DPU$8dymQ*>r1@h3LlDMsj7AVA@Bx9M9@Y+QVW?`KN+I#ZCnO~4V4@$^c( zXzwM|u083D)MqJ{&c;&Et`U4Gjf4r*OU`(^Dw(M#v9Yr6V6owtIZ5yLHv~kBFW=1C zajf+d^cHy9ECoXV@i~b!7XY|EC0-RX*0(kPxYGG$=QHeo*lP+|uFoFuPV!Hx(XZ!s z^^+r_liF;mD@X2O`NBsHVt|tJ*&E$cnsp0wIo_^pFoIzXWiJDn6cQVy{Z?^VA*dya zvIx_ZO5I((pJ3m4dJPH$l4hUf4xW{_e)D0+{8i9BD zxn;wUhxv;@u=0-0|5Rz~oDGw^t3%pI@UR5ZiX8+&kd*dF@6x0*1zIK7$%GB2p$&aG zbH`4cQWrVCQj37@Z)(WDXp)(OWW;_=a>v7wGi#_(nh_6uG#cVLB-Lz=ppB=f)fXkD zelu}M7&IQD2ME|yI_zvc{GBYWuw{mOKrl!4NEJ_@bWEF(SnlLwwRe>r78(yOubxeJ zUP8O6)(#;PcUM}}ZY_cid~c2xbz+qeKqVTtkZw*Ue4*|Ma|lrr$QW(i^LR5q&Im8t z9zXRnfDXb(HNJCV&;IJli&J-uUKkjCH86A`!kRo_%}l)JcHifYImi53IyVIOKAy8h zE|0(Z3FM565AQ>h>gVNl7DWuyTSx*B$RXS@k&QSD*%U@Rny7W111r^D3)Qq=0RVwi z9%3M17pgp|q*3|7^_A+6-rOqu8MGyJB7jO?EdWuN;D2G!!%t?}(N%-MCz#)Sc3UMr zM0gqAsh}6P+N_lU`VqB%8zdcVw)3bm_JzEntbYYrvnx1SD`nkF%K+Hme2n5~V3IEV z#?-;$DTTpIYe|la&Y`wAiW(W>EWA-2_0CV;)8~<;!RzWNb0xv0jVO#1_VQM$(sUZM z)qr44Y;HD|w4YccGa+0N{t7dcB=%fCekV@bUWxI1o~{^nQQP8Jtd}nBiPcQvnZ^xK zRSQ+Mjw07%S*$gcnvK8_o{5K!VQ09@B$cYNVZxCB-U4}lOfUvLi`O^XD(a)-T!ToEP7(AgS8DVcs3^qGlZaV7isJ-lHumh={}AK*Kk zUq-@vSq&PTq+A$}6=Uu{dWQ^o(K^X)GbN?7Knr$pW3qD5C1-2|=3V z*F#AWzzW^T#KVNfD@kQ+Ar7^F&JbEbj(_%c6%Cnz9R41kcZ|FqQaERD^`|c= zSM={H3zJUMpG8?9TOo9VT?YBIgNm49UP;4|432Ov(CZJNmJ ze~yA^Nn2`7pmL%G-h+lI!=i_~SXIuhdGzj=u>F`7OHGoshg4^gKF8sUpxtnj(+s3; zx%C%D+tx1~6+FuI=}<-1{}{UswuSm3Hn$rGLnws8FNK>^iZ1mhmW{MvCq_Ns{Hth> z*5isxP8u2#HJ%2Q;5KWU#LdjzHe=ph)IuaML2Z~y>9fIkVI0kORix*?oAPZobC$99 zl+N^k<_9s2Ya4w95=f1TGYy;4t3~$SFWlo)^}arbKY~jF-k6skNEa+)al!uN3}O!@ z$tkaR;B%KD&0OCgI50v@ZbuH7(ptK1Bad6@UfV@9)?~ddX($M4o@OrqD`%t_xyuKO zfL&!CkFrLdkaK#KP4s?p@CgliCC6E!>!O=4(U?hxnpmm45V0R#ESFt@MAt+?)yhr3xvaafIG90i%t^3fwvAtB89^6a7YoiPTa{1P;K z>_3O=7s~1mmEPW*`T>$h-g_6lo+KH@6le31GusczGWd_Ly{1GPYPWHp-1 zUB9Z;I&lWoLw1F*3{xS+Y%z4QO87sS*UF@>)|4&pR6?zr>w+G!?WN*Ns67z6@WM-GO(&O3K;6Y-hqn_;;UTS@NhL6bDr-zWAF`u z6rJtt zIlu@x7K4MB>@~?`9eYlslOj3JH1;NY3MMfzVQD?9M@R&1h7wSN@7>=nLJ&|wJ(JT3 zG_*)grN@B8B^a(EZ8r5N5iE0wF>Ab0`?dP@5h`=i0kR;rK5cmHVUn^DxIBW%>%BiV z;LjZlNEYHU(tg_LAQ%p1#lCFabLLgED^1QkSJR81JrrLOszeh_-W0h!?Sk!#aati~ z!hfpSS+tH0t|6=CH%t{*Z+qeykb$*C_fJcVjH+Z>5eA)3!!VqaUQ5nSf0$V7|4B_+FGiL1+P4ET)L-{D6Ec2l0T_&10?`zpn@DL9 zr!C|Gf-{=E$0w6~BsAQ5HlLd>dy|?PRsYsJg@&09xOTVxaLJOdeH`()h=>b_YiE$X zCYqqHl?^G4D+qeJj#|@YF$NtM+nc*xKU<9VqQWCK1o=`m=*-X0OZG+zgLb}%7kXPe z?3+-d5E`SFOd4J-FrlRm(5`ZQkzz!ct)~S?!q6ON`U*RL&N~Jn)lm=M8WcY-8M#;I z@@diFO8cNzT651W1D7emqj{WhRroo`dTel(T;<(YV+FIck_V#lAITi)Y11OS&;K5Z z(tUDRn!Sew2(nZ0WX3;Li3P4#E->_H)Env;0MNflhSwy|uw256>GN8M9D2f%ALI`o z4Nd8%`)WZ18t=Svqc3v06&?i7ky==R1{xSS#o+TIt|w_w4FX#B$Fq4qUA%s8=%ZzY z$uDvJpw{(I))M_v*7I+r)zZE1)Gx!H)iAL(%G`3{99C!gm!rl!SCP2(M`?mJSrWGi zvHMEd+kl<-%MnoVisOILc-4?09=9wD%aj?n$K5HvW7&5K6>QgQWAs(#iNp}ejvoCy zYV(PHZtA`JV5rL*irEG~v538`p+)|8@FPEpW~(2p93hbIJg=){A&ZQn;0T^iD^I6Z zVZE1MyEs8WJmbXL*P`^5Bc*M1;;UfmaO}Hes9M`x@A7AD>tJDpv3$%L>!HiP=WxEY z3Ja`9(asT(Qb#jGbh`YI$lBwS{{l6vB+|$EW86q--4YxY_e*F z{J^$q7fg4+Ah3>Ge`<2N`|KBs(jB`2Xv00$pAyAr<>zt~jhQHT!d<>I7Y%$m=@#8l zX!ZF&J_ogP*C}=KJq>2iCqulV)C;`-mmG_S*Vqut_;0*r`D zAd^C=r(R-TGSkO|21Qx>McW#s`6Y^--6k7`H<$IpXb4PMobZB20M927{?iDQ&Nz|= z5hxx(UQOly8qNqRtPwBt0PkB(%#IF3${1gmL#vJ3xUBy2kfz8|5v*A3o|-v(fs{`rY) zCc){-)dT7mj)n`-Vt__uw-mK^lQNB;ZUWZTNCz{`KYN5K|Kz6?`eL*w7M+oAv5lQz zoA%|xwuhpW9LWv%3%PH0rTWHzpkx8#;Kh2dOBoV|zxt^hs3Rd8lDtU92u(JyFtvEp zSI)%G-70PjKE_n7Mlr3R?_8yE!?P=1+pp`MT_jT;1Uak>C;}cqvpCGtOk3#z z@Ofvl7_~5^%{f8jnpgy4pTXzHbKqvGrVhtNG@=VY*uJaa=OaH8u2|%tsiDCbQ^@UD z4F%^{>xNfpIL6F_^A`g6P~SM^N4+swhFjmkZ)S2F=Pdha7AB`L+o_OzftFt5C#jI$ zPT;|wm0mcw-yg5gR`soo0<)qvm|IdDAzII!wTYT?{8>xH6BlcXA>o>}r5o^O{>BmlcNr3~&>L$zMc00d(3?wINqtdTFEih=d_n>n_Qe(WVbCGV_xM)1Q>`GGuq~9VTg*9((<`*Sxmc10{F9#Vyh5uVnm#=W=c(HV3#QKWz?*esIryWU=dlj8R?RLLg=y+%@A`tl5WdMP^dLpUK#j&`rW z8T>dH;lis`heJ>6ih`Al8f;KK+$^R*SEzl`LD^0=ojZ5>Zqhhx$?j^Ap2J4r3uGUQ zi?|klKFSp$F;VFkZaDRTMFP@Mt2nwrBLwcDT-Ap}k;+L`_AYOx~ zwk_sq8TKr8WD4$S^XlwX;;E`w@c>OgvcD`F2@d7u`Wu!Hdf5JIB7tK^nj_<@=`cC_ z=aD%pji#Na2u17CFvo-7K{t5+K?Y{J%cDOD?^B~;=w@~ya&{KLq!i0DiVi~|Dd2x0kwR5cE{H+$X`@KBxE z&|db@CvUm9eOfiMja8Fp-WBadl37picusbEmAQ5NR4oS|O`(v=uZ&KiSKipkmoUTk ztKwLw&s&tm4!jgCUmxDyP^(BiK0a^x+*t&1e<~n-sG4YG|9pl28cv0|rzcHGqI`EC zWREU9SPuf1-BQlr4Q=7!Hvceqcr+{N$)JG?n$oA}*!=rEw+LM&^DwQ(7NCFtQt7%y zqQNp(R=5WLt6>O>`Za<)*Lba>$vck4lb7?A2i5OxG0gU$d=;X`ur4BqUo`A)I#=YS z$rjkaR4of$O!>2NelQ;~iJ&e?u9M|h!tn4dJ5>CQek@HEB_&zCTtvaV`CLc6pyA;o zQA4umI>$2PaA{>^ksew1Yd26kn~EZc`i>6&Ft|cvfND8_>Y;YeLo#BJG*08+-#!Q^ zNm|DM%V#B{+883gmCZIr{X^$iS`-Bd2 zQlgp-hy&CzP@y98_jRPxXYCYUp?)|j`^1|YFJ#)&6$?Ct!KH%}w@5g`e#G|Ay!Rp< zhJ6ShJlT?LMP``MxKmk)AC$XWPrZ3OQrYK)ElIL5k4{yJa*s8 zR#<#pyU(l-MKMXnHOqI4)ZK83ys}x^+j_yH44*b`nvI(fHqg5+?zprdMpE>qhDnc% zTQ>0cSF=5JCE5GWV{O(!>9F!BimZ*Rv&`BdV+ED-cuN3s&W*XxvTDWc7s7^&D!9(D_VjEDvi9KdvwRiY9k zDdv?5Qym{RfCo_1K|W10WtPhq{$J_^>tK8O~2X5kAn0a_4*kF%FvJpy^_vJKS{NDs1AT4 zC-3K*^(bdF#yxjfpcg%_(+T2im0&3YpmHD6K?TJxk&)*jE{;`WBLi8hD$w4NGe4H% z;*zy*8(7HJy_z%zxTWynaL4HOk|$OmX@`aAXG!&Kqt>mKfsY${CoT*ECUUv-;l#`^ zMS+5KbA~6Isf{v`_K%vv~%Vzu9C2QC$YInzZkD(u>Pgm zJQowjwL><69|HB>sctsqpmabk|939WuV#EP-SDe1))ZIZ#gf>ok)|%Ul|Vi(B(DD+JmL<4*Z}gLo_qic z>t+tw_D_UWQ=R_YX$9pu4#zkHP{6aMv~rN1gzjVO45_;ioOibCWHgZ6d42YvqFHhy ztI68gh{e;H5-J^kM*#vMl(n-1R&TUi4Q@F2$;76|*7rwdXYo%_{HcdXtM?5DlbDIA z{j8@&!02+?`_T(p@|sZ|TSXCEu@PO8r0VkTIiWOLZBIaLnYAxitUU}txUWEe=?Qt8 zn*NU#Iq)}Lfn8Xp0`3d zGwt8D#mxEP&}Nqbvv%9WAbyUqmL(rCTsrbpiq?o31lqFDrd;OGK(N5mS}BJpKV+{h zTi?uu%8(GStT)t%JE)rr*s$S-S^-$IsGm4DxkI_nZzDnK-k0VQ$alK6*IUIrLJ5Ue zcCr{an2fIP3FX7KbGr|m>CA;MmH66E|M zot|bH1LX$b2<-@(AeUsbdKL1G^eLKA_y~ZJN{Ic;8|(|-@jFPWvVR95OJ-Z-P!X(n z@!P+0q`Rl%lbw<&);go{`iB(@k|cuQ;>*IJ!g572#3kch0`QNzxYhkB6kn?VSCN6q z4G{nUdiw%h{N?dM`TQ7stw@BU=)g6fW{{d zy8W}Xzu8}fE%*lrDob9B);#RBsc1^K*l*}K?wXt2>iE;=tG29#IGnG`ahaMRD9o`MYl;t z;f&72!tXcywKn3`TxthXW+Bz4UV|>(D3xmByqr&KE+eO^oq25SD5P8ka$#_y75}LZ zXWZQOFB_C+SuuF7evST16jeN3ya?)U(XKjv@`7R_OFJQ7&EqG4I@?4RcSy6Wl$N|G zxruH=+dDh0Cw_0{f!88GhDmS&TFfV-6Q{5i)r=%|7=A=oLeF$bZd(mITgv&)oH|rf zv7v7hsI7q>rf1ySxkw zNstzJVn7@@)t2mV!~L92HQK^_;>yQnnZVTR@({jeU`voq2U%QVUS$> z;BD6h&(a-n)4vu&pV*N8swkZ3?amVr1ck1`gEFsN)^ZulV;ta(5Nas|UZ#-~Vu>Pt zMQsBkx&}c*jCUG>V9)4qXPfA-wvCAHW+{E;v2u$Yo^vg()Ue?z46)6oNiJD&uKHA( z>4Ee}ZBaTk0%LWgnh7BP#iV`_o2{RgEk@kfsSG!{wMkNH0+6s-78%sme5KY|7j!sY zO`U#@!I~QCNKQWCAnUOEGK(Dz=yZGa^DW6A%s~&f$^SaxynqN|&dX-AfPkb}7IUn& zKQ0RR%4o2@k9Uz^0u1lPHwY>RhcZlIjVFM2)lzLrFaHbaqW>zZ`@B-7(x4iI;aF)6 z*lI{C^iK31ajkEAi*{-A!~WK$X##ZkQp)eLM&Rt-n1WkZd|rf98j4rwa5E1TFE3-z zg1b<{*oLf$`WWOspg?a{rj2|lt^*+dNkGL>&^EQo%_$=D7H>vK=CTA9dYos%okg8% z;_l+Y+`$7LJ>4H$%FnN$HecojC{5}#^T4%UAzLpn!Ue%i-5yW72ixZkHRcHGxKN+z7 zue^r^dFk91f0R{1w0-)uajSdd`q3$o^_u-nejA%05Grs>Pe(LM^OGF@j_9ga7 zmoZ8%`nDaUqT!p3=Vd~hz-%s=*WWO!lqkxyj^hTJL^EHZj2^1UNhVEkIKNXwXaG;7 zj;I&$N<;%c;!h)2hO7r_SSvS@g>!!LK$*G}$&PU`2A1k^?tvWBiLv&xp{D$%J)S+` zUDN+p@RQ0-TU0@!;-7tDm7b?5RpN=A=$+>mhol{eJE0kC5kqC@KD`;mc4)WmzwkBA zPZxCkVUU$>q>-srv&{&^(JK_^s>})jN>{76anx{bs?hA~O&xM|LZuZ&E}#x#fp@I^ z;ZVKa)VZp<)kiXt4Hf3WdbQ^#gCFUKg;Bz>(Fp)hC%ZBVDEpT4bQ)B zQ_#=ni2`->h~@AAsY}s(mq%Ma(nx~*7@19_tZQ*w7W0wxp$PrVjt;=9#OGegp<4*Z zhTBrJ4sUv?q*xZLOmpG4W)F_PJb^hX$;1M!VGh``X=yt37{2b;gJG%Z627@CcEG9AzJE0TV(u2g~e8X~?=BbW0P>(3>rDBpl)ZQ`2!lIL4T zbcHG1ptPCtP;hR(DFzdb62UwGBhLSEq{G}~rmNK^r*~qxBO!|cFJGEK=O@CpN%{|$ zyrxr$j~$>_JH6OHHptLeP866T7tZuqo%kN*B2;-#MMi4Y!kK! zuHwzvZY&EtlSja!Cgk^OSuoIspC&txWH5AF7u_}pv*biE=|2C+_ihYcp6Fz+APU7fxb>j{xgyRH zBKbd^R0G!K$Cv7HTP+#IE~GuJWmE)HIE<9dwTl|EW2*;`0HaTcrb)I4Tq~zu`(r)D z?HQn#c0YHEd3Hp7;f0d{7PwXDF)R#QpPm`0RM?}@%`>Lj44#_~z6ZKUAFlVc2^z%@ z0@S-R9p8^=5q;mA4PA$8=NvqMWJ5(2bfCmCHZ`nqmThyDV6Efe$y!T_4BK0#P`yE_ z8ATkEE3^`Lc^LGXN$NsjRMr#R&`OJTlDc?Z_@sYg{%yd)V;6hmi&vgsFU?N|-Dzud&QS$W{L-ZXeF>G*6Gd zxiGzVCI3?^@-a1;lVz?t%V zC1D%<6&J)`0%r6~wwHvNqtgvmv9;YEMbosA@EPn92t!$1LB6n}qm_8sY@w{BXsp?A zE!|OJd%QsMMgTZQ{kMGd6C%erQslPLailiK$}f;9&@c+_V0ws|Vd+%KojEBGq(g1L)#)E<4HfmvNRFw5>Wac%FapCw%ThYFUSsOwA3n0- zs$2~&KSRdw(Yg^art6F;d{J(e8^)ad_g1Qx-Dd_>c8hXz5w^q^hQ!tZ3I)TYnDN9 za0dIBNBdMb<+w9nYXdQEtZ%71b-_u>3_wRs|+M4L+fS5+G}4CPDHPhkZRwh-IH7~scNw+qv1e;R&kO{24-fY z-RG67P2B{gF54Kox`{VPwi-&r-Z0eatBKxZ%%h;YtKSTx%zs84qipq7H4a42sBo&* zJPsq*EKh4-^pXr73E2R-)e+=xgZ_hNg)=4f(cnQh=(d0FbNd-|dYhRw(2imX*J^lc z=gOB(oig>PW#y379sMJM0s>|!SPv92n)o&&KD-6$`4lcHZ`3_VQiiKwz!(swa0>Z2 zu!KR6tHN9v-WS0$!5| zDnmLH@JC0jN+rtK$12d|#z)k*BT~+)%M}%O6fDEa?W5xDecN#MIb(3iOU5(tO6Ml) zQjlSxym^M2MJ_`w44zR#JlAY1TK1;i9V3!wQvUlAst%QCDS-y3tPmkMDT3}6NLi7Z zmn6x)fuqG_^l99iQPRko2)xcyZj2U%;ZWeIu`_O+KbSZ}JUCJXN* zXiN#8tocV#x+f`!CdK=U;Tuy6`g4H{arlIatuAY0YGP^>dCo@nE^B3<`8?WKfQN1V z+-2bcr7jaaK?f(&o<&(uF+^FJ+>58KNIy9b>BDsklsf8jHL>FyS&o02pA9Dl<&&p; zMFe8}viHWSqNR2Bacd-K?l{e&=chYTNPISd{KF_`1w1_#a>*3H7^@tSO0>45#lBTq z21+e;m8`h@@six5SvcnZL}3Ql**0UqugJrn;vj6LEkJmp50p`ja?^|@iw|L|nsN)oW z?LPOk+u3?9yjt&T0RulnygX!L23@)6W;1a!H`RWPpCmke?JzUjl{;C-R4 zmH2f~p-f^4z$@zP>z?j0tWm^uSC;krG@Y+|UYwM_u+vt4>Q$NSG&Q~>nWPk75imX; zQ}=sD>n;`l7r?Exp1ia5!9E3uq@{A(RP#56UZ9dmh8FKMY91P82z}{qd?GY2RY+lDb$y_wkbDK-XJeso?;ItE$=Uu z+4a`g>|6jY!G8*oEvjT+saUtTN|JYK$`UCQOiLg15Q9g~>B>1B6zb9%x%v=`bJyiSW-kObU#@u5p@V4qYPhq~Fb(Zk+AFvW}Q7%53l zrL&1`=~Btaw=HHu-2?070D4vLBiFN3$Z!9wYylf(@mOpvEWmsDec~;Dp;Y2HuK6nC z*xDQu@>Y9lqxp3gEm(%zypawO9a;rNf7Cb+cgcQK8yOMKZ#&+J8(&IKy?zguck6Gr zb79c>ooEqeoq1Vocx*vV-7&{iCBNyXvs^Z4HpvG_X3salQN?@OFS*pIWmc%$Q6fU9 za)MN{E-PzJayDRb_w#k`TmJn?>|@P>1YsO0@!Lmr%=#ikaWb!AF|3btQ@2K;g;w@gqOBJ_ z6q@@A102QgV=LW(?jjViNWfdf@Ji*2G_IeV_!FM6U&~B zT)5uW!RYuON6iUv(zF%e&%k;58sa$Me6!xuja~%cD&ZSU-UhW?AqJB77xbg3efK}! z&l_HX7nq;4ZW)sMj1{iAzcaM9O{S?{mQK^X;vdc&Qg+k;r~lttSm*G zU5XQL;zBJsX&-4J3;NE_EhDtdZ3UXytUgbRwc`EP6`}X+w9oPh4G{ciJ<@- z8w4nT11;aO7xnB>E*=+V8>Wf8c26CUGDHSUmaoX8y+?7b3I7Yn(8qz>!fild#A=5* z3m^rg4()w@$xUH43 zlzi^}+1Rk%Xm!q8mj*GJ1;EuA{JHtPFjhhz8*V-34kr4fc3y#b+aXpAWg>Zcr4w}U z`_xW1C!WhCWPYB{c)f$X3c?j3I|L9B!pA~>!cY>;Y-7iEB?z@V+~OMr5++A&%amf@ z!{%BHuV^S5usQJ*18pfdGwpCt-D+^Yn--i@9PVvX!rw$3X%W1X0gNShe4q{vKY%*% zE2gc0Zk1b-pKo(e#q(k)&_yZKJq0bmE)hW`*Q7H@CpBKE52AlwZ^i8-pcXLn zDjzw>hD6Q7lu=7zYi|(;N1a#}Id$EGHcemBePN>(=`CVCO8@J)FE5oWn1<2-Z~HA~ ziqa`8ULH*}>%y2sySi*7#B^Ql8IZFH6;m8s4MSCDe7e+ux;9VRaP%18>pFG!c(Q=} zK@R+;0j5xrBrSFoer$Mtf9Y>WEb7L{ny_0rx|U$>I*x0O>_}NF$cEdc$~Kd$@P1B% zo}u{l6*@7hOI0p)oe*Xz>7-gCE4DILA@WO0`%h4wF3%drwP!-Q7eD4F4lPB=#`W}q z&T~iwTQVh}9pY%n5oBbK77%wqFj=XG%l^-h>~@=&L1=NTE1l)DOBcN*cU#QDLqXrk zb1tFzYT_=jI{^}vE*suTh$vnT^s#Pc|6h_cfMi3qa_h6uVtg2@p|0B|eQd*9j21aJ z?6&yO%9ArQRlYoD+T%+pv^`tFu-pCc03vF=%+?mgVRN7r-yCCRJ+lHzdg}sqc3*G0 z2H3jps3K`w1f&#JZoRa{MVnyTowohXjN{sBJNKBaY>Hq2TefcCtz&c>dG06Hn!*m{@nTOA*-+%x({kV*$YoWp?`*mAP8PoTEf5P6(8Ui>urLx zP0qABA27a#dxLvJFYTF36S#6UP$RH+GO@oJ7;k)+ub4=lUr3h@k+_5Dr z_EIpj<*1M`pj$}(ho}Z13&6}j|9&A^#$FQMx{6OFXDMP+__&9^ON{&y}&2C;S2ec%do6_`Q-{gx)^Cd zM+zH&Hu#Urjk%D{l!Bpv`zr`29i@Er^aj59Wi)t@H=*f_Lcl)yBi{6$HHR37yPG=? z8xqxVO(P>FPYRU?_GixW!FA@~|15Zrsb*_gsx58P38vDdrw67|Y)^$N^>yyIk>wm; z$Mn_naRxIpGBgex;Kaia$r?h?8x+&4`y}yrVvv`uf$e7$w`RdjiF36{-ckia3?5-W z=bx?$Qy@csZ{Q|~tMDc>x45H+t|wtmw%y5|0#}ujNlPx;Ts+-l0|>m~3RAW4`3bLL zHNH*3!#LgSNQ)x<*dd9UKw3WYc{)8+Rfq~EC3Ax~jBKRqCo+)}G3RGPdiHfPV3}&x z0%5TkayGAMunQEEUSpi|HxY5_iTpTO$&NkOY;PK8!Udg}fCm=l{jhIl@bu93eB4H5 z5SE_A%piEUNAp(%(s^L*6YSq0bUEeqHP&8>&Z^6KwLY>vK86kKq%FK3HwJ>|RoFV& zonWLKc|_(}%4vvR?_&`-*BLWN0~@U>B`<<4qT~A?wpSwK<^FtaX2L;0RLTJ7j@a43 zu(NB@kVon|{y}#N*Tvp(-#AC%!NAJDiDMcVtmcSa!OyByAi$uFG;Zc`FMj7#h3+(L ziB4z;Oxb+KR^WiG;5(Hfz9qs6QfTYC1dDI{jB)vQQ1~%xrbvbf&w5f241`7mYDA~3 z)I8rf(=OxK2HK%z`*R;xeuVmPaPQZ2bG+<0CT3tK|D2Nine^tC&@S<2$WeUyp{&H! zkHfqKHmOur8P%nnO^etOGyQJ!%rlepkX591$YK_wAEM^F+gV=XSrp2L9GOE_9)Ge6 zOGDc&9Uw>ScOWT0pxqN?LPby31L#Y`%!VQ_tgig|PYNYLXN|bQo6KN_!4L(L^~WcS z*VFu$H-<{D^)hUE*|hEl(tmv|fX5yFjWKk-CMH5Kl~&6yCvEX04_pQV`3qp7N8t11 zydkIXV4XznAFpKNkLv{03AJn8cQpBqYvD4lHR^5ol(GCE9f2D@AR3TV&0n~ig_5>F z5wS!GjtZwkbz|P}Nd1Ycf+G&pN-2Z0^D-ELKKooSvHHSbV}yAS?y1B^ACzlLfocdg zGBEBuzdhPTZg24tsMzTzRp8>%MUD@H-g@A-u#fo_vC>UJ3$#&HXiq*{7-j94(C{~% zdchvNI&fsaMVg%;-s#0~*$V(LI<)WQ)Vo661_(noAP9M&l&P_a+dv|TW|}8?>#LtiP=*kt#gB0E3IWno)sAa$tvoQ&e8&`PBHSBqb!xFH!SOH8Xb z%_~(KfC0aPbpfh^H-x>()Ax)Y%6d@)3{7a#qiQ(<`>}W|yH`4|tg-R3p=;xqYHgoA z4VT2spz z;*aEje`viudZ)kjM#b48GU;_EVDUdJ7%@Q|9vr%&1wgailsingQ>KuuMzhGekyy+@d7SL&sOj8j zyYF=}3nUKxtYwYVAld|!7&mMOeK?7&?Yo=Cq!n1Tjj18XC(j@RRAX)JgcY8BCtw=s z0FeQJG1V0|(?jBdkCmoT%(^3~b1L z4pn9^AGNG=9htQ=8n&9sZydw@VzV{_>TDWyOwYin}v57!}_u$|(-yZ0eSaR++ID>4oV z;BoUbv4gq`jS2Zd^IaWWab*u`2wR$C8>doW9}so8p}`JQ!S@&V7Y~|$q(ebYzV-nF zZUvwb2Q|VIe4{dheWJ!GyNH9~WjxLlR_Pm7jguY(Q9l)E*XHaRyw_$Qp?Zmd_j#Ir zYMoBe^09lifp2~hmG+%GGP~1onqsKJ3~99uc~;srUXxP~?4X64M!9@HX`K`lg{U8{ zjAK57+tx;7-17Vl$aBmzswB}$RAd%?5g9*ab6(+Bj4umz(4YN}!(ST4-HK0DVvWR@ zyUbIF2xJWqjotdIGcc*r3&aVzDLOnSz_@&YTAneeg1*K-qupdE8h`STo? zZ$NcME|{1|5z-giV%edbZNbleU!vt8(%n^UVmkHVJYgp@4h1ui@P86))MO!R2r0S6Ux?z!Wr3;hbtXq z@xZHk+FTNq+I1$2e!qG!gZ)2;JF3l3>=py5PA#;OaOn ztHbe^eayeAY=*(s#k@%0#JnH`4v*W}I%~*ZJ>E3XYbwE=WE8Zsbb4ymk{XAiUK-z| z`e$?mLyZvDhij8PQIvoQLIBw$q`r^b+DTRXss3pttZro+44U_7H%|8i_;v*x!0cvD zb7E?xH=Fe%)7LW{EZxAd@_^koq`Pr5gl|Wb{B8rbEtCB@yHq_$r^|+P>Pd+uD$w=s zurTtU80ENNBPT&^$?Xl%&aGkl+B!2Ps+zs4%eG=Qe0^|Ep6cDYA}@=pu{8u(UhV{p zWx7!=+YM}>0dwcCS8h8D_un?f)dsy3kcK6<*c45RI+R*NXNUf~Dd?S%kQ_RPQNcgz z6u5#nYtR@Cfc~k%M9}D|UI9`J;;85YG@#=uob1cRlCIi2Ygu?rbG_2J7&VY~I`|_! zfh0LBVX`D#4Xf2U=giLH%SNK*-B-(GG@Fl)LW@L#%Y96NA_YvDS8)hnNFEoAo{znd z>(Rg>p+@_lSd1$iUHXn|*o}mCY~m$)R0v}Q92gU0uir8mr||^pQi^dPR5xg|eT2x! z7_uCa2%}>Xa#)!;#w#|G`?2w&HMPA;D?10hO>XvA9BW<&8r31Uc->9kG1^StNf6G0 zM!;;!czZ}^SBB7UW3sO|I`3kn72^6yw<_Y2?>aDhaF6)FxA%5eAUPHFakF%!3Q~LG zC_$qcF$FTbc!Rv1^*9(=-}o=;)Oo1bCrqp6z#XMId+fuqKX_aXzbr$L%K4EQ0KeCz zvufMm?-`OCHgm|Db$60eZt)3?o~%U%1=@rezdFSWtwN)LZ=KB~H5-zDX$A27La*A%6JozC@Kr8<3ThS%QrP84Y`s29z zp&M0uU%2U$vf2WZENzDm9~$}Z+SBCDmli&t@cXp;P~G_Vaf5K z6nvg4q^FJQw3@x^XL2?!lAv3&ya)!0QxR!=Ok)lQqmz)y@gT7*6x5}KLT@RlbwrQ0 z0z;VJtOGMCvG&gJ_}yG!O*e#OS=cBr-*{P*W7O0avbt!GRCR}=RbYZg0J>h;W-7c8 zJVxma^%T>bdU5~pQgHuq&vnHkK*Rq~r!9%MFRn_giBC^PfWXyz=gcL#Wca^(a`>*_ z*+a&~kdu)OCT;SGP}H>eTUi4gU^g7^ys}%e#@Gh$o;*X8@@s>JQgBFQC&Z+Z}?e zmeIE+0E}n$7NMCfkmd?BT~r^O6oVEn#jJw^U$3c9N-IH42{3h){NwBK9l16@2;>7v zT_9Ny%NrNN^6U(F1@hqI(DHG5Ny#a#wHPyp^V3E6Wu8}!BX_W>ilS?}l zOIicLEQW!6%}4+8<2AKSX7Gs`ZAkNjQSh!PLaXy%*eb491k9?6>jM!)LtmRYX;Md^ zv=aB*1Idu{<|D-yXGp+F&W`#bU1uU|rouHMeEI6^RG_)NSkIY=j*!4VzT*1M7Gy%A zt|@`cBI#90++MROkc;2$nRwD61sD3H=hu9@Xx|p{nu8xNV&I!jXX5I8OM=0;R{W%0 zRu5q*fT=jJ>Otg#!!mJ9cb`sk5_3`XJq3W@WZwd>$YC$CBAd2L4mAw zchsY0HpnI;=KTg_H1zh{j85@3uims6I!qA>pj=7D26O{Jz?%vl4}rEoa7_FGxis%c zr(i%4EqE25Qr$Vd`7#Lr1GtFeE`mDU2%ddP4XR$%YY*Fr`#g^yS2DdTFcYda2r1SM zN09Bz?XSO$!D$=(8hns=fjTE9&kP0DuErg1Yy26Ab!1|YzuI1zo68znq5we<;*xWs z@^N;^=yZD~x*6*p3snS5B4}WDX#1%1qg6AKU#++{@9p16zX;cjVw{?Y{ON~T~)FJ0P&%_V(TYV~nkkbQ>c>4Ho!;tx)P2<3Bfg*-=;ezC}` z1m3)=y$k0&k!Wb`uw_@Ug+39^QS3!cTJ2I);69c@ZJ(7Cg0Q~$aT7HxUy(-_>RuN| zq0^ejc=S={K2X9S8=eTKvbK6>$X$=oOr~CaBE(8$+{@k-+cV6@tztBP&FycgHr0^3 z_3zfNRVKarBv_bDSgW_>ECy(p-b> z+A7CcRC!`GOo=#HX7rJpuEa%a`GAoVj-I4CjG-0kqP_jCcZP$ZKaN!^jTl_;vX%%K zDZuiosj(cTj0~$FSN%$os#wl;v*O|8#0$l;LON3ERP6}nIbx0u2&J0&B8ZtFJW~S@ zh-kqpttOw`5=~rLvy2x4L}bIOvVq@|(Jmq{krfX*45*abf?c-i@vSfT>Y>n*=ho5M`Ym74n9 zwF*$m!Ns%>xtuGqyLm4O#{j6l7c$fLW&HGG3Hg~phe)$?Nj8gBNBhvT2RM5w`KJ+ z=?x2iOi}LYBjzXj*19@OKEmQOpTSNfL`taU@idaTiHVD~JJ3-MEjtm#8M|LCd={~% zUJ*qv;Kc}E1Owzc<8#ys5m!O~wxG|aMnG;akH2Ed+ z=`b|9ru~%}{Vn9Qt2b&XaC-YPC2|5glW}q_n z*=2KaM^=m7Np#4vpD?ly2y@yARaBYP@LWMA|ENp{9rP86s0{u`Q76MU;eEVQZ?@mdL$}cJX-CRr5=q- z+64{~8V)Aa-pMi^{B_NCjx}wpMIf+N_FRMjHb!*p7&eDVXSYP~P+A=Nl?fpxvuc+0 zlRJpsH9unsnB87J!ZZ2Waum|XSc;0;)N~pzeY9+$_Qq^$Z*!$;nA)8Zy?Q@)MpB;; z*Q;XrCx@gbX zllPcDuZ+01ZvD>|$1gK-lDe)r?Z)g`B-adhkRi#)dtcg3pQ5a72oxaN;ZlURBQL)Y zC*Cg0i~Ws}=4O*gqo9g7sZ-+Cp2HE8(_qvO-!9*)s@1y#!K&dzT}8d9jP$0+&_^1^ zZXE?e6R!ka0Mf5xvytfnhX9M1Fp*QQC%ExUA^wH}3g7Upr7PB#Hx09Pz5}6eMwc6# zD9w+O#4GS|W*+wP&E?Fm9SB|wuR$|LOeEe@pRWsNi0J#*?ZD1RrY!SrUO`k1;!=Jt z>7D_bRVJez6oqFPf)#_oX5;PD)2&uvJ7V9l?0J4tIDUKKO)!#P*5OHqM10bwn;OBV zk$j<~;H+M(oH9Io%^%vp$87ErK*<};*;D~s<|`vnlE5p6(%0mw>#fA0h$?o!2H74o zPta$j2_do$6FbtRQ<(=2pS-bW(1rA(b{;dQq#Sns6fE#oyI;Q!d)cF>t3Sj$d-GK{ zLa+Ol8q~0zzlX1xm5b7)=p@$H2~l+dsagfwZwKb7n^2lcumt3t-HIFTo56L5b7n>P#ssAYjv%6%fUZhO`UMC${=$l5?3STGa zXB3Hhbhzb7nTBf!yEawO@O(qXRHgdh=SB-7oi3&Aw&P*r%0N?OvrsU+~o#v9<_NpS_-6o2)+^ot`(hJQj^+BC%5*DgPS+l#WqtH&JZA@ z`Q|DvcdKzWGCXGPtfnO)AYnLNJ6<)C(flm=CLHRA?e#jzt~9Tu1i+41i(p^YKO6}< z>5$%_ABp(mHQ7-0p=gzyfSyMBBr6MXuNC25^+woiV?`1V6WewQdNtDgVrJh2OwWN+M|cE#;;^25rK34$5^e4x`a&ve1v z)ear$2=uxvL#BD&kF%ZCqCdp<+)7K_#E}C2q81k02h1zM zY?%fD2rv5R&E_==9)xx}rZ=lc{1fB5er?M|pkJC-Y>3boT&a^&D&Ipx+7(J5wq8l%GQ|V&v&2-D`V;HZNZu91Rs4CFTZ;j0 z_z(2~A6K|;v7oT$Qo;UO>|cyAOQr`j48GMG5U0GyJvSZcfTbNMf=3mRAg{bu?Qzklh=pPs(NetHZS)78b=-hPHelI57xd^3Dn3Cv{L6!Mm2cHnpeuC_8 z*E6}D+ln5lIW8!$q2qOJI|~a`j76_N*ZOQhQb4qcNC~0zr`vHm?R+D}cKuKojyKqy zj5=iFCV~;d{9Wz@ML!Ug_oxD9N@pbY5z;?~b)H+VOUsv$NRoUIKre1aaUs!sw90w) zh~6odyf^2qLXd#|QWM*3g1=CqE1%8#9>Vwdwk*HN%t~k{Pnc<8&ItP(Xh(F3=nmf7`c!GvZ8oy+U;Q^d$HwpMoz;u}e!z0d=xLK#O;kjY%(F zQGhBEH7U$21m?Y98FcAfS7TJ3b9|M z>2r7l53V|=DLCLKD88%fkv*|?V_M8}kFjQuBT@qtTF_$cpd4(r#*)0tT4YO^6WX>6 z9A=Ad^C@{#wi+C&#VYU_EO1~~3eraq*9c635ta@Hal(ut!L>(zz71q+m`WsuO_FA2 zvP#C#2BnQ_sSv;;3A3@i#8`}vv9F?=?1|zS$)WQBGwa~ht~?Q`o56mf@Cv=s`%UV8 zH5dHDsh+sQzbO2>F&3(3Is^VUCx{emsfYsUSzr7`07M4Wu_z)b81ZZns0i z@{T>ss0g*u#1VN?6qaZr12EoqI^CVFpNBP@zKBDTxC1oaWe%JZnSc$59q(E3@PY?5 zcBN=sj!E=-)8K1kcN|z8BSboWdouyrKR-(&Nk%P$mWgTAT_u?a%4K>J$Oi^JTM4VDp)TFqLJq#H%FIaT&vJ8FTI~o?wD-vvPO+S2w4yGx1L zgdPd?Zq&LsFP~-`q+&Z=U%$l)Xgj(p0`0c8+mFcC4_f5JZAf6Mcc5BN8|M=hgZBnw z4OmI?XmKk&%?qUn`aa6zP*1IqNbeJ!)Sk_&8)5*oU0c*(Tz%z3(?LVYZO9bviS4Pd zoIFWFKE`f((oOx1g)wkoMI<-}eNc0&0rE_*#PX8MOoFPanEzOP9LnhhnzBJ#1C0qA zdsS}MI;G%~sY$^BHK+mB1eDAl9bHY0LCUF4Qkl@<^oq0VMLCVk6N$V*H%-0Ldu{Mh zw`Iw=8kcTO>h-k_fSk{tsbyfEVGb%)3{*|4e$|>Y9aeDsi$8NjgcugfoNKdIOpm%9 z77Hz?zE0Ma<7e^cdH(NrD{T2xIT0kcTWKq`O$?*@j+F8CNODPmPXl;EheIm4_WgkvnzHGQ9 z+*y7J&`3wdaQ=PLVKNYqjw|b98$3>5o};*0W^;jJ(87{^^7kau8|Tf9hpH5Df#L+H zWRBth0B~Iw07;Vpy$icL5UakNhwc&{3kyX#XZm~ky7GYA683Awx&wtfeu6!gmyObw z?rWX^F+k40ghWWU6339|3yxh1La2RA%+w2oaIwaB zy}3m-dI#>h{=na%raYpDYu?p-^prL;%=eAeXKGy=vJMmp0SzxKb83Fs^iBoSe0>*A z7ltpK+g;>CMsCZ1EbZwCgmO<4TXR0igA!yaZidRp)Z{Lku{9bv32g*QX6%A~EE{^Y zk+#%);s##k@_Xe2ZSjy>D9g_cJ*jpr;W zliUQ9*|i&g zOno%ZriZy!y?ynyhKYIWRWP15(+^L+tFBH}d(Xu$P#9o0+S!@t8e)o_5Q$cOs^5~2yM65)`|+jejVuAE=fZkTlAPlK9wX# zQJQ@cqzxmA2q}Or))~w3r69(yqqHo3Dm+_8+=Y_#rc$XqQB$QEzJ#9BQ}OaN_`44I z=9+m^afFID_-U!w%Q{})c@%y41{-+5mmf+yT@L zKI|>_f2CWdcgv|o?}$TMxZr(x%}iu>;;H+^fAaB}qmWISD%qo`>JZ3XNpW0P_aBRQ zB7CfFv&xiz+Bgw+@}VHdfZpn7vn3rvYhv1Sv@z#XYJ^6Vu~>F?7gbv&VkkcT$6h^l ztbm(&S#yXkJRJO~4w6dnQmsz|A;e%OTDC6$ZVnn|DKw8xE6)?&w*A5Zx3YB@fDOhW z;KTG@8406;;MoJVWOdhU%RX(zJ$#Yps}U8g32vuis1@99FnUe7_G}e>Q{(O9QK?$< zeVMB)5>psJrW-!T-;8h?rH&6@&IoF^Naq)oyof3AVHN*7NC(dQLys|YW|D-G=MHQPsKQ!y)D4=jBsx-_~>iPM3e%X2I7h z_%`PwY@Xh!pgnd|8ouA`E&p8H2vcx2x0I}x({Xw#Zl7*qIxalmf~i}FFFpm%@-fP65OAJ zNgJQ+GCSfNcb|ddfGJuY$-4@+MT;BFc2<+bT6lSgBjA9hPzP(V zrA|dIL~G!%ke{_m2?s)%%v!bWI(JnLY3F`#x?r*n#vNhHt-%oH zUm}U+ z7%WjYVV8vp(nsNRn%F+%;l%}t)b|lT7aw|gIV>IB@qP-ECsFjeHy^Vr#|~4k%5-Nc>WmaL9d7c!HsCt!eY4;UB%Io&JllS&>lT zq03_&L5mMQ(`RN=-q`Y$J3jRr*)Nzws{al2KXi6T3Cw3@r;ZCeuS$qX&fm}m;U?gZ z%BSACIe(|MMdq-Tx50;A_-HFw-G7yFK^0G zsSq9bqdIMvaS@o+9A*PAl02sDM?`A|eY11Kpx9^WKKuEz85x}rQmMZDJ%livq-kyD zlk>VM+>b4|1_ARp<3Pl$ytHf{)5h?`XrnRyCLa%N1;EpW3!w6OT-pbf^O-dCHFM=G zv5G^2%(NJmxhA_Z{s!Dk)Z%sklcLVJdvT=}kKVKo1*7-6#Yj6db8=x1vJU72t~W$) z(~$$4?ajdCf)@%3e~Jf9RYS>1e%4@Z5>zJbA|m5P7OuEA6;TPMrFsmGpl2Z}GQ7)G z%;IV3{Q4KBV(lvKFzo{4?T;QZbaKsbA%fu|7^o|q84Z;Yo zD_<^~xH=Q9obvGTEhAz4owqR1`-eCx)DW(xt40G)`s3G&qDQ1}CLYbD6x>D5h)tw) zlawJm=*ODCZH9%|#3VhQ9%&>k(2e7sip^-t1l&^#Pl$arr%e0eXP5M^=?Nap?-1*; zAdUj5I1><}@}#P*VH=+f0}=p(H#w$t9g>b3P)Cy3~!%QTEBjHZDqPEX1N@ZFsKn zSNU*e(AwfEQ4lOr6=c!=CVbx~(_l ztPz^8T^@W{5T+m_AH3^Gq$GiigXO%+xWUp-|_tbl>^dVE8FSCoc`d}mS1=s(uC zLqQKDue-l}7XzdxkI70HR=GC6^Z2ZaD*P6pF1CS9B&Y!0A+aG$?;*1qc=*!ngfRqv z{SN!=9IHhNbsm<)L0*W5mao6j6nOh^o_BW^B}*M$;P<#;5KR>Q-rjSMeTl5n0+pcN7$a_UM*~`^Py~Y2#hPQDEZGL&dQ_Io2m~z8{G9Seq@u9q- zD$cWmx;BsF1;D=MpXiHnBk^SHqD46^mTR`~Cb?1fvIOGk5}+(^gKu4&91_rOcBOa1 zdLaxx5B6}@zcC|y8nO=g#~op;Fv(*vaW2P}y--5xoyaM04v%!dyl#Mm2>Jk@X@I`o z<+H&m$u+^y=#;Xm>EgK!)MJ?5-l*HI7E)F;qgsBmhH^GjdIIn~WnI6={=2 zUCiSwUB7|;O}Qme%NKGnOzEVPQy zePn4(JS)v-v+?(P@vV*)q%yx7){Bd1pX={Pq3t6?MWS;xJYA=)8&w!<%xYuhrmQ3n zI-q1F-sdGfp8H3eolyU$`%--$Jn3M5oOaJ^@GOv@u1{WQl+(A;%{*DBnQ#}3- zp+t+y$U68C4~0&0#T%S%PM%e@94FSq!~g`^SVx6m7|A2c=U!bH)jf?4g%Vmvme*J z!t;0%Y1zdqa0;66{ zNVm964sF&eV|cSo6f?I^lvRRnsle( zsWm=}t~!moo6{;?(1^Hf=kykBd4)jbYv)GtDXv_g;KUDuOzkSn1X9}l>^6Ozo99jW z#xH5_@8BJ#Jte+{CbtNuS-f-fz(NrHdt(StC6a%iO+14+-Be5-Rv5EpReCNvgKhi# zjS457!j=r~P>AM>0N3j!LYzObia5&ZOk(&Ywwk@2*-Ti zGA4JvR2g9!kIa$e*6B_;!9aCWhrfQpe;rH4d4-R&VG!S>xWZpiGsO0hr}5a#$=3tt z5V6%w)XIo3bmg_mcV4g?2^EIBL~$+GhxLwi=0fc=v`&J147rPZNS0t91Efvq4!A(V z@YFy|f2A*7KL&jNIMU9Na{&z8zvaRvfz|#H_Jc|ZvZFJ@m7TOzS+W$!w87Qsx1Q;N zxK?m5g@GZbm=(8ttmyOPpy5Sqe)N!Gj21)&WkQ4<5h@VF*aSOhp%ZFl?6wyXV}CI zZ0lLzo|xr0`9q9FOq{Scl4+BR@?S)mWP(#v`1Twq#W*wBWS0a+F9X>Gyaj!l-Jt-r zJ|+9Fxgv>C$Is-Ze+HU%hHUm3x5-pkaLr6PJ)ow>Um%}~RV;3zhH)yn`uj5b)&sEo z#>@gP>l1~klFStd^QgEHwDjW0mgmK)CZ1W&`v$HODxD0qd*kpAfG0c~FmaAp4Ri}W zlbG`i&RTnN1fzxgd|{yTHil3M90hq!aYdeG6>jA)IU!%yzQ!iNo#6N0;=LoRHZ*$I zOw-TN=!xU^Na_}<@dx#Y_NT5T?77&|631@njiz94vTLDcj%E0c%lh+tSMxIfk3_hB zlJ-%X0F^u3V(E7cXjb5U;Qn3iq^d6Ou_rF1B|ex>Q6r4oz}B6I*#Ttt+_rbFei>=7 z=wOa}LNbWhDY(=Rt#%O`ASZW_rkBT&cqL0GZ9DF_aUUx+OayPZeO>WY+61gsu}C1Z zYmqTOpE%7-JN0_r=M_Gy=WIDsWG;=}wO@0m>|z#xVAql_s)AZgJ~Vx+1Eh4CNg7C0 zvqvFHIB4;Iiy%qUD9U8)3PE`JzgjFMllVx3kdpKR%<*YvXg&7l7Clv8qdVu6dfvF% z#is)=44Y-5-|`iKiQ-rD+3Wb-@rowVvTg;$KtIN&Q7&gnrqXOjE`-d)QOEnb<0HbC z)WZ~QrJuUyf)L2+fJXde1M@Kn!DY$<_^$K8yGiv-rF{{Kl=9ZsJ6(S|fT*dSVMk}d zWhVtz*|AK6;NSxr^CB{u{bqr^$}%RSC>I)ZQdKl4>A(>%GR9$1D}fXONkC25%2KX2 zE2fiRBMunbGJ~O~Y-j*j(Iy}P#4QW9bpe;ak9V&2l&GCmaU(-h>>~F5oV%J%PKuxy zE)_LaXVk02OlJx|PZ7R66&rm{|Hon@fX0_bKelU`z$WWEo)FQWy-@T9B$6HPcksw9 z9F2wL93~bqXiR+d+4s=-05{hmX%({JJ#-teLyPRWpRQoVrSlwZ>53I^vGTJtOK<#a zKI^*o`KCPgu_B*3>36EYMAGlh?T-9!XHIm70Q$`f(YZ+2Q%T?CAGPr5($@$uc0k1s z+4p4g3_00un^B)~lny@QRe1j-kNL4QeNI}rM4yFj8bsn&Z>Is$^T&9Ny6|bP*Jc!y z;^;565R8l{lB6#_#!ZVm)6o{mV{5tCaZ0ZZ>VA^i7oE1dhMlV=WGd@n1xF_~akSn^ z;+0zZm6i2L;V2>bNBn;Oy+^u0;mKcj+Pw6-=}r#Ka+;k@A%d%Onn|4qI_}?KGY#M^ z8=A{FQ*PcLEH`WQAueRGQQ+2)h^cBEw$vsWkZ|DN7BdvqbWp`qBfD*>9`lVechhKc z=uXS|zyHkBLhuI)dQYbn1*usjPQDH|MEU|5yGSifJsb}MNZfow#Xy*!r&u={d`VNB zIc0J9e;l_>fKCIvU9PI1`3C_+q&f0kZ#oP?<8p}r3z6M_()9p?R?42s;NBKbq9d?Z z&m@qk6wEC$#lBIgqwi~t0N0z4=i?3&&rc-TEBTG;-$M~ps+ zY!$K0W^cmL*z5UDgaPkiidr2cG5~xHgiLAjGrGKqS}cYAy$$4!a)ilA=I*dOx)hZq z#;lwpLEYnKsuRrDbfrBiJY{}sUyc{*h$WN)uscfU1RM~chtvHSnmcYuUgZtffa*&D z+18MGmIV}2HuX$3{4nV_lFkS}UMTvWM*PJ6)^;nxNClTej?gb6G$|AP6{b6}R#kjh z&;xTq&O9ZlGyiPf`&HWo5BrLEBIoAjHY$^@K8 zyY$HxIE|%k8eQ$m(?F``)WtT4j54$ z!2G49JzrSRK4E!b#V&*O!GduN+CM#&DqR&C?#c*taGtZAFpJ$j*qc{yEp9CQGdb;1 z#J?uLY(EZPX*#71C(}NlL1iqI9YVAhiV4~w|FEU_X{zCt*?AMTf%|&i>QP36dFYBs z%DMzd;5mHsL4P~P6T7&OIl+}XT1CvEaq)};!QEBBfc$pMM}C6?KcQ(|%|e+S+G$en`}FZqm19U4XBg2E(?{6#+q2VpPluh$$4Y`k7-(S+#N5_m+l-3kq46-iI4HV zKy+?jnSa*r(z!VnGmmO=i_6io-6o+@e$}7EPgkD+&lsY@agD#~vy2F*Cd{zQjsoeo z3+j;CEX4J65v9|D7NGQ^Z^Ka~vb2P{q9e=OU{H-u2mO1X<$JPZD`3gD#@BXPfTA#l zF@K$hZp@Aa7M=~Bv-}6?wvTr!Qr{eR{l*z36o6XZDoSO!R^*wp7;5IRN4zAg#DHej z3+Mioz20yI1&>ZrL2T)w#E$z@ObnR7&alP<;56uyvObN3cERsnK~^hJwuPb8evqsL zQE`(r8ZMpcBSKpOz`c7S--UiA(4V%0CNZ!Z5TybLi^Fc+!Ku%?{8R=PNI;BNX?Ep` zuRG(?Db)SjQo656U0VvMNdOz z^taS-kTozJHzao#O$lq^n%=eYQ3OZtX_3o4>&stex2;=sVErnj=oAy7-mJ(w+H*y=H_x5`acEcw(&X%! zDfbkoR6pEb`^$-qO;}7LZAF6auGh+|7VjM>tpb+h-Gjgceoe-BQqk$bY{FlkWxNQ> z??frqlKsWP04VKn;@LQ2VxGoM_K9wxBB3OzUJD4=^RbrA7*dsXbLFkILkecGdVwH; zVO+Q#Z`A~v=i~?p_L9A)V|RyoV%Ds32#KvM+_+bb`BZ2wxSAHcirT5ScWD4!A%}1$ z+J?-n0qZ@3Qw26Y+{K_Rhz`|)dQA?g2gFSoRWq1#0+YS@qNkpW8D7>ho_z*XP9fGAczlQnn{UxmtCH3WISG(w9TW1;A9{tDH`>mJJkJ=FuzP zKCp+V#<){c8u0#a*HM^?m{MA^hzqRE)zce^cUIe0)Wk9J>IOge-C4YUhDWCf;sO2O z#n(thS^Q+bFmd4o#)znbfUYQ^rmb^R2bs&7?shwUT_d;00msI<@!*(Fih}H*-ThEq z>!8DV*okg0<*?WXXN=fh5(4xk9S0O|C|x}p<|EAC6(DC4f1evYweM5 zFukyw-#YN$fi*LTq^X+-{z&AEReB6ulppxJMf%L>+}dc6{p31o%33UHm}J6d3a^}` zORQU@rA*BvhRx}?aFxzkI(;`4)UE|p(O-GoM8oB@_VzSNHAfO0it7IC+HJ%MI)KA?fIE}`ssdHIVJU53R>+fTDy1;+ z#(I!U#$O|2gEGQIa!=SHEQB&C&O7x zG_a*rk;Ro%4~A?^Qqb%zGKa3yEi?5sWV@jO)E<23ym9Nmvl|3LH+qPpCn*KkO&okC z%Wdwfc*7r)$MG8gWtuhV)2&BB6&S`UKf|k!C6tePavG;q(7ZQHN9*|&3{ovFq&$rK z((;@()|RXQvX!-9GZtx+-Rpkq#d#?DP-U>pcD$!+X2yF-K7gDLG#uY=X*uWRnm5Is zuMzid7~s%EX)cJD!V?=f{Ts9?MgU&+bfrDFwPWb=Rm}Vle0C;5`RY741hDDbF093( z{b0YY^1+O^qp-GKDrS=t=Bi`n1N$AqL2qv%#H8YDQ69?~FInAHw1b!ww$}s4L!MO? zdAldhRFL(@>HGHBJ%CLGn~?ObwerFXtAXlAgY(QqqRZUT6A%@b?OXD@1b%pj@Doog z53uovF0L1Hm`K%ieid}`GGj-;YnH9O9AlT{uxzu|@UcZw^V?G@lAMxpD6y|`&7d#O z4H%qPp8x5bQPpFzd^MB&liVDW%D)$Q8F=FyO0QpYro<>q%I(BkXqV7~elew44hpRq z#><;(Iv)e~q+o#o+J^Pdo`Lm68CFRZ3_qQlm}Jz6^V-RB@+c5eYD=)aYlm7n-`b&K z7c1-B-Zqa&u9Dj5YkFk`dz@FUk|wU8+cs2buZ4^eGz1piC{Bu(6P~6)fHhAo6FYvB zD=*%m)OZ6R;TcxmYMh*Ko5~lwgHaN5n}#mHjSev<&u!awne}GIOe1h(^IS>=fIDB| zeOGf~|MLhB7WF_ZXj1<+8&DS4o$fEExqR}rLt9W7M4qNp1AV6-ikDvoUJEU-^g-oJ z%nE|g($PIX5@A+8htzAK)PdQ?iL}Dc;)bt%ZPpz6dvc6Ot7UwwF@%*QvcAS7gQYxW z`pdCEoH6DL7Lw@MX4}2%XZ$=ql_-eYth|qVY3^9%bUS*W`q1S_+o6kru+V@IFkEt= z5rPIbUb;y`x7iSPKsTbf2^7TuTwl={ zPy^Pn7=!qQQ}BAo_2&kxNSgSV=DfY?fzmbAZXV6_VvpGcn5{e$s!gT8vpY9}RUT$P z31uyL`-&nqI(5DTS(OVN1c9A|_;#Axu({mEzludtRWqFNg(&dU^$Be2`l!Tkdnu0t z&+RYRnff5-n2+ahek%iS&Z_J5eXb#*u2D# zAL_y>U?!BFaeq|cMX%7N)HC8Zl0Ka zzLc~VokiujH!Ldbn%27_q6W}WATjg^%`##s6W2qt^yto}vxy|VTsm`XJ1C&HSs6jT zRkGMnq3~&n1q>r2uWL`L0v|SZUM?C1&aRCoC5=fnNm>;j49%#>W?%W^G+fu@zk30R z0cfTvJ*WHHp;gaK{2*dzgia1tNn%?LG!Zd^REc-Yk$A^b%az7FZ)#r2!` zsAilDsZxy@zYn_r11W|4Q*GWi&Im(V1kf+)l|6nRwo-HM)*O~{7 zL30PPtrY)1UejpSS^zuo*cmqM;SB4#N$qO!s8Pw{LzZk4u-r6MF+w^z+E`6JX#)!B z#b*ADr{xceDqVH=wpz!snX>CE4s3zG@nRv89O2$^iO0yHwv~7zN%*e^mv?Bl1;OCl zqCYUZ?TMU*0)gpid7qw}a{?Or+Sivh4Rf_5Qj6Lz%zbb{Xx5z$;LdPF32hOfTut2H z&R+A$AX11psO$@6W^77u>e5RFT^x*gJ$1q~5ytHU5%nZiET74BOwolz-W22$m^D5Y z4d=MNY*=O^*$rI$Y{e)?Z-oA$oZojkVQOF7mye3Gd+L?irW8HRH0dtl`yJBbYXbUaJ^k8 zQL!%FR=j?)2TaVlz!PYIEBeqz5V`K)Iu|FfmyXO$*`#iDr+B3111rR z+bjr}up^PqJg7OJFF}lX^n(C{_Q9Kn?jYGME4tNq%SfsvtkYAT(ndH%Ar2cHtfg4QOmLj*vl|JYQc1`9LBGx@4At1wkdr$ z=npae3w{Na4Tav2It7Y*`P+i`mLfgQqvAV%ON~PmdlN^_`x7idwCVn^CN>;$Ia!Kq zsrrJc#`R74W{-p)4ZCy_q4(76@_eOX$Kbi0Irh(P^;g}9O7c%-<*41~>sbZP-yf+H zRJRcsWTebD`Lle0RO&dnbw3kE6}0oM9rzc4sA$-bKWsNJ&o&dP ztSYV+9~Ns^;*y`L9yGNX_az^q&af4{@zs$Q*|qP^6TUqGl70wVN zd3EK#n{q@zw; zOx|#hoXi3*E9E+agO96YGI)tPSW#-#Nh_b^2`33YB)ogIl(bEQj%FR18- zXjrHKFE&LGwa@#r!uu6%s4$cLwwgEn2IL@&f^_^^I>&5jBRAlr;T$_qkV2!}H4%CN z!{d66Mv||a)|1MMr`~N}R8twFnf0XT{|QyeLH)tgB_{M{4t;ux#~X4%KHYa{&;0(5 zSC!~MmVkqx(0asp)}h>7l_e{Wg;X96ve3zLM<^P`_)S-wL?T5uiF*|$PQUSWmHh)% zh(bTuy2I!d3bTQxbFBP=H9;pCYrZw!(2ckQD^LV=tE5Xh>Hv}D7e}dGm*zM^&sdYe z=AOVj*7ARWG(RIC0nd?_Q+nNbIo#DrY&*?J&ylo56*ZcUDn^3iOvJVJVs~lh`1^`+ z;2+2c#J3ChdxRM?;0}^a_={6K)MtjWZnoNYH3Ty?*u%9(QB!;T;*{z-l^FD$S0FFq zwm>^_O@kY1)-AMf0Ss}5R|S&&Jxskq%MxJ(Y*054yK#FHTLyTw)*DhM6=P~j*8_Y~ z+F|^Bn$*m`MxKjvF<2|kjGF(B9k$yi6WaG%k)jU?Jn$1yOXMcL44$r%f>mMsnE+b@ z0yr(mw;um@S+J^_baaiNwp4TsjA&JcYX?CeKi8VXjm?3*tH=&{esv>$A{et@4dc^o zTPn_19r2njB~6=JvXMID$guG-_sZ~2e?R^F zykbk4XHfCm2<7TFt$~and_$oa%gnCv2X2FXtK>TcOZZA(endP0>;m28z&M{+)TX;L z+y3~bM_~ zg5Xh108Y)%ak@UiTwO2ni&9v}#vu#gSmWe5$R{3SQ>!#*g6a^z%4!oG7-4CJZsRul z$dV*iL>&$9NOYEqBtp9w$WQetFn4}i*!X(C7n7@6Jix}@Unb7BhC z9l#p)WFgqRy`RdacFPy(eJa8tjM65<4&*5gT}JBd0Fz>9*2>8FXyinS;GHDDa~2&% zkx!+!s=l?^>$mqqFipSB3}Bi+HO1T3aa?Z%Ww-1$)8dh5N$h!M7O&~RSZm{#-Q!pM zHNo|F6_!=WYL{6VPdf7h-RNs{mb$tga2sVbu!jV$p)CW(=}4k$4eYw}q8kgx+AiXH z@Dw|xDe;uPC5No~UQhEP&qnIxeC`!-K|5CJ*7|4|goOFCRhmZ`zq-Kt1c{}0g$`;u zVz#vn6?f%{6~k!~VKm|+Frg*veAB;5yA~wC{T8gu5mO@_-`Hs||L0FiS_edU#o_HY z0AEx2lc05M=MxoyFibnrFSlLJ--SNk4#h@XNeWeHstjp5y>nI{e6tZ96c%eG3 z>xVdaQ5&OtBA1)E$6&bRC4KKO(`dqCsH}~&JC7jte7*S;Yi$sW)_i4_sb<-pDforSD z9j(|120(7oI~F0_qeq^bYyd9CCH>oN;LXbwg_E8L<8Wb!H?a|x7aICEqRJkUuJ%t~ zPiStXONUe7k3XkN*ToYCmDf+3aQEQ19U0t#DYe)NvS4=%IkM&65wko_(mtp{F=2HiCwSn zp}*I6tAb6{b4Y<@gq`yld#LzII5adK!HGIf(5ntXcHD<#u`UKXKO)js1Xxz+Y(e*C zR}%?UL2sxo?!fAfJgP&=i~p~=jW|d|Go;ijr=nL+xTvq|Gm=P(Vzg$x@cFkDwa;tf zDn}5Lot}E`yGGCO?g8W->W01t+cXst<=YT8e*vvaj*dSJ_fT$?Tx`c?~0j$m;$wwR|EU+KH!w+)26&Yk40d=|E1rDvFY|Zhi+v zQu1re2f!aY_XXMkn55=W$&e3b<8#6X05-h`$`oC&nBZv5)1O_Kn!A7TVCnNC;YISZ zK-?50k$5m4U;=y39L4*kn75g|88fCSg}?xHKhkZr!+4&cs8>K^gn*e763 za=F-*y=}6om+nSAfpTB_!F&`67fe-?NC>MmlnR73nu95><^i6 zy<2w8%q&JN^$Ubowxsb=(W1^n9(dAyW|RHgK-BO4rU&KHFS9xP#{8HTS5O4WJ4< zu^tYU;&mZEfmD>A^BLFxH8|F#=D&4s$m_fx8-Ywa=JkH-c?X}$ix}ys9s6?R>T9LU z1q)!pS<1-;ny+*@J!jsDNekG%vCENpA*kB?VAwusJ-2H+J7?iab?4I6`UaPqz!6-95BLB5XiENA(8Wv-q*_mGVKJJX~n7?eMvnjBd5oAFH$&pp;0;` z-%hcSt!AdQ06My8)`+80))Jeg#GBg#5FGA^vI@NKLVHn45C0(8s-qP39lN25?aN+O zSf^ypB(F`f?g z?#%6?FnOOAy{}L01mo`feGx z?NY0f)g4Yois_H{Zhr2Ww9t)UB>9!NOGnH8!v;K!?9OOj&n1`SrbmY$H4hp#XqKnV z+rVOr+mboewHWTKCi0DbvmR%ps+npTEyFJp5Gq%G`w0?4&`BYO5zXVip~Yzve`#<9 z+N`uuJs3<$T8bmqh;V|7W`LT1-#@6)>pArnO{Lrsf>>1P6x3%c;152 z5GxK&TDLv-DFwC%uEb5x&(vw(Uw>VpRqhJ`X%Yq+o{YB1Bq2 zu>fipf^~bXUp_in7G)6An6}6bBjc5J$sdMTv}`?8?spuWUpwy`c;_9NF#A{WC{V9s z^X4f0j&Jx&0$gSZ$A490yV~vT(L&_~6 zFuEYT>lPua{31t^oixhYLnn3B49>@L6Yk>aNY69-Fzpp+40Au>{wrjjb@oLJwQ;1VZwv7Kat0pXr5!Syq_bb(wp-R<`DwGq}O? zpULK((6SLU4-@*D<`{`+{}1T5Ij5hrcTKxm09SRZAim6HUgDhh;lyk6GZ;m8Gp2k6R?^SWs3R0-j(xTtkgC!TFVf-w z?q1E??7;G4rh_zGKdA&rct#VHtj#m=k}yE+_Xcq+pFNiUnD+`CWj6HJU!Njp^39=C~E`0S&~ z%~!bIEj1#Uq()U;EIT?|yCn<3;@v;95$la+!^S*=|&bttS>rM%o0Advh%MsM_^ zjPD#?FuFi}Ej8iaR>QV_A9Pzw=-wu04+)0(;1! zK|_>K+;5J{NT@Dtd`d5RqqB(;V2)12`!i#+E?Oxgp?az)X1{O^OgpKN4mn? z8p{*QWQMAP(3b3-7rife^OD$W3kCI#UFwf0 z5t3S4i;HCKxtH+a1?kXCT~NBR@rvNvOIAUm>BARUk#Y1+sk{an0)wb+L15d*YRS$* zgC4jZ9cbVf@TqC&)gJ|JjQm-51kSaI^YT*OgU!!P>;*=nbI z#m|Byd*_ac%TxY}FkWL5YpYJ$R*bx6)$x?ZJZxD0Zl1|P=pg*Kyi)9d5e>iSDxZkD zeTO0YRcff`&*|M+4=MtdO&=<~(|n(J{Lb>51eSUpBd8`B@q2U;;k7wSSE@W6+uX=_ zkAnqA+<5y>>aFVA4P(LqLG5wh1C01Z;2przwzzMzE_F>1MqvRVU8671oJrbSJ3fH( zREWE{Xrro1UdxHc*{&@l9nD5#>uOZy9rbO$yX)Y@m5}(zpLgCgbeieK0dF(mVt8<3 zqBqHH4&ppr8rdBd4^P{>4D)Ow2jyLV10$RJ=BclQ3-?oCdD?!Hv4uIKkrP2{yG~%6}x&vdgQ6t#DWrqU`r*htWSclR<0KRsk~kDP72B zGa%M!UuaTQsVY1&K2bE}kClrWPVeamQpDHll9k-4+Vmf;o5O$(@G5(PLh$&=lLx*m zv8X(q8XXD5pv)}P4MVLuug!M#FwS?`_t<;lf}8YmJqaQ^Ok;JN4pL=fd@!x8B9 z1b5n^HS>`)`Uw`OK%T9Lk{PUEVUkJv{!QkOxJ;D>qKT%4yt>t=);E{p-e+akMm|GggmBZ=xu;jQA9s1CP)MNN$<*mCa?G4%FX1Dv-OSbxety=`lztl8u?fV17PQo60#m&}HR z2)q8wdB3^PUd8AE61u7h1^(S-GEMw~(-yR&%@vuY;XcrM;esA5nvOpI*(xJ~{^$rz zNiGIw5PG_-+Ru#FIS}=b3C;xT&Xnz!G3#u}Q)3?9fi`I4#cSqO>tA-kJ8)4T5KNf< z$6xF*L?MnfV`=HIBxk=H4E^(AeOW01r1vtn87@4sn!W$<+xifvK4%sL+byXf6A?E^ zth?rp?y(9u5oZJ|=@#x!D-|Y}#V5gy!N?&>$P~7y-U)3W2!}MGOeTqJb|l zo8dA*g`7L9kBk$tjvJ&$>u}|$>fsj5b8&S25b+ec{}GyycbcBm)B#Rjfo#8sI9VCg zp4wu!9v(;jr$8jA*A=@cq5xC&W?Xz;kE{{K=-Rb?)oweX9tz@jWhRK!pVcBgMXI17 z|8ABJp9T=9-pm%NBZ!Q4%yf$>9Joe6mg|o|;6KIRBx2qIBU}|ssYu-Bpu54J@(nk( zNUF>dECKcRfND(b<&?Mr_G1!Zg2x(ndr$4AePBbVe8aN-3_Z`RKR0J)_`WZsr~rC}3fb?eK9|(n(|cT}7nXIAH&*M4WPbW_n{Cla#4|Dj_bVAwM^k5u zGaYrLaRi6QU_c!EutR!!6ywFGdcvQe8Qj?Ly}P>8SJ(hiahzN!iNS7ct$uSR2|GDN-**l_pi24}fR~2^;XJE@!6V z3aZ(38mblVr{ITE#rw`(Wu=1>c~@4AdgSXXC-(-F_ED>KKW7WOr@&{LbFeD*TT(y62)LEoy}bOPrW>G>j8L=Ap6rCyG#@w0^r2}~jwJS8P76Ksx= z*1lp<=)1v|jU#vfL+$mk z#Gel@)Gpa1C%be&D%-%=4hE8Le56}_n|u=V1z(NW3Se-%WaU|l?hB=Mhv=9)YGcBi zawB(JYj3q(spP;I!EAXnRODz(Rjv947~Es&-(vPmfV>W4O{ZYHr-}lq&T_Q4S;R+G zf#RsoJwM5Wwo^(`p^ul^O4-*~@a1qk`J2z2Kv@IW<)x&L#+Qe@-;K!{;661iR#-~Q z1_~G;V3SW$J8w&NvzVVWd!RuKA8zyJppcm-9wAn-8@FvJLpPc-!Zx0W>STVXhH){l zj7*J?b-y^gS~;t44F8&l2V~79mx8&r&xoMki&!p@yWCpB8r&OYV~BPpymg|2jNT%{aq$U(+#P{X+~* z1&srOEvrgUm8d{s!?4Z6`Sx6g;anRaa zeg*rv8<<%egIPu1p4K=(T4=gv6^z{ra0XbhLw|FVD?t5AZnepSr9e7<<-sW zMdJsr0&eRX#$H~&DrE^>Ui0mgOxR@d4yOc@!5u99FN!j22Bx&k!+hgvZ`*uR+%c&vKHhIa*a`vc zPc8bHuD)z^Xc5({zA#Vq>+L<@(t&6HW9;8Uq3fbW)i=e{$RY{c)P90bG;!qW~q zBU7EHoYA^fpvY7PUxLdJS8&cKIS{cS*5=`MrF%}Y%a2{3T5eJDDhQ$!H{L@n%azm$ zU-_Ml3oV)UgP&GhPpKS0tr9S9VQK4BCh*T#N@`8D^T>G=KsF$ySwoS2PcL9g!liSnDdThU;zj7=89tu@EG?P9-`GRTu{tv zEkdujuvQHICfdpYnz06FNbQC<8ln+Bk!>>I!!l{~(`TV>^&9fTBJG0u8fIC*0@24A zwft{UK{Tnfs+O0?$*a^|5PY>>NC-@miN{9y(|-bP;U9A=Cy}_rq+PctFTFF zJL$hQ-$GAQ9fTo~7|@m^A;pV0B!N;{`*!Z}DOeg+?9M18xAN&<-4LU({z#!zd=*a; zOEPQT&+yQC71SKZ@ye)GIM4T_?6vCRgKKH_9FsaiCL9YClmcaGeLD(L%ewoa5zvaT zZgb4_w^eWOxasp~{?*tP^G_E+F+!(lr$8%2*!<7RsK;IUBDckN#%Mj3ZRNg!rR-693j*lH`aXy*-lcWGxpohZ zI<4(tf7|=MFd7#lw)ZxIAQf;-X%2>}_4rIR_6f8*s=~9zv%m})DsT1G!G^%Qf%ai1 z3A+PAlzA#XlPP4^I;l;?RLx265|fkIpvd)wfO75wxQ0j}uOo#dXc^xPw)c2(CBiWK z(B_qnH3^ZV!a5r_vF6dJ9hO&+Ka-Mse_2v?6@(~I7G@m?q~0A_Q)?$T88Y*4akUG6 zGR|@2ubz^|a!U{t9Z^u}UIpFmix6D|N6&AucG9J(4x#5!-#=gQ4BBv7%F?CTC!d#u z_4IdJ_rEj=nhk7;t=byO3~hH-ja7Pn7hsI7331oB!2)HZPiHY-Fwh}|)$6>RXV=Bu zXRK@8#W#Uwr9~u|dj6VG;#hhz{V(U~AKKb{U-RZ4JA&iP6+XW+BSEJpD4+CjBIpDP z`RLxL=|(>oK_GS0{?EHZ=s5D};8c|@t>S2urREv_o$V=Zg~bMh($htq@%&@g%3gVG}Y#H#^-5 zTsDZLbOKIR@M>iWq;@>HWCJ72fNU6dcxMx!HcG;o9Kdx^sj$^7$vbD1i+zt|inBQ3 z=EZxS#9$Kji<+KqHsRbU=a8t$9Il@wq8OiN5wr&F91l_yWHq^u5krHs6iWd(PBomfnK2z zFgI&OmL?b>F~4jNnV`l{WPetKeeRByNA4z%ntv|o3 z2eS$Z*PC5I>?=dX%fK!T97GfMCc{xW<2nsyAEOlbbq%Fe zcM4fJah{s}u|3)!YaO?LCWWi|Y@u7&8-*39K%#RE5|Gz*VUbOf2x{;yNY#(v?1LT- z$C=lLSyxXqcK{rYvS(tI0NH81kuHq7Wcr9+1p@-RvVTB9C79&g+(X+SO;$q~{<$O5^nv3r0p%}|@e#{yj;87xR+&n*7zi?(>b~rQK?3C`UkMkqcS7$68$U8u`dOBFOxA-e)Qh0bH`*o z8M;R<*71hw)Es;(Mq-C=&*c*#EpSu@AT$PcnpMGBOz7y3NKp)kvjVr^J=`uh8JpP9 z^^YBNNc`p#EhF>awN`m!Cu6?szu@7<5TJUw0c*9)dQX{C_f`vL6C_Z}IQ+;qu>V1n0P2?n5T&Wyr*d z_R&Z2P^8Y*;a~L(AEQ^g3|yU3&JMf=riQpic_#~gTI!ucNmgoODQROOnL^f5zNHpw zNBWHG4o_Gv%_SuO6Vy@J3m@a~?&@OY+h>RceK?}V;R7o?qY{5L4$bH5U);f6NJsvB z^H+j=Q}LL>4jEBh(`GE9=egaaZO;Xo6MPAkgHffcNXtV}n#_kCm1jL)W6;Cz0*~u{ z!!PlnIiSSAV4EC5eIW>BLfFI7@&$MpV6)J;qI020D4?K`dO;&uPE@6B89KIZw`}R$ zfpunaCkNYj<{jkdkEej269HxmVOhskidddIAB=!4>+LtdmX!^GMN zlbKBL;V2F)S~wTZuXV{98?+7t)z>hR_i@@BM^A`B&$12vkPya7+Gb4hzbkjP%wf0c zzq3C4(aZ^RSDP7#+5c(gBdv1XZ@W`lUya1P>-C5eb4TYKw`cD}l%?WHYbLQzl z{uCGJq}L=jyEeA6tLGxx`RTFJHSTSdRiih#de7wO7wVi}@7~|s~LN84KaD&lo5VZ_6D_DS;m?mM% za6Rxp`1N89;V{az-OhVEO>zwrRvhMl`}6Bj-U^|7lh1c6pB3~FhIt-b7HNDYylv0> zFKKc48I}2#_?0~mmbKRHBan3|AUe>Znnd1l{jjv`!&%;0!-%ax21L3OpA%R|L1+P+ zm)<)6W=dU|8>I=xi;W44YB*Bf9$;xux+{nSQzlnghf{2lm&)=T zPzNsk>7e>R;*?8AFaqP51=m|fqQZwODGikH8l-%}KQUrgf<}1#tT??vWU7jMk0!(i z!Xibo+7lbY6p?x@KB>PS%&(hMnF^4s7z76)vWESCa(?qNq`y>Qm1LtWfVTe0KEMm8 zh1^G|u4OYYD$@TIsf2r#{bHsl5G=Q$=!91(_+)MiSQk61Sq0Jr?33HM$x=*cY#lvj z7FqfIU>4*8fl^>(%!n@hkVJbx{2+u{*CnCQL0g{M{YdOQBnpQ>`s zlZc@3r9G<1)DFA-&+ntc>T zU1%@D9#brPeQh+>EZBz&=SN+QX=?tSW(;-d8-tA znemmHsPSK%TEzi&^CR+%vFgU4+#Lq$5n*Y}KT1{?00BFk%+fkKPe`2RN88{N++G#I zD~J=|Ip}HZGmIl!mPuKzhR#u${q#ORP%#wb(nVcjAl#tCOao^wrljod@~ts2*#>?p zsPR4*?AOEIC6Q|BZtlZ}r0Kp=e4NAxrUx6dcGZKjHoi2Ja_|X2>Yw-w+R5SuL5K)a zm7$CSp*)CqF7{nk&$R40K*?D!@jlcW`#&5wFDt3!$+glOL+8{S=FB7ZWE`KGVkuMa zXH-Ed(FPctYNmRmFoEG_kj9Xy_Zbln?AGuji1GGFACFPLdbE;n7aEyd%YbG}yHrs|7*8V!neS#oB;BicW*f6OeZX)(?!wl-_#w}H=(Cs6TR z%_>|g((hp=-%s}v8Y7C+`Nlj#CN6HRZi7G{=@_-fWMVQ79LkO!1bvb=dtmW=^3|GZ z)1QUq%n3#(s%p|a`tp(2P*c-fnknLblc?3vHo6RTy0%+Mvsdlnsw8YRGH{5F+EVBR^!LI`L}CMV}B9wQAtCf}{Qb+N8(tf-?#! zQ(8sYF7;-d`EB2D{I~7)-a3Qh2>B`XEs(Mdi!P8-7%xdU;}8{w;*rOP7;0@IV`2i) z1hH8%vFoyF#j%y1G4;gnvJ-)zeAl(0NeV{4YD#$c0eoLPPbiu$d-Ks(8t( z$EWLq;;q&VD93-JJr0Ax5hRam<|R2tyY^JFSE+tBbN7^3HCJ;jzIl~*yV$o!Cl0ED z09<_5-KR0miDsB~alyid*Lng$9!War90u%1P<^vc4ikxV$ucqNqmP*&P^TZh=sW<-7H(wqz znEbtFiPqNuVgH}HL&X3`xT7c)xM2)FlR$l!VE^Jjvdu9unt8(j@NRWjv-@_V@JY*d zL8ox8K}$5dgr%Rq<1R8C#I|fposC|De@8T0%Y|Y1Y63pXGyq~mm4u)(Tv}(qpWZ)S zw;0jNDtL-uShZh5qXRM1m_$kM_+Eb*Oq;0Vw~z5lDq7gE(3IypU@(B{O%%p$`qHNH z*$yo*Y-`N4wCzt5-!nXw#K@&8;DoPf6*74thd)uKmF&8Vfka^jsfT9Mg6$HaQ-dQl zOkVv47B?~hSAGS!PNRXlZPcR8hwZnq%p^A4rxTaM)2=Z^JG~}S9|GD)WZ#3!3Yta$% zezVGXbVTai!?|ns>y3KWaiL8?`rlp)di#VGcJ%PqT3GDr_*5PojxklT&DiJp6Q4$7 zl=V^+3rNp_&)S6ty;n_a%lK} zHsvQh%0}NR6-(-rxFn>crJF5nlE8U;dPW&4BrekJ*@^&4S?yBPog^pd7HYxO^tTou zCMeq}30bT@g5fwNs=d)+1GoM5I6h1rGlc`o4BQxwZeno(Wq4775$c;DMZe9&bnz z`l4CK4aR2nQ@AVB*Xwk{SeeVH@RwP{AVcak$(8nKuUw1r#?c^lJ2(C=*cnF&;y=lG z6If4#7DoJ76V(7k?*8sCyj;hkFVUy|Tm@G(^v_X{eWOfm8H$2BDyT?8O$9XZ?)lc| z1lIXI5TDxLvWB8!?0sNpA-R4@QjjOekN{v1ti>GLi+)_3yCnyAZLr2oXU;U04U?*(}=^A9hCXZlzI$`Y+; zSRdXbZKB6OAJ(fA6Mzn$!^n^DkV(5$bURV)NUTYs;Gee^do1B`hX3F>++Q zR^BfSo^;Vx9wq4MT&+2r`bpKoo=%1Ri!P24w>!E36D`fZXvE;*K- zmnd@|++<%kp@BN0j|@vZO||c~&Mg1;SU4MLP+SxXV9!82phJ=~VWWu_%M#Vh?UvF6m`ab|V2{y**ektdUP zsYZ0hb3oMTraK&rP-E@Rd>B?cMe@z8g|NGw9cVRc$UT+2kN*uHIwjkyI83Kgkxp7& z8)3J}>fLfULommP*AL?6l0QmZ5rfw}`rcwy$4_EEHor;Kyn=<63h%yvD2soy$Hy>$yr2$^wyutR2} z<|?w4;z*9OQnlcaTve<@HHe4=L90%DVutqxSdQFr&Gc^?u+v?o%)nyI&Pin37F<@@ z5AP!l!`NEvuw<$Pov)>RIR1EA`NR#t4$H}Cj<8Fm`?W_i~{RL0hXAJn?0dV}v<=LS-}MginH#IQXJ z7XEnrmYE)=_N(gVqdGg$aq-6qTP8Q&9OqLGfraFPxV*WHZo{+XR$Kd^H2wBI^5*3* z*wEdIuA@?lmvWbWcr>e1eRbq_7o1qZjBcpykZ0cBQLFh8n1{}YFl9*cq=BLSU!tFT z*XRPOXe-F$968iq3Gr%-OY8kcXHK`h&;!>(7)%DUFS{BeMyHT(`R;pWprS-lBRDh4>%k{f}>@5I8TS^^BXOkuXm)X8$4w> z9dz<1L{)(q93r(M#7aiByri%0t7OmttxIFE5t`_BOr626pR77B5DVdC8GwR94-nHK z)uxg@XWHM`VoaS_@Nn!oz{;S!t+nKB;umI$GIXE)g6SOx0X;No(=nk>4qI4Js* z1wuVaTuNs>_Y7Qqo~2VIjBNP9C**_G+|K>*ThBByC;kHd-K;LXE{G)I8`&M1u)jq~ zwf*FrrbmQxCzTm1-%V`#K6ZW+FV(kfmq~NK+F)iQkENWo`mBwaxQp%~bKcZp)gCxK zV}`qayxQM~aYt;8Iem0f5MShKePisn5n6&yylyu{PP)V$i5U;QsSMgz4iE`rRSR7L z?K5`1Q}RbirHJJ*zZAz_xNd%Bykch&;B{cjOSIbgHdMDvIUj+ac`5A`S5?MM&~{O& zSpHZfiZ)6E0!yN{NQL);r}ZSCtiHq|BWNGmWo9by^s$rLNm$>El5>?>*OATJk8=MT zv*qD`hJ~F`tLRv7~r@Bywo$?3><*H5xqmK~QPBqBXS(Z$<62 z#R@$Ze(}y%ABgG;dk!r4^&r*VX>y?&^qC*kxSpSgake%>l6h1drXB#Ynict#1nQIw zswX-34?W>FImrjcE|katTC^FV4!FRW_h6M{LC~Xz{(S_bIqE$>&3ZOPS$25YLAoOo z*$gBRxXiJ4G>$ub$(ZK}nCuu_9K9EzUtpiRt~{Q$AB0Qcct3voeRl^+P46Df)VheR zRQ6R1RQmn9Ba50*+_h$P`tQrIO19q=F4u?VH{!2&T|Z}N-|mR0UFPvm8P}{(7zTO@ z{{ZF{R!41{Q#Ho9u?%>^+O7tUia0dHr2C-|!dHHiTNi5!$rLh!!;zyq=0JK7phd!AMu(s*@3~n6hr8cl6U!Z z*+({R5?}eM$zt{bSrNvMELNXN|KmYJb+Vpqoxa#y`rie@aN(jF{~rcM9TJPV9cXhj zISymf{&68mc_xWn{#&BMHon2w9T^MR-!t(5N=ul^!aB}O@S1yuOGPz zTz#6n%(XyPvJe_KPCw^PZCej36=+Gk{Y6Gxmlv11SzaLg>C(Lh_p`nKmKYGnjFzsI!~ZK#;}? zVHIbuFmG!Zy#z0PelzJjlz_w2LzA{?E6(ihi$C9~KPd)xi{1sGYDf=iktuZNfqT5z zw>jU^z#a)Pbe(K0SlKE2oht3!NeJ;S=b`{ft{&ez!LRKdB1B-X>HX^6?JB`A+@{w{ zg{2*=8UPnX$3;Oc(=B(jp9c5X+?lk>u@P?6RAq2C%_%mTcc`P9S_b*O6>HdHaIjyH zqB|xPr90V%gSWA?H2{J-<)*J-?P$`HlyX(5tpUIqUy1+qrQm!`u|LW0B|5v+SXBWz z)8{LuVvQjigVuRTl=#W^nP?3HtHuJ-0%@y_lN0QET{Pbu%#EArfJeTXA*f#(iefUB zJ2%!m?=M(#3!s_DjJJ~u9c84xh)Ph`4gDM2Yo>0sevEzmzNJ?Zv|NRmA+!vao`|a| ztbI&+@2zr@m=@tl;v&MoU16Pzxb~5|;u0`n);w7OdEZ;UF#P*DcNfb{a-^3Wv28TE zisa()^OvpYMo2=t72Nbx&4AW1b|?Zl2E6Q9uCq`<4gZxH3*!n2XFtZ%JqTA&VEEdI z;Tc~urRhf;=ywntJ_@!?wW<(W(k*>#@r!Xkz8;8iz6U`xEkp3}s4wq0u23a@1{b$6 zfkFR|IMGk3xk5lzLq`EOl?n{B6;XnZ!ImgW9)3Nc!z7ksZ^sb|h zg$SaaZD@@+hn=NCL?A%X>-?cZ;bPw>Zalt*fsc&AO4$-;`_afnt!s1RA@{fXkc@vG>>S^Y9{7w71ue_~)0m6Wu3h_ui*%7ZTz}+;z}ovatf}r37IT zS<9nKb|AJn#2Ex|qu2o33U_*e&2z|A0Y}tNe}{5rQn{am>i$u>2i`bBig#7?;7_79 z7j#556va-^?j5cM(y|w?)+@(goTmITE@3aC2}EiF*%*3IS4Tydxo$CWS`%eQH6A-6 zT^PS^L#^J3w&Q3%UDy2*^js(MMJeDfGl6+eJ(?xpa48~K04w)+faKEOJ#i)V05{1ahujRGUyR z%$vH?hIB`ZXyh=M=i570A>|-+MHK9po!PsEPKnN(U@khZXDi(ra6af6qLm z2OYNbGGIA9fbjrG$E^I2_(hdgb_!~k3B%VU<&jynofBNcY?NTK1XCB}G(VIA&v9CE z8Rk`sigwgJph#qM<|uY&@12706!sQ{LY(Yd0}qLGlgmq)gX`J(a(X*3CPw5ado}Sj zwc*76McNe=&3iXk%F0JkeJk5k+DwGuTd2kHVZ?)JbZ#yDNL4x%tNH_!8m&x0YPwSc zJt2>caI|{Egmkd9ZsbdiHTYvZeDYNTiXJ-N?21-FE}T&b`6~|F$Sd2^$+0K?TGb=rv#V7FE%khh(vT_#nK%y|otN^GTaI44w5R^XXj=fot<}M1c&T z>4*rQZ!n|Bu!+@);orW2xZt58L)~Y^uFSJeQ9>D)c%o9`&;E^kX;1WLI<8--O|_cs z)H|F- z4t}V1h9KepY_sa>ANi1dWtq1f_@(AQj8YHG6z86d!DJ7RK`PhR``V-+9l!n40p!w9 z=o<~dB#su&po}h+Sk`1PgPmplMy1ymO2>nO?Ma|M)~qO64G_s#d@TLB#N2E*G8t9G z^v~d4&Xp-V54Ck-2k|(g~4hEtI^O^<|UpEpbHv8nqcBZ7ViJoEf-3 z=mVU(6hp3$DO)@SD;jY#L79oWKvDIt>P|jzgQsiZGkQ~mgjZ&z#C$EVyTL(@E0u5<_0jNFi>z_;FM)@*fz{t-6hKd=; zxhN6wW$dFG_FKv`jg4y|#5|9p{ylP)OTL-vo=)s=Tu;>6)5!*+oUohS;UPSC+eHx< z6`UT>2T08edxA1V(Ykd3G<8q)Enq#wgAPR%vCX^ldyB8EJwvNy7~aY>LgqIVxfTO2 zE(Mv9(lO`FZ(xKjtz)8|EzHAPMD9!ukdV8?1FFpk=DZ8^`F9s27kZ6#vR?AdiPXSy zdA!0>WmjdCH|ByU`vJKueCvh`!Pag@?&ad^q@uy{6zVdr;0q89to0p|!lw*TV z7S8I|nvZ-ak7(Wm&VgrrcPD)u!0QcP-}THJ5o3qEF6X2M>NTlt(7$en^UI8hFdt{g{X7DDm#)DI~|WCH(BWnl8~A z2i|>4K`(>_KMP$PLKkAHeDA2fk};i4d|rDH33Ac?EJouF?!ZqtjFsa=_!h_D73sDH za*3db3%W;K6)PMi+6n*(`rRlrmoQtF=#&?JB*%aUAKHo&fT1qx@K6lUX5 z4Ff>eS^)CAlfFF4Nx)Dz>i9?#UceV$(}(ivWZ_Nu(S>!L=LA@I`jHT-PfZ7)O{tWk ztr4Ft#}J`AP7SRgM#y~nB*Os3Bwx{?-f2O1(nF3A#azYrAYU^71l~|w+P2ZY$_CLW zv_0c7R+9q?FFzRJQeBg~yd?my7SNIfC#Fr$M`s9*W7~J*9Q0fO>0b>uOFKIV&~#O6-CcjU zrVlx;;kB&i8}YdeZa#3p)iBJvp%bA-N94Q{YPz0*G`u;sQwZ z)eR2h4YK4ak|HyBmLbBGGB^l67_6$9*9Ne*{p(KE4c$()ecQmML?_K>7t(gwsPlnw z4iw4rGOph>!~7K*SGnPUvk21SW-F3b9C8kY8UWkox-j>7b*c?ookR3>=pg;zo4Ep|FC3=IknggO1L#9J-a18KQt|M2Nvuw-CV_mlE%2* zmYX7W(9!2!FYExo_NN~qWMkLnN9*Nr7nhkYk_0COarL#YV6^hY<1`*g{weMB(DJ4o*9cFvx)|2mD_)BO$TE6+B295viw^a}Se=UGT)(OWg!SuiyzT>pv?~IrM z-wWrD@kJqtnCPcTw%1n4Qg8g$4#6baY6U9iujTiiXR(NULeGX-Y6>rUDDeaNK%1?) zzc+}IfbWr`SS0a8iO%^OAaUvoGgH{dzrr?ui3RS&%f;`AL7hwuTgdo#0pXuC)_BrGd zJb!oNw-zT$rLP0-RS3dxWWnm5-oju)E--1Y;49(~91Ca#{0TQ>&RK>dq5g~kL_jqh zHBns$H2Erp4B1;t8|I@lFa~Z@QUnngpt-ydC}}l0g;$~(2NxAiO7sF@YNqQfZ)zGWQ-L|N- zu>OW$_JH%9Jp39g_NftSEVG}0bU>UZ65`KU1+@g=56GAZ98_u*)f7wg_I-H@LpO;_ zVGmG48ux`p16|-I0(Z*~9!8$KTq1GXYW!Ghlii(sZGddbDOyZ1Wtg#ufpL!2moffq zF*cFaKB3CQ#f#5hY2u^ra3sn|H8E~d_4tAPGw{m4+iG5@Ws0@ z6LYY_MnGj2RHnq*)LSV!etbqZJ`QOhwA6?$R0-3=hZAU#IM|uz#UonIq@Qq`XFT;M zoOneF-AMCEzizlR=|mnU_4ZjiRKG$bBokO-?&Eh7Kw%KwTY7+wl--DInww71tUtj~ z+(p=SJITLC^))UX^`N9evHN;n6_TClpf&+sUdOm$a0~icPHW7a!qYMj=@SAQtd-_w z(SCoN4N#|p2XLRh-!F}SgFv~}DkG(VLQ#FP25(UBhJV?MW|H~u8xf-k$LgyZh_1od zVLUzCdrl&rcV$A7q^E+a~~@#eV%26YM_3ZGQyNXTHl8qPIHT2SeD_p z_JnQ4Ym!E@(F}M11QQT-vlNE>Yfg8`zZ=LIHREz<6iNzGqBkW|H=`N4A`^xis67| zpiHOzgg8`PB)T{y^4Ldh>*fwj_L*Y(OfE4#mZ9Jo{SJ&_e(+zr4)h0TwA+d^A#0=> zvu+-)n|pH#Gq7&+8m;$gXGT^eF2X1=fFLUPZDTV3eWhOY)xzZjQ^=A4rn7hnG zgY%X9IEf%OMF^GSQFW-bynLzGR?@ApEC!q!PN$NVqBYq_^cuboA2FVxW2Y*}V9&Vx zyKZ^MR&`bpI^m@{6}i%W){q=EGZ3Y=`g?~T$~Wdm8|DW#(AsLtw5n_bX(3s|W7q?8 zGeG{z?pM0yV9R_}VM)j8` z&DW@@=x&Pn=aborcJr_`PYogT3rlXeEx7QvCW<=PA8H@0?>?l(ynf>Bv0LdCB7JMY z-iI;dhS0Su7=fmeI!c)Y`*oi|a3pWj@IYCY$M-&h7F%RX3zYSwvFo3koM2S|(4BA8 zkpeOzmJoiT$LYqeZRo@E7@C(ONvt!uGR5n22eUtllm&tbtE4c{0N~T4!lMn#ryO64 z>Wp53`7D0D3Lu4dQE&;G-BCPz58DW4AdsxGj?-A*kSmv0zg9SUhWLuI@Z$NdTmhV0 z7lf9d=ldJeO0T*hc-c>O!Y@q>H;pQAT^Z*=t+Zq|1HZa)VcDTHn~K$tTEF8L)d}Xl zlMvi1>+h~Rf-!&R>}M}KfFRW69p%@m><=8}3p9n+6ywfd=kI237m6|RZOW1O)iV}9 z!YbXI(Q^5UCb}#oi{?S5w<%|#QfD4r`f7?qAJZpWqYv#F)yY!Nu>Khk=fm7fOSO%L zA&Gihhn6_%TL&f0jf8L7c9 zXqbU2r9=c>DToBUpLcp^aH2i(g4KfPgKrP#&(Nw~5ShiYh|k@Jr2`{p3d_mr0wC=x z6MblukcS|j;6LH1Mv=g{PW}|gV=7iZH^Zr7I|)C;f=4dXLvR+~sv~TD;ac_$x|M5U zU*=7%as;e}UluG(4mb0Sg{Km65JazvK={x^^`_ zyt*A0lkQ6Ufp~l3x9HI9LI|QD2oao4!hP4YWX4g0FdgT3`j8rOr7)w8IcAYmHcEkW zLu-$96o~Prh2gnK{ihu;GN%U$pdC3O9qC^cn8(IHYrzfj$aIko+}5fOpJ8y{=BlG^ zp%9SEXf$3$v|*KnFi0?lyE^}6KO-brK$E#stTO6E-jgj|Pk?K1vr;4{Y4s9o{=ajV zep%a(=;5*`ag1ZVmTzco6*3=j){iSOdNH~?%y=_KE$x#MmFRi$$n5s?wcC_QW8tdf zhx;zVL-9(x_On@otej}{jC7fGe$012Ho^#wN1ZnpWn)&_$&h8v$k7s*)JY%$a6b#^N4VhTNz9&(`$!g z{V5tR_XRSX)H`3*o*JUASQ5ef&(h7EJh#|y+cfjdS%U4@OUpq?!8zFu#~vcBGmVCw z2IOt*>7Dx<lVhq%_t)@yd$ z{q?=$y)=yIP0acca=)iYmcNM#grme%6rwbXF#jl^Ii|;0*VAA&*U4PDVIdWrSthY-^n*KtRM3e-A=pk9XWw5@qOg zlV;~XV?Xeei2kBo;5a!QU$R>z46MKuRK7XOEl1UqU69Rt z00mn2qAHB~qhT+U9E&dJ%i6KuZ($_2x}Lk^lfOs*j*`By))=O%GIx4NsRnCB?0ZNU z5~OUsY)G z1YH0a{+6&Ml!xGp_8awmf`hIe+7;8sZ7`f&dQ==<*J;ts8JLK5<`|jyzzr5fE=gqu z*dl&Lx=H-wKIH#C-2DGpi3DEjI8&1eYh zm}~ClCeQa3d-`@n=${phV3v%g?ft>VN~`V$LgNB!U69L7B42|Vy?Q1o+1%6fvVo%X z+%mX9B;^Iyr1Q_`$d&oQk2e4^G0VSOVvrabE4eaW44Hrd7kO%>jY=Q#ow3lM%-&1g zmcs~PN%6`FvGfQ+^=i@{MYlR(){=`%F8Ut`r*Vqeb-2-)neN@5aN8V|$1jvHjzJ4# ze#ygWsmiKz9)Ku3&WV?awpb}nL2s<94k=cbheuO(zyBHxXLjmK2(kJi$1{ zuD9uCb3{Q1xHZ`s=gRq>t$;~%qeS@+H!k7*r>&b{|+b#OmQ0D*W)dfB#22nq% zs~9S9?9FyEcO(fLyX%=*7fS(ygXmpM7!6rKtARiUcr!k&nl{R|>NrmaQtz-3jun&0 zlmNGq^FO?_r`I~m%enyeAFsSB{Goo`A{F153x|;3(4^~mkV>0sH7Si~g^`>poF)Ea zyx%pyu+UWCM6+lJd*_1zfhWb%e#~HXYBT`-V?sh=V|mQI<|?56$+XLNL)3~#*~~}B zcFyOrH1B)TK-SvNjy>Y7k5I%r^-4eJ-E{K4ByZ5aAw^Km&t6+zySk>)=lDt*0cyBj zta0X^zW{}4kbx9GJQ@I2!`uvFR=QX;Pd#u;OSQ8a3?Q*$SMGtAxD&u;OwSA#M*Ci` zUX_?B!QDfO_c9&4ZF_i6kPoJOB$9mJ_ScKGr5v&toy{C& za}H#IIu~oM!;UOC-IYV3G&co{V0=bK)ObJMsz$%FECA?tAumI)K0rhD4s5JC>C6dT z4uE&7Ry_M@>(%UJ+AFgSUkl`gIwX+wIA@IkfCx^Pb`8y_K^WHW4kLYpHQ%|^C4@l= z)`QG*tX_bCk&~|!I^NA!>>fN~7ZIJ|81&OR>odW3NhrN3PS{gHUYzH&=ws+G3yLRH zYfSR=Vv)R-1Ly>v!2B|WP2|7Z2Id5D%2`P7Y7dk@aC|eK%9C=;u+LoOIqlF`f0`F4 zZDDuzHl7djwX_2_`Gu?6)4Y(|$VY}v(Wg*FKdOvsjCA1~P%pJ^IOpuE|2PSnE1xs` z%_NE6KmGtn7QQ(!4E)frO4=iI7cWJBH;zQU_o~D&2knkKh1!%>q`vYFWi4SUfLNJ+X0o)6 zexH_-dXMjLE&5bm*J)NlOe=>74jeGq%0#+(0IMF+mG!<`4{z@i4UbO@s%L6glB6%NpVA#7}&F#A0F2ddu-A* zilT7|&CJxgu3i~bk2H?vKe)wUT{eP!UIrE6qc5@z8++r`Sci_{N+rk3$+G2>IO_F^ zGJx}HyF#UQ$4ig!r*}vHKpD75meW)m+vT326nERc1e_*m>k}eu9xUgcJpiRp>JIwT zh1a4`agl$XHjTPv5X@VZ2a2Q{Ik-CSb&EGaI=YZ_93?~wq5l<4_N+J zw0G}k5Zi7b(#R|Wh6be54*MxR(pX0hi7c$S84}gRO7K@K06##$zd3!O@}qV*<&~{Q zy}-trND*=3F?3@8zf!2kTMk|EZgn%up-1p_&@&nIR8}R9|KwsC{CrtzQ!_#KIc|`3 z$E9_@yf7U!JwMIrYkZ`NggeAPO%P&%mw(MeHVXw?RACcUY_qum4+hhIE!<9}9Mc2g zB^ur#kYBEKWM>#{YUPhN49$DWTRI@9={4IfpT!)^SaQl4S!acEt_l22(i0_s?D5Ac z6a7APK#9&?GebM5o{Q&{R3SD3X|-ofZ|TvCu4Lv@k3~LgGUOBvqVyayT4&7Mu z^~(IB+S=6;+gcg6NghfgITCOJg;96CH(x{j{<(=ij)s(^ww|4WHbzfSV{m4zyuiW= zP4Lu8;ZbA+V=-^y^#Oc-cW{viJN9up=<4)QvUOw^oh|8iy3=*gz%){-Atl3Vg{wH^ zeO7}OizmTd&AX5M6@jdKfe3>;nlhz^d}^u62`5&eV3y^fN6tuvA4>qHL@4~`cd2W# zE7}P|)WP4r-Y zHd5ET>g^}kvG?jp5sFCSgvkppWvny8250l!g+j5dTF!we{V3+?5)>xm-%+4?DicBT zB`2J>IRO0^2*=8u%?%6htHVq08#^NLo0RByC+X9bCBXP<6j8SF5PrZAq_ldUWsfE! zkK~#`Nrt~bU^->U)>)hpWu14Y0J?xFZn<59i}>X?|+6x5xq3#o_jzocv&S- zARg9g6<7yQ6%b}ROpvz`%G8C(cU24C<}uh5RzC-_)~YA{vc?4CTipitgVY(5W8JVG zqVUR_vfH_K!0A}OL8iwR>$94+0idCV1qSi3P}3MQ%h1etnaaa5@26uiD0(%2X}CQeFZmmF`Z*%kMC(O1xn4+uK<8Jjzm;<;#NYIy_yFBHlW zj;*kczHeP)n;uC6`2pQ!V&1PcT=U8Wy@mO5tBIAMJtL-)in61f=mCu98((D;F-1C) z#D*p1d=l0_&aU5D#;_@108yETtwUcvl|XJXkuTCQ;J^$Fx|(5wE@;Kn0 z<)I`nxHWyKn0LGbu%`$R)~n-EsU&nLON2gx(%oUdG@ZG zQHm9`=X3Q@I$CqOnX8GpU9eWb)XojOxS`4z@JLfmsL}cMK2}PNOSh*y)z^>q+(yGm zJHX|wC1dpuvtvq07~U;I@(;@s{Y~e&-885N}^PGdUlDwdPPMZ_n zw#}VcPl)LNLY2b6Kp-7&g0hu7^lTG$AZs@+;1CX-STssxHztew;F^Ps>z%s6j&AhB z9n2gRnCp@wJ9+5=9qgADIETeb7ivy5YQ*zpo2;?5JClNd;2&W-7FZ%0d%9&f?TP}m zD;*2!OYA9W5qRzR1J{sH1SQtg&41Z!X4)@!sw6P2@|HQ>H&8B)$*< z@)?rhn+^~~-RG7k%22dp<0JxVAd=3;uO@Sb^De|Err2=GDrs7)fYSkYuCX9s4(fpE z^H8XRKXg{qWUdEq)r5!YE$`>BQm77}4>cFXpv_Ph8^Y6XCOZMc&3_qtt$Rw3vm-d0 zJc_B?J{e(d0%F05Y(v?F5h`J})Nb&&LwT>oQ>Ho_sbcpOL49QWg1h2zQ@~In^an%m zf>Y&%|7da~>66#WmAll#DOh(it4)EaomHa5-U$+>+GRQFOl3+@wVWDTE+d&r=o-%5 z5e^7>W@i8`6aRVK7l?-?=?@iVd=AQ`rVMPMGc^$V=Q8MA@Z6`5Vs$YLZpp{+f9uW{ z$tb*fG_Hc|k{U2)E?#~~G*>u#kZg}fE7t(b>I^WW?V1%21fSdjr4X3Gu3!1zf^1L_pjYED1P8ueJox-p80t0yXEi-R2G+ z_MXazkb85r@cRjEs?VrKO10w;_KAGXg%Z@U!E)%Lcp8*tG-4GC5`>Dm zoXjZr49ZVd!y6GLh#h9|aYsaP4!-!F>bOyOHcH^*(F@PTujCr-yT~KXUHG--NFK8j zg%h;w<{$;ORxRoQ<_X==*Z@BomR}F~y)jmHx~@;p$H9|@cDW|IV(G8dLbb|etvX~T zg|Q2a0ZuGJ6CQM)TUKT6?9z`>8n0SAFp}Xa#bWu_@;Tc)0Q+=PIEHrDJ#_e$@lQO; z=TQm|G^bn8y2RNWt&~oM9bzT3fflbrgP((;2v53P^aEat2Cp!<4%oH{k*+6$XChmd zf?6@m1ouQs2R6$xmTMhYdm#F|9VEtT5lKQBv$34wQd{v{%-7B2N|KK-LS~G~UU9V} zO#=0$N;_kknFBkZM@ma?@{&P5HQZV%fC8|JX$)o9#-QPZkAVG|a3_-m-?+)MVVRKt z<#|SS-?)gh+7$1uX+q);d-yqRm=8SFpxGsIgna82?7R^Ij8u+g8 z9AMOLbveMTx%!1IN#2yGQVZN;{jv|gb^todWO}oPh=)~$BYhLWDMBI<+{YgSmc{NXo3X79CEcD(k4 zGaA@za)f0Col^9M0{fHe4f>D2?X%^uwv~rgfgIUENmk(~0|1NDU!)E+^_rW{6xC$0 z;VF%p@zOJ3(Os`O>vz!_oMDS`F-RVD_c+Cjs+C3=6*Z;poW zbSGS`ZhYj!ZJto(af#`AY$2d2$m*6)q)VxYHNEfS(3|6Bp4{P zS{VfJ-^i&9z`*AP_YZ&2imtTn5X`tH=Oyr!it|L~{*qc&!l>vE}hd2jc~bx^VO*@rV9z3Qqx76DwD+}1Wl(Y}bP z*h)Y@%PNlRVEvdU9X_*qt!x_rz%?^wXLM7%HcYfMk?hIIIUQYog_m$;ZG4j09Z;pT z5u1eu;3}_~wsQ&H`kEuk`0ZIP+so)29N&OE=60|nN0j91BM%eOz4RN`q5O!k=;nef zlKf(aP-ZTIE8^anB*vm(ls}V&;@g}8TH5ths7T#hvAAU4fRuQO@#~+d>3%5N($Q}p zFHHNK4_+$LeZOAPrJgT^SR$P$P{4dbNzeIY#Ibb1{cDCPVCB#C-4vlTZJy#83Rf8& zSPOM_VJm4G?hp^rh^=yM&UKWNzPBwL8q0y+w(|n=VfMxlxomk#^II&a85E7;#rQtV zR{aOT72U|_ScM98505wTN*@iR1&4d4z^z++vQzND-YG9(9<|C(OUd%fgzzM8O!*e$ZN<)F3+HB^p zt?xZ@==h`cF_pjreCg3bl)r$SA{$7`yoIb%^ka5QGM*UAh(9BSBWp?^vYIk!e_5wDMGcpSavYUzeu9B20FZc|8StWAP<8|Xv5 zpa!Z@or?)3hY&4k`!?YYE49B&ZS{~a$Owcu57_UUR0z2~>8~PAND0rXk(O#H;>M#q zhaVR(oLx8Evgm@4e=!pZr>D&&;j<=L$YTF+y*9{b%5^_o!yf`q%lEj;{(RZq8mKXd zY|R4In$OR_AKUJSNiOOz6Z=q-Z4I!L(TipOKla&!FZpfyDuLV-9@&V{a)uO)N(l&l zE`pGwtJ-vrGQ<vJE2vRVm4fLi^CaGdQz!*Eqa=o*$z zR^}v(CixfTnKdoy^oV>frJhTgF3bLOxrZsXi0|EowXtDVY|fulcx#mtZBm>5)_51{ z_%xW*e_&h_w}5k)j0XvL&3h5VSGH6RmJ3260fQN9lb1(DDF>LFx&AR5{{=`bD6gE9 z@sI_++Ut@VXU`#~aj3tv+2F7kL{va(HGjX!wOnL}{pN^F2=TZqlLGCH-10YS&Hb7^ z%n^HW?tzo10(_(-5W{AW^$6Yu!O}HB1BF3iiY2nd%Q8NY(vLD1B;dMtSmG8(8#3=e z!vnSyJ5lq?C}e@6J9FCW5%+xL0AMQ(KT zU!a1iB1oPDp@Y^Xxv|7zqFu#ZKNi}Cz^k~w2JceyRC5$;Iz;y@Hj3E+{ExSt zn_E)fXWeYJgJOIHDZiB9ciCfZ>CFIgk=RW@dfdh~!ltw346LkwlDO_an0*W+{m^YF zLg)N?^a3_Nv*xW!RpwcZV!V=+uD?EH*aJqN=iqtE1HIOk-EBXPr%55kc5o6hXCDF8 z)_82Tyk6l<-U+fs$ire1Il!^K93J-(CKQc*3(y|U2*j`?BT(+k`JV(au}w+1q=Nzr zg?()7YSBgwLK)~enNo>TA&efbC2z85CZrGhG)0LaWfh2qs>xGq6TQzsKZw|o_n#ks zU6XNu(xB%qT_!vyE=i}1Yzp%weg^%aEztL>jWMR-Af*>MHErr&WR8WX7t0Cg*sdvM zxFZ(#^fxUiN*KCzww33n>j|3N_$G%Lo<*_56%j{s`VNB69Yf@ig}uV9B&8T06+o3# zR|Qf^1`m-g>MWr1FUfRq0Zy1>?l4LN{ERs*%7lG({W}Fq#F{cT)O`LR%!GuM2rY)( z>ZwAZ`Ze2pO;cji8d56?U;tXvM6}kZG(p{pGaQO1n~$~z^m0RkEFCx9izKxq?rD{Nln@VL$N-=A6Ny+55m62)Y-8uU4l>rA_a$zOg5eP}+E{C|>xM zbf?FB-&ryx$#W*AN5n$3S;Yb=9S-q*li{30GI$?8^(-Y)sr)CZGv<%ec*Ml(2PXMo zFz4CLCv$rViq|fQziU2}W28GQNAI&eOfV&Qr#6O9zfS_JY#F*I!PmXJ2<~+(LV<>_wMbkiacH)ys zzm}Q&aU+z6C4|P!$nS&f``-H+-RsUQ$`)kGLKwfs`fVEk<9TFO0ht6~bjySwdhr^P zhhl0YigYf;H}aVS<2_sZKJ?vCo1@NEw}gFJcMk?8|K! z&hi@uCHUNoHs~%lgYY7e=lS8gW@q@R0EbHs^}LFJ*H@`K(iNZq^|vOH8OZ8zbUe)6 z?)#ID>7i16tb{+pv%|;*x3>!FHuuV07}onMkR(_eQb-;5*jy22AM>XD z$B5-&5!(UIn5|$LSNWdmH)m-LP}e(75${%z(;GO;(Ls5p^9HXqtjG2auGwc}P6vZG zr2y?R9W%FN$V~##NYZArBF;;!gF?$pV!vtZW5PeP_s9awc(`+8LnVaL3^o4tqhgh` z90d)>JOqoPrY5u5+>I4DNzx_DkN=yWe#SXqa7OK5V!j?={puWY$mec0PPH1#5)E%0 zI+8QMmZfF<@(6d#%a>5NvpChV^^%KO7NW`#)zZU+8!KCTv3!yCmz*bC4j59*$~p4> z&=9vr_+;GpTcgH-3Zom7TFOqC+X98lPZ-<#*^OM+_U;S~o2OnB6Jjqo^EwIyw@^V0 zR#)5+19X-1Cxs_b=1MYscWfL;m#xrP3ej;xuZ=zHks!PQ*G0=-m{NTrTG=qx9N@=H z9N8Jo3~uvWz>SEU7F#jK=MXVdl04zic(C(?Yj|mD9iv(DwymUpk_(OCcplz)8eo?2 zq2nZMuF`V4>O4;xER;$Fpf)DatX>5ffPchiU}YphhfMP9Z*I~Z7CiH`KVgrl9P z+yj=y@#m9p2C`bz=OadRmZ?Nhk0Hd8}? z@3oW!c1ga664jqCRZukEPd5{sZk7@W%YRc$7iD~QeiluS)kWtw9$|;O@S%=> zkEw_v!vOWv%9;pbYr)D;^WOmXCkHl|u75Kub;O0cX=M}!=+8OQKPfklpkcq1_Bj;3n8?zUH!WYn zlno4THrD^}U(yU*=64o-8!(dSUi|gk|!mT=k4H7_r&S92$tX8tFHq#VA=XbN-`oSRx zzm09ra6Mr%&DKDL;+gh+%yRZNr>A9=Ai!MpdE+0lj#U zxM6}a6nyQLY@v|Cl2E_5Hqw9|Q=Kc>)L-IUlJc=Jx&sZW8i>?n?O}zp3WID`Q|-M~ z=A&prdmg&v;^`2{hL)+@=ZU-_hn}xk^aI!mUbR`{nKaP2Y$bx~h|p6tV6_!5s25*< zF(m?Sy(i$J@q}HK-Ns9Z6mK8PLU)p`BGw&dUx98QV|>Z?g|Y?uGskP#WL8>zsuX)5auu9~Sn&--qGLR3q#^=AgM*KhKb%Ks??T)5J6*ffl zNbdq5`{w0>`ejZgX7s`{ML-mH0D&9IawRUVJSXxb(G#PK#B?s@g{ z{TQF(x^Z%$`;!=H==?X#es{{x$>eRzbC;~(d*(+w_3Uz0leBijtJVOs^7L{Q{sAu& ze2)5%jsS_UsRTJ)HX2VMXLr>DHgqzz+WaP~4HLj#nJq+C4XX)QlZ7vUj>!U@IY+qM zN$vWl)ilmq6&3>YBSzy)Lb#8e07n)K)1=rd(2GF@sA|9I%ziTFTciayZHlq`2(WMQ zwWYYOUs?;->~!009dmX0bBb{-+Cz#U^gXk2CICcD7(7|DQNg=|;atq_{A~-P&()^y z*)IK$F^tZ7?T-AF$JS%N^c};FeQ2fqUN=g|cpzAkbb{SVGd@Gk$3#CCn_iPL^JK3? z6CexyAaONrWF=FvJia`gmtt`CBmUP=j+Gp!v z@XvHc!W8HS_el2kl3ik6*WaOp;21N@^7tbrf?Q5Dff9R_xUx&<17_mYBKE@nkx@H2?Ty~L|!NjLMpTmayNLwT#84Kov|SC zOTJr#<_cYAQ`FLE`uzwv(e%ubedB!Ps4Yd6n)g8`iP~YY9@$IJX zbHhjWPvizl1VbIF`V0bRWtQ@H_}BQ=;KO68p(U?tkr{4BzH@G%ps#qXT(kHh61Z@= zV*+^B?G|_ksr>;i%YG39`964e1SrFA<`L+Up!>PF7mZ*I_Y>WdZhG#%uou5>QhJl( zb+`VC4$3S2dDpFV`%G}Tg=$u!5f>iJ8Krhy;hr+vEnlNbJivi)Ml*wT_`i#{WgO`b zd}nLEc5uF1`&kL>B3U3&!ZT!-cB5u|;rCjbaT#MjyZwwJY*xZuVmE+}oX=H8`il|{&ai)@8 z?M66|rx47i(v;RYt^%&l zGEtdK%^7}3uQW9!Gij?s_Rz2szlsY@-uo7-0RhHVD&ZOJl#9`=rXVbR9{=7QO?z}jBE~aG#ak*g{L`c7hBLuzjQ4{x#tJ}oW}qAfoV%q5?1W*w zGn3$&9O3>>4)z~DD^|!{N+^EnXK>ear*H8tX=(Eaym9a&>18HEMSccyb10&KPB=3c+8pwM{Pfu~~1YO-c8QaP(=jRV{?oW%S9lLKW+@1sWYf7GM>=#HeW@ zLR~DJSsp885`s1i)uX|BswK$XQLxwr7KDrZLNj@w-z6lqomDNMQAmqySPFY``ZZ(c z5G4(z?GS))$^A#FVPN3PPptQ@adPeR3^YQW-z%;dz4!jZ0Dk$GJ9ia;EQ8G!=t3^o zrHaLA7MNBcf+Sb*6_reZ#iNw1=Z-NN_ZB9zql93$7^#*rvWh_T|1uENdMP(+$AAd< z-Qol@sYis1<0gfoWLiHR9ty@#26i|xcvjrNWef+93$aV$lmM*_u+>_4bx!DhhY+S& z)8=6n<(wzM+tM=w+ormbPzls6GmoCC3Y3hcN6vjx4uHtH2*NtxaB6?McLP@Xi&)yq zsAKRp)9LA1VU@6N+eHtkdrJ)=Nu`A!!<)+1Vf0lATCte^$j><)q^cM74%)V9baL%d&!7n+8a8vxfW;I>4@3h@Orz0kEb^r%E^C^!tMT9Iw~g>{+{7SY&b zna(2-qzaO)m>A6_GMZTf8~mvnQZYugr06jfNwvRMaSnaM?-CUc-5*BiP~|v=&$c_a zRMA!$oSuk!!JMj+u@B!2Lg zXM29_K(?bFKit&5r&Nn1jvu_`l?&Ea!J1+2)*^Go4>U7y*uPS?F9Q++`Hhs{ny)0@ ztcqVqPY8~x_25lN3_kjds{h-HH$2fw;qr&QwS*50@`gzo6lb`hGGfR;Si^ye{W0|x zA2x?wf-SFCJ2*VZIt3mV?kSI(rmVRkHq z<&W|&yvB%-T9=eSt(#%TUubCeu~k2g38U!#=LtJ5h$f|z~mI0_nwMr{Y!Y7DW06JXjI6UNDU_M4jtEn)l6=p zPewaZew7R3LfvM|i2`ILj;XFWHxHAf|A+A690G3^a*s>j+$^k8GSjZdLSnine|6QQ zWVOshnI~#VTFflYIz@%X-Vn%+$=2Bo5-+W3C>pyFAqxSmzHhpBv<=E>7L&?b#j5Yz z<7ooR!uia)V`CQLMVdO|lzR$AKth)6TtS>>i{(~}fCM7P;$xBVpEljdR~jQ$a1J~a zL2W^2KYd*s4#IK9Djh|$ap2Z4x55-Vkkp9~F+|A*?IC3l_M}nSCROX9jb;IhaeNhUSDNZX&HqzhQe?4A>?h#j6(lpxM(qB_f!w}wYDhV*-Y?tJ7yATv zu?eeRfW-ub*0a1$zz3=ON8Efz-eU9>@v08!$#CQ0fEOL$yppn!Aot|MPQGEF%c9IN zda7A+g6vGEyP4gLxZC0KNhrIxR=$$hvV#BoZxEJ%AtU{RO;QofQXn^CzU|A8cOm== zwniq1=?&HX0FSjdIhR5QwdDp)Pcc)!Zo+u;UDmr$v>o78TlRrjW*KfF&?v53c#JGv zUlN(r4_hONsWI^z`DN;x8!K01J+G1`Dx{@l(!A0$#pbX>fXEk~Pf3oeyg+CMI4Vpy{-w*Pbz>9?s9 zhouUbKm*)AI5Saz8C9W0spwCQ3k@|hO^CJ?!2Ft%R&`OT<;1Xb#&)YG4QWv(d%zK6B6cr1j(PjLJRp%buqS{f+)eC+dUFX@-|{*>8^0|^-5Y) zDoV<8MA*S+wC{wzn@VKryS&eiV<;%hfJiidsj~tVF&8N3o3DYyvKuMu&4(}?;3QGW zl6fjCL)1(PI*M7HK}FJi74cR65Qe}LADR}mp#evRbr=2kG*5j68T0OX0*TG5T@sKi z(#ZS&r6HVxkXCJVD(=4o)E1ohZghcS>3yp0(#=z1bSiZEw~Ulq#} z3zFxOsVD2)Uqx=wSlT=Bsx}qqxn`Y*S znD~Zi{0V3^OT#e--+$2iMf+@n;H3bZFxY%)OSg5jSks_r&9T#MYd|oOKu!@#UV>?b z>FURYiYqMF_i%4@r?uU{;Iw^^v+UF=Vj<=mPVHoDwm$AUXdgStGkuiC5C>%W7U&*U ztYZ*Io0n&cOsmKh2AZHW)NH9iL=R-(vu)R_*tjBw91G)k;}-zEJZ0>j*Jq;DfVz^IMfJuvT!ACL_#^<33Yc9awE zKio0!f|WzFF#87y+(89n8y06Dds>2vtAxqAyv&=wLo*(K-GH}i zJu+NeKC|j6p@*5{q)q(b$u3`beuZln<|ApZzQ1GKh+Z??#lJXssLlj7r2(AzD>==d zpIC*U@{jCaa!qotY0uiMV4{e&6f${P&1KA_{G*8N+p3XKD#6ABbKVI%-kKYws{IO0 zQc}m)!}2~&K_)ap*{qh3UC;0@FzOz~JzMU4iR|uOTV1B+1jw4EKC-mxiO86AgD4b0 zyib<@!OqP#L9#D5H!@q0*yxr<(5s_+mNvw%B1iyq025!nz9xc_)WCwA5bAD<+?9H% zk2``NIXh%H8Xj&C@dN*@GD8_<9sLd=o(o-EarE^=0PpOC%PJA=w7TA@!KRRgTxgom zhvf#{ub|qj;M``2hZ?v5&4RQ2Q6%2b>ApIji9Q}}D9KZk_zD5Ar^f@Q7DlhMefIj1 zr^d=yd-rBL-n?ZGFb7WRrN)~}xlaZMkS9B)9Hwps8)My#Q>YF{u+m1+M9IiK3LzR5 z&$75tBbkBCS(2n}^vMEcC}HMZsvxLTB{-S@0~8rW(JFHQK<)fX#y=!4Gj?!nNwIXG znAvRiiky|iP+rQU4dz`+bKvBsn1sH>4bA8>wrwq+nRRW8p=x~?Bpt2tR70Y%BEU^b znU@E0OaO0WgS@TE;DuE95lqaAnvHnI9cw1@kt$qeaXZ&)=f(C65&;|2=VR21nc@g= zB1b9%@dAJM`EeU$8(n}RPFN=s?oOQ`gk9c!l`aC-mYs}Tl%>f%ulY3mhOS!RoNKE9 zZpCNwqH{0Lu9{^8L3Wnk@c_+~ELzg)nlC5AK?CzPqssF=eWQwvjt_gXzN&dC&-H}O z(wM1bkRa5eg*}m4)=8~++v(0HbLh9Q`_JgGfjuVo6piv1LEUT*pUr4Qb{4gXaIlZB zY64wxP45Bva{PYQc3JXqK7=(U$z?eh0^db>@oz1NLgB>Zv(@GGs%}Eagq%S0XI&g> znM?LoTVYjX+DrSQVWoglal1iiT}Y>O-rU5Mk}NQuyp55oD2xY3c5JXluQ+pz#AH*& zcMZ5+2XEx7Y5LcTPx_p?|1op}`r_}}Kkd2@PC3J0ByRWvJ8V$AMO_uQ%;%N;pK@ZF zT@It{*vkKt!DnCGZAqB=0D`F!?aEYOIFn9EM>mK#N#G~m323HaZY8R-7r>-uDd$qH ztWl(!An)r+ao!>D{m8!r1}I+{K7EsqH5`lMC-j8S8*tJXFfpU8airZS+1r*W!cdcR zVdK*su{O>Tkq*{=!4)_#GUNv?fusasvP8M{I7ytRWw8+oSzRq$W&}oyeR7e`Bs6CP zYqX2lf5YRrr&gXb>(p$9g!f@UP4Bp~!nE|0qVt%wgABq_fBIoj@9^x7nwe~-zu2!b zDHzE98P(q~_V|yJleUYuo5T%>XSgEbP#X|}Y4C-RDRPH#HUb$D4hHSxhNH|Kco}5iMv8{z9 z+Iw-DJ*JdA{kb9&09R|R5uoxHGnsOYbj9b&L`_=8-MX{ z4hs{HQC4mMMo!$?fR$$srdWY_3DwQ@J`Lfp`Bl4E5-_(g`VF?hW4l_Ud^s3biB zSbKLVl6)inzMzE*8jGeYwW?YEAna%46E)-UAFM|DWp!j^U?62M_RNYe&#haLr3ERs z|CVQ1)-KXMvu)`{=<*CBCNER#wN6Pw4=$x6t5vx!ia$}6onJf zQ5Ev`*gP#*ZsGIP<9L2ZqH1NuQU{YjP z%FtF@d#h}?jU%-)s}XQinaKCGiPV`<9M9Gx&mo%(|5DD9D+Go8+8H?{Tp*bXz#Jhn zm5LeH{Bve+Mx`)}zkQNBAfOic65j}MFHf%6%2vq&pFn?}6&7S#@G*6yw4k~KIlv?o z^_*hZKGV3_Q#V7 zB(Xwd=19ZNMMGOURE&;67`GQy#By=|rEt3MXm1$!N_v!6|15GvQr2R)`+N9*4!!Wg z0aY9oXkP!R7Z-pwh-2K=U_Hcw-ftv+iPfSAV!Y>plBGMk%yi;4B*?S_0;bApb5|%X z{T`)?Zan2;wbBr9m}y!+s!?aha8i%aRynjvHM(g6Lo=x{HB}9xeRB-VP%WjfPb1S5 zitAGQ#|HgPl1i#Yyb>y8W`fsn$I)5Xf67XFHEl4O@KUOoNES!mFiIwh_K^Se%jwM^ z6>9`OK3O_+`dwT^*861jD~Z}_B)#iMga7HT2&KeZzZ$$yXd>J>h-3#zc-MS+SW0;P zC5m(7dzDHGmoq(_e-?53L+l>8W`?c1%>!3mDJ{k*FNCBxZAp-1dCAk?F-R{xzDv%A>U>peLQhK}Hy!*qc#J%y6qPcH zH}JlIxBS$xCLLN|n7jX4VS=$*MdrnQ%V78snl?DWc{( z7B%=5M_H)ezRw#7q%bqnfI`5h*1(59_oA%gqSNH7Qu`|zx9l!}dPP@^@xs|DUx!_+ zWpp>Ko;%Z&#?DMS{9(~j1Z9(Qz;&36ucM0%tZNiIL`oRdvtD#P?DYrE$1n?#%>T4`5B z8% z+_ur0ek3wz2rnzKzdh)De!8W|H18#g>>zaxs5kJe-&{CpUt2`a$CpGq#9~lE3SIb3 zJ?#RlHJ1PGZv8&P$jY>aJ7G3L(_l8Qk>>dE7)->ZdyRV}k4kf%>xTgPQFp=Gv+!u2 zU=BC>C1;mCFJK;` zM+d4wiX=|Z86f=hW?hY_z@WIP6USZm*P2{=^#2>iNIA3tymwBxBN{Z|%Ys02$u!3b zarFrsKknX;ogD27%Vu>a^9K5qk}q#sIOJx^Dtok;tHszPDaazGw+l~q;bRb)@eBNk zwdw@z)@Azlk7ka?y=Yf#KQX?bC{F>`6xz z`P4LJa@{zgR)!*r_iVs$%!+LJ)l`M-v!l z_1JHQi6tD|rs=_jvPsNT9L$H4Gl!oS@ZMdaFtV)_qp6M;R0+|LoNL{a80TKRN4>Gd z=pBX?b(2DTEz~1)Mdav5bCf^fkg_o`O#SQ+axj_|pCAI1fBe^!aY3y`G_QqYh zTv+CJ_@I!q>I8YPg^X@)0Cmm`>P|H{l^Uu`VRonH+=W0^Giy>Q%b4CEa!GiHmMcV# z@-+k=Shb!WQ9n4-&caYhajM=jJ_2T!m7?|gI6g>t8cUb0pgqn~-EAK8 zr&ec*=p4$biwNftnDW?LxsPRihJXia+(Rs=2eA0~3EUAcD5m2zG8cIt3}SgyvT1k4 zQ-(xUpc0YVBY!L8^Ap1J*O6T#_9ll2q5lh%g|CSQ)K_QqPV2R1WU&UVNu;j_N#P%o zbSa_3`R2>4U>JuJ#%5qFJj;dZE+gMH94|zGc+Zj8+-LBHwc4Xm+RX*?YmM~1t!<`a^j?zig3L_By zI#T#h(XwG>sh1h{;N8i4u@krLRI@3QEFK` z}w&1{D~)B_E59W z>*J0?Dj2x|;i$IpPK4ybtIE6>zjAtI5_<=O$Dq_DkNwR}Ot$PHkf0rptB_&@?Rtx$ zNQOJ;(fMZL)eNPXyyHvm(CfVE3{&p{v=pwB4O?!Mi&bqO3fL(rv;fF}=y^@QiQ&?F zJQlg$m1Sy8e3jBq$FE$yY3fIwYa6YyLm$*oncZxH+>W`?&)>Qo z%)@hTHJ30srMKY5r%%O>X_lMW-|x*=;I=F*t4N6CwhsJ|G5CgJs+4NXzsG7S8j?IBV6ZwkzQ(bv5QQQiM$Uw+iLhiyhsy07S0J&WC}K2C6s zR8n-$-l{GDWbYgUxnKV7QJU`8aw>hMgCBE6xV{?Sw&{sypSp(eNaAw z(L)DSqN=>P6)6VS+&^R$Q*^o`M$WTDi^75alv0zzRql$ z{*roX^%k6^vpZ57Ixk8ns0E9B!j-~Wa@C3WB~3<}UBS@smladlTUoQ(_scvO1_vWD zSbcKN_~(0gs^r6`T%ja8EAKDudno~i@gxdqxHKq5A1iE_*`2t(?eQr8!U)iDQDiI* zf;Hl=&;piG2ptIENFV&sx;j-2^S$E7UR^y|_ENWH#3g`EHf$Edb~yhEl1VS-f|~ag z?098P$zXdsAQASw76ABZGRhwZwi1MN(Kf`Ml-cvAFGoDhcVZ8K zQq_>W6oPp3;1d}_`v_8nMQ#DhClsc{^#3z^-UPAEztrkT{o)RMJHgKNb!eO^)$Sjl zn!7`9s=aoKR(JdC1A;PtA;pC2=&+%ratX?439wN5A=ztJIxU6#fOG*l6scZh?qDD) z4KN0sf~CpX0!_IR7(~W+v{_;JGCLvW_<9nf#1JijmflqT@~0Z8MWq6)n1NJk8Pcd` zg?Lj2R|TEVV>*|^DF&|7ek`SKrE1y=a{cD+Z_rhv3Am%UQY1kpAm5Q z6Fp+Hwk9O=9c^6AXGw;bg_ZktB%fe3JVI_1rwtdNu1pwY>~OyG2yFU^KJAElh^E-u z8p@!_y543ORG~0Rk_I@j1k__gGW2@2~_ z>eYa?K6YLy$3~8l5Z}B=<498p+D#Gco;NisbeX2e3U5ZZ^;nzL7liQTC-+Na2hpq| zr=7fLjBeEnxMX|8r&8I)Ts`M7@i8rt(`LM0RxaCmrb@^cxpnND$TAezI$*Cf%UmIf zSkzJTVF+$Bt0@`r@FQ_}a+T&&DJ)c{x=S=(lgyg3-|2@6jHYg=Rs3jRQDB=4f)u+=dFgF$H^NpjjV}r-(H%?BT z3VD7RTdy3PJ5B0B{r0GVGyKzw`j|#)H@)8%Aimh_Y683%Wb6%)C6KV8V2wZ6FR|M3_PmL*p7Lh11oMXAT<;jB8_&k2HU5~BC zHhaXwOZSQs?9E{)RU~W-F($n)$#)Hk3LO{syv4KR>o3bVzTudUYx&nD^y~*-!|zS< zu)H7^Shp^yK>*5t!_xm=8`$la5fJ*26fbuJNG@s z%cV8oyKR9GQ8L?-t6{R{po%-vkq@yqbtG%bTIB?*2jagEoZu)UxMH+{LSdr=QaOjk z??9=Ec$i+tc#(%0cWlk@MOz3?^zk-(O6bK3uzA~7bpMtjCTMZg%({6`d-%XQfVvTW zjmB8@T##VV*xXrRPbvi5!Oa$RhNbA3gU{_jj%6TRnEjcca&f(9r3rnDcHwKYwMv$f`AJ?rHf< zbMW`W43D4ipJUHafx!Zib!6FU(tR$|l899IIz20tLk>Bj*ug7;N)Q#F4bm}zvD+)r zBTIx=DNJe)NlrhbnuSIP$s5KsS&s^&*Uy9-pa6*{q*Kr*w|j_r);wHnU&d?`fVb!j z73zdF^QDu(!0VGaj4r4z6X_|@G|dHtAK00T<}K2lg8jS1s}b5ZXo8}TVHYA+w(vOp zU0Yliv!6{NY|3K!Lhd>Y9SEf)Ff&#HKE+GWeY z=*j3n#CjZf*imvL3O7lpO#&mp%Z{2z+9ygO>fGWon86chK-fz|@M0M*U;fI_=U%w) zgC}nUH_u?_C;4~iPc1>}dfG5%n9hgh%-a75ceZz#pB2UuUVo&t@0Wr}vHmV@QSjNi zKGCb&H#5y?;Qs3;G5RB_&{rQHC$o$y6|jV8rF+I;Pe7|25s(@yr^H)OV&-T?!w0!F zk{~4-uHRpG1~zIYS49 z=BEMBp_`?mmi=Ha3eVUU8Rb#Mw|P`7n8J6MLi&TVKcQeZsLA?!xQBp30PTJ%P|ZQk zb)$klu!qO|;PtERoLmTiD$oft?ciYpGCBI0TK90)?FsuaKnGhTw{o=1W)R0ay8!u< z9`M9?-a@dxEET&31QDyCQYlF5>Wt=@G>T+2Sgpa z1{+?gRwG!C+L=TS1Z%z6<;Gz6!tTQvrrF)!l34VLSe&scjB^K>*f}$k70D-8KKurx zx5A)HMbRJv+cSP5>hj%JeNn^=a}OHb!DzRwqa4#V&k6YEQMt}GG`8M&sz({e0`NUeHkf zE+=A>ekOS$6#a<}C-@mL{o@S_fD3An7g#bJcYZEj&Eu|b$?P+(3`-VQ24>=NmCeNp zgk$JyIC;X{tNUaN>U-4iFod|#>=VHyh{h=x(xfL~=*zJZOD(m4w~*;~f@i3L?{N7P zAi6Kr&)1wL{$AR1b>yc~9wWmzihX=cAm$`~U(phRXHYXjBTWK8?!z&1jFsL6bBj2XH+?CgcWj)DIW|_xE zr^?(tr!H47uV%O&eqP>~W-}`%B#%fp#zzU1*RkUd5p;W%OHoe_@axS~Z}!d!V;Iy( zyB@ccFAqZj5s(b8q4c|$Hn%!GQY#Pws%|>1JBMz|{u=`1+anY#rfzchANBFgLHogu z^Pwax?hPVT-C!HPnJ)ycq$5`ltxZ)Q76-Mr*MD2}beO1iQ}2mS!r4MraQ1fpIwqxl zB~zCa+uZVATHkk$4V~oc2Re#n3!$-I4~5R;y`o)x_WMq83MD0Q8p=#!p!}z*GB!*pT9PTMh_xa zyeF~I3tAbyx0RkY0VjoOWIRw7t50#Xpdk7&$1LRvbzp`Quv$T?Wz&E}Af&sl|FEv& zDd)p2G9>r|Mz(A)Y*>Q9l2dypNVK98XuVG?HIs>xEEfx`u_@jlTAxX=BzOL7Gj9o| zIzl!xt|r|B#@ZS%Q04i`vhAgYW2M`*z}&?L3Bwx+=p}TuGS&1eiM#eN`p~;UU5mt> zMj@D=#9X@ntg@SLjQo&hr_URK42?8f*Yxk=odwguvcCABA-*0wIr&u^*y&2%rOwo^ z`?)RRDq3wD1$lMQyVa^844y>ion2D!o?IB?xc7fo25@*6^HQ5t*~_7~)I#=j*S$VJ zlTJAKBcsKp-4&%y&&=w+QUu!uoZR)_29U1JtS==3j}D|&VyQ>D+OvcMFQCA z7OKrb4NA#+li-k2Wn`Mff01-dZblSO7E3L?JdctC{%aC(yWFXC5P z92B{{;^?!2qJFh1FZn_R^?)1cHuD5`_}DZ2-59j;tA)#?*OyXF&n3%^1RpV=v&xuq zzvcvJkW=^dQ2WP!JE`o+qPjb;q>46t3Jtb644|Sx9vCO5FNw)pGQ?7$O0uJwyV2Vh z9!2Nc#SjR_UWa@A)VZ(d=+gPG;#v!7pCWpxa=MfhuA95|^_=UDR`?ADAa`E&1bw|u zv#C`_je0RREaY@d&GJ{==$o{0;KkF#6p%->AaZ_`;eF0Rq0!--J0E%Z6CJ>Df7T5wnrq!~Ume1WMZO#B>8?$08L zk1(Nq7-0d3Hg!7d+J@wE2Y|0O{)$TJi4VBbFCwhhBd8oqrP+{7CW(8_*7V9I5?@l& zT8VS*u#GpRl6}{Ez=u1G!BgH77v-$5V331`U&*lfad+l4Hblc5%_dPlJelqL0)-3rd7PbbML5DFA;Qp97!Eyp zv9RfxyB_f>jA%6kgGT>_4$pedpERnB)3x8_w~dW&Bh>)5rwIXVuZfyKwzVe7(;>R# z2`-S-g7wn=xXP{Q=3N)d9EFtSoM|V-%?)9&Ec~&3u)WEv_gY07{7pJ0pe|#BPHy~m&vOKN$y;Bhg-uPMr02$gmwODr+moe03AF0t+z**`` zY++GG)%C_vx_Uw(jnT0uses)ulk2-6yYXkAuv-~~W_fX|aRWVX!2iJj6=|p&<@W@S z2IIekoMYgOpm?aWa`#_hBg>yx;du?DrPV&=_*>CK>ffZ-DUZPJ`QrILAS=2{AYXn}6P`TqA|Rc` zjYCd>4!84&-RrZi@2V;$o)9v*Z$SUMy1j$QWlE|Q<#7Cc`+(eEC$dp6yXPjy`auKS~DQEs zn!O~L*g$~)L>jD|Zn0HUw7C|CYGLnu=E3v2K7huoda=*jkR6D7x5<{JK6UI_e2IEV3^f(J$8xaEm1wEv6f!d-Jd@3~N z6I+ba!zIYu7?gAZUNU_!1gsWgkpa*_QN%nL{u;*F0kO>)9?!TI%Mqb;wXXJdSw)Iq zpb%|nU!#(3Ayd!pIdV<(evV%oAH_W7xzeo~P7h3rzq<{OxAiFe zI3)vwzfM~-LyTgcl@$hhkDyg3MMVIcXo0O+U%p)W!*sKRUVc5+C*+#L6s_@@}b72a^ z%2xTHf_f^St#F1Uv72Td#he%>$u_Shkp|=?f2#X6%HWSsnPd-qzRXu5_uV*JJXNI9 z7E2plg~5}is+VFm*`2Vk5g`^TB<^?33fdfvc{cR~31Zn04}O?#A1FRaGNe*9Hgl#4`7 z+me>V25;d?jOXyAHfTbqCf+^dNvsyZS2HU$&VaB!dNLspKkcEls z+6##muuP$+z?+&qOv*{@Qy;UUrqaCzKlM1oLhNb{STgXRS!n&7LdBTrD`-w>PjrBD zsMDmN5;9Q+gGbXCz8zFkZ&rlz?{-X;a{WK>+$>-w=>3Nz8YQGsU(l;w6c zC8Is_3YzgIf63Th0P2e1?6`Yj@@^I01Ry{VKE+KO!6V&Z#B+b+j~tLqGSWLNdeCm^ zq!mteAtyEwm{1%(4Z~m^$x!Z>G7=iaXWqXkVNyB4e7n0~A9#avH-Q^^8 zW)HyRUlt$F7=+#w)Rma=0aJa>%JqP^y0e+wpYMm!{j(MmE92F5S75Y`t`e>(uf9AO z?9-Hxa-khE+pX!ukN>zB;Dpwlbb(gQ2jM3|yW!t3GjOc`Sb(aRG?pg`gsu(>0OC}G z|AauFs`_4-x?|#E>bj1W-AfL_uU*xOQ}*+5{paz=c{T>I$16zzglCT_8fxL|;gy&) zbOGF$H*e~bTFUFWx^}|Nr&HUp@FMAG)%;gEi{YV~XIK}dqH^jR9RwCxz24$x258oz zv4__a)A&a_KF8J1*#?7~2)i8+u|URHaqf|(A{&qRyF&_HXLYe6UPR1sp4r_tk!!I1 z`fQ^g5$<2C5&7)Pg>Mc!`xs1*hj4|-wS6GaS~mu9v(wxml!V&x@2QpK+&UCO43R0l zSL8+8Y}n!B zr(1C2DDW%b_487bk+lZ+0{UxS+}`4J3lx&