Switch segment storage from single vector to vector of arrays

Previously, PV would run out of storage space for the data
segments because data was stored in a vector. As a vector allows
contiguous access to the underlying data (much like an array),
it needs a contiguous section of memory. With incoming data and
constant resizing of the vector, the OS at some point can no
longer supply such a section of memory, causing PV to abort
acquisition.

This change fixes this by using several chunks that are never
grown in size. Instead, new chunks are allocated and added to
the vector as needed. This way, the OS will be able to provide
memory until it runs out of system memory.

diff --git a/pv/data/analogsegment.cpp b/pv/data/analogsegment.cpp
index b1aa56434be2c38b4b8166cf977396c37110333f..d40c40f205d5cac5440585a71834425b2c0d724b 100644 (file)
--- a/pv/data/analogsegment.cpp
+++ b/pv/data/analogsegment.cpp
@@ -44,12 +44,9 @@ const float AnalogSegment::LogEnvelopeScaleFactor =
        logf(EnvelopeScaleFactor);
 const uint64_t AnalogSegment::EnvelopeDataUnit = 64*1024;      // bytes
 
        logf(EnvelopeScaleFactor);
 const uint64_t AnalogSegment::EnvelopeDataUnit = 64*1024;      // bytes
 

-AnalogSegment::AnalogSegment(
-       uint64_t samplerate, const uint64_t expected_num_samples) :
+AnalogSegment::AnalogSegment(uint64_t samplerate) :

        Segment(samplerate, sizeof(float))
 {
        Segment(samplerate, sizeof(float))
 {

-       set_capacity(expected_num_samples);
-

        lock_guard<recursive_mutex> lock(mutex_);
        memset(envelope_levels_, 0, sizeof(envelope_levels_));
 }
        lock_guard<recursive_mutex> lock(mutex_);
        memset(envelope_levels_, 0, sizeof(envelope_levels_));
 }


@@ -68,18 +65,11 @@ void AnalogSegment::append_interleaved_samples(const float *data,
 
        lock_guard<recursive_mutex> lock(mutex_);
 
 
        lock_guard<recursive_mutex> lock(mutex_);
 

-       // If we're out of memory, this will throw std::bad_alloc
-       data_.resize((sample_count_ + sample_count) * sizeof(float));
-
-       float *dst = (float*)data_.data() + sample_count_;
-       const float *dst_end = dst + sample_count;
-       while (dst != dst_end) {
-               *dst++ = *data;
+       for (uint32_t i=0; i < sample_count; i++) {
+               append_single_sample((void*)data);

                data += stride;
        }
 
                data += stride;
        }
 

-       sample_count_ += sample_count;
-

        // Generate the first mip-map from the data
        append_payload_to_envelope_levels();
 }
        // Generate the first mip-map from the data
        append_payload_to_envelope_levels();
 }


@@ -95,10 +85,22 @@ const float* AnalogSegment::get_samples(
 
        lock_guard<recursive_mutex> lock(mutex_);
 
 
        lock_guard<recursive_mutex> lock(mutex_);
 

-       float *const data = new float[end_sample - start_sample];
-       memcpy(data, (float*)data_.data() + start_sample, sizeof(float) *
-               (end_sample - start_sample));
-       return data;
+       return (float*)get_raw_samples(start_sample, (end_sample - start_sample));
+}
+
+SegmentAnalogDataIterator* AnalogSegment::begin_sample_iteration(uint64_t start) const
+{
+       return (SegmentAnalogDataIterator*)begin_raw_sample_iteration(start);
+}
+
+void AnalogSegment::continue_sample_iteration(SegmentAnalogDataIterator* it, uint64_t increase) const
+{
+       Segment::continue_raw_sample_iteration((SegmentRawDataIterator*)it, increase);
+}
+
+void AnalogSegment::end_sample_iteration(SegmentAnalogDataIterator* it) const
+{
+       Segment::end_raw_sample_iteration((SegmentRawDataIterator*)it);

 }
 
 void AnalogSegment::get_envelope_section(EnvelopeSection &s,
 }
 
 void AnalogSegment::get_envelope_section(EnvelopeSection &s,


@@ -141,6 +143,7 @@ void AnalogSegment::append_payload_to_envelope_levels()
        Envelope &e0 = envelope_levels_[0];
        uint64_t prev_length;
        EnvelopeSample *dest_ptr;
        Envelope &e0 = envelope_levels_[0];
        uint64_t prev_length;
        EnvelopeSample *dest_ptr;

+       SegmentRawDataIterator* it;

 
        // Expand the data buffer to fit the new samples
        prev_length = e0.length;
 
        // Expand the data buffer to fit the new samples
        prev_length = e0.length;


@@ -155,18 +158,22 @@ void AnalogSegment::append_payload_to_envelope_levels()
        dest_ptr = e0.samples + prev_length;
 
        // Iterate through the samples to populate the first level mipmap
        dest_ptr = e0.samples + prev_length;
 
        // Iterate through the samples to populate the first level mipmap

-       const float *const end_src_ptr = (float*)data_.data() +
-               e0.length * EnvelopeScaleFactor;
-       for (const float *src_ptr = (float*)data_.data() +
-                       prev_length * EnvelopeScaleFactor;
-                       src_ptr < end_src_ptr; src_ptr += EnvelopeScaleFactor) {
+       uint64_t start_sample = prev_length * EnvelopeScaleFactor;
+       uint64_t end_sample   = e0.length * EnvelopeScaleFactor;
+
+       it = begin_raw_sample_iteration(start_sample);
+       for (uint64_t i = start_sample; i < end_sample; i += EnvelopeScaleFactor) {
+               const float* samples = (float*)it->value;
+

                const EnvelopeSample sub_sample = {
                const EnvelopeSample sub_sample = {

-                       *min_element(src_ptr, src_ptr + EnvelopeScaleFactor),
-                       *max_element(src_ptr, src_ptr + EnvelopeScaleFactor),
+                       *min_element(samples, samples + EnvelopeScaleFactor),
+                       *max_element(samples, samples + EnvelopeScaleFactor),

                };
 
                };
 

+               continue_raw_sample_iteration(it, EnvelopeScaleFactor);

                *dest_ptr++ = sub_sample;
        }
                *dest_ptr++ = sub_sample;
        }

+       end_raw_sample_iteration(it);

 
        // Compute higher level mipmaps
        for (unsigned int level = 1; level < ScaleStepCount; level++) {
 
        // Compute higher level mipmaps
        for (unsigned int level = 1; level < ScaleStepCount; level++) {


@@ -177,16 +184,17 @@ void AnalogSegment::append_payload_to_envelope_levels()
                prev_length = e.length;
                e.length = el.length / EnvelopeScaleFactor;
 
                prev_length = e.length;
                e.length = el.length / EnvelopeScaleFactor;
 

-               // Break off if there are no more samples to computed
+               // Break off if there are no more samples to be computed

                if (e.length == prev_length)
                        break;
 
                reallocate_envelope(e);
 
                if (e.length == prev_length)
                        break;
 
                reallocate_envelope(e);
 

-               // Subsample the level lower level
+               // Subsample the lower level

                const EnvelopeSample *src_ptr =
                        el.samples + prev_length * EnvelopeScaleFactor;
                const EnvelopeSample *const end_dest_ptr = e.samples + e.length;
                const EnvelopeSample *src_ptr =
                        el.samples + prev_length * EnvelopeScaleFactor;
                const EnvelopeSample *const end_dest_ptr = e.samples + e.length;

+

                for (dest_ptr = e.samples + prev_length;
                                dest_ptr < end_dest_ptr; dest_ptr++) {
                        const EnvelopeSample *const end_src_ptr =
                for (dest_ptr = e.samples + prev_length;
                                dest_ptr < end_dest_ptr; dest_ptr++) {
                        const EnvelopeSample *const end_src_ptr =

diff --git a/pv/data/analogsegment.hpp b/pv/data/analogsegment.hpp
index 520f7168247222015896ae3deea7256cdfa115f6..52d37a76c3e67fdfba671ae46030e390940e1355 100644 (file)
--- a/pv/data/analogsegment.hpp
+++ b/pv/data/analogsegment.hpp
@@ -32,6 +32,12 @@ struct Basic;
 namespace pv {
 namespace data {
 
 namespace pv {
 namespace data {
 

+typedef struct {
+       uint64_t sample_index, chunk_num, chunk_offs;
+       uint8_t* chunk;
+       float* value;
+} SegmentAnalogDataIterator;
+

 class AnalogSegment : public Segment
 {
 public:
 class AnalogSegment : public Segment
 {
 public:


@@ -65,7 +71,7 @@ private:
        static const uint64_t EnvelopeDataUnit;
 
 public:
        static const uint64_t EnvelopeDataUnit;
 
 public:

-       AnalogSegment(uint64_t samplerate, uint64_t expected_num_samples = 0);
+       AnalogSegment(uint64_t samplerate);

 
        virtual ~AnalogSegment();
 
 
        virtual ~AnalogSegment();
 


@@ -75,6 +81,10 @@ public:
        const float* get_samples(int64_t start_sample,
                int64_t end_sample) const;
 
        const float* get_samples(int64_t start_sample,
                int64_t end_sample) const;
 

+       SegmentAnalogDataIterator* begin_sample_iteration(uint64_t start) const;
+       void continue_sample_iteration(SegmentAnalogDataIterator* it, uint64_t increase) const;
+       void end_sample_iteration(SegmentAnalogDataIterator* it) const;
+

        void get_envelope_section(EnvelopeSection &s,
                uint64_t start, uint64_t end, float min_length) const;
 
        void get_envelope_section(EnvelopeSection &s,
                uint64_t start, uint64_t end, float min_length) const;
 

diff --git a/pv/data/logicsegment.cpp b/pv/data/logicsegment.cpp
index dab6b91dee295194f343115c7f2cdeecec5657e6..a62d9e2f4dae7bca74e79d5f6ca04a3a6a7f87b0 100644 (file)
--- a/pv/data/logicsegment.cpp
+++ b/pv/data/logicsegment.cpp
@@ -45,13 +45,10 @@ const int LogicSegment::MipMapScaleFactor = 1 << MipMapScalePower;
 const float LogicSegment::LogMipMapScaleFactor = logf(MipMapScaleFactor);
 const uint64_t LogicSegment::MipMapDataUnit = 64*1024; // bytes
 
 const float LogicSegment::LogMipMapScaleFactor = logf(MipMapScaleFactor);
 const uint64_t LogicSegment::MipMapDataUnit = 64*1024; // bytes
 

-LogicSegment::LogicSegment(shared_ptr<Logic> logic, uint64_t samplerate,
-                               const uint64_t expected_num_samples) :
+LogicSegment::LogicSegment(shared_ptr<Logic> logic, uint64_t samplerate) :

        Segment(samplerate, logic->unit_size()),
        last_append_sample_(0)
 {
        Segment(samplerate, logic->unit_size()),
        last_append_sample_(0)
 {

-       set_capacity(expected_num_samples);
-

        lock_guard<recursive_mutex> lock(mutex_);
        memset(mip_map_, 0, sizeof(mip_map_));
        append_payload(logic);
        lock_guard<recursive_mutex> lock(mutex_);
        memset(mip_map_, 0, sizeof(mip_map_));
        append_payload(logic);


@@ -145,7 +142,7 @@ void LogicSegment::append_payload(shared_ptr<Logic> logic)
 
        lock_guard<recursive_mutex> lock(mutex_);
 
 
        lock_guard<recursive_mutex> lock(mutex_);
 

-       append_data(logic->data_pointer(),
+       append_samples(logic->data_pointer(),

                logic->data_length() / unit_size_);
 
        // Generate the first mip-map from the data
                logic->data_length() / unit_size_);
 
        // Generate the first mip-map from the data


@@ -163,16 +160,31 @@ const uint8_t* LogicSegment::get_samples(int64_t start_sample,
 
        lock_guard<recursive_mutex> lock(mutex_);
 
 
        lock_guard<recursive_mutex> lock(mutex_);
 

-       uint8_t* data = new uint8_t[end_sample - start_sample];
-       const size_t size = (end_sample - start_sample) * unit_size_;
-       memcpy(data, (uint8_t*)data_.data() + start_sample * unit_size_, size);
-       return data;
+       return get_raw_samples(start_sample, (end_sample-start_sample));
+}
+
+SegmentLogicDataIterator* LogicSegment::begin_sample_iteration(uint64_t start) const
+{
+       return (SegmentLogicDataIterator*)begin_raw_sample_iteration(start);
+}
+
+void LogicSegment::continue_sample_iteration(SegmentLogicDataIterator* it, uint64_t increase) const
+{
+       Segment::continue_raw_sample_iteration((SegmentRawDataIterator*)it, increase);
+}
+
+void LogicSegment::end_sample_iteration(SegmentLogicDataIterator* it) const
+{
+       Segment::end_raw_sample_iteration((SegmentRawDataIterator*)it);

 }
 
 void LogicSegment::reallocate_mipmap_level(MipMapLevel &m)
 {
 }
 
 void LogicSegment::reallocate_mipmap_level(MipMapLevel &m)
 {

+       lock_guard<recursive_mutex> lock(mutex_);
+

        const uint64_t new_data_length = ((m.length + MipMapDataUnit - 1) /
                MipMapDataUnit) * MipMapDataUnit;
        const uint64_t new_data_length = ((m.length + MipMapDataUnit - 1) /
                MipMapDataUnit) * MipMapDataUnit;

+

        if (new_data_length > m.data_length) {
                m.data_length = new_data_length;
 
        if (new_data_length > m.data_length) {
                m.data_length = new_data_length;
 


@@ -186,8 +198,8 @@ void LogicSegment::append_payload_to_mipmap()
 {
        MipMapLevel &m0 = mip_map_[0];
        uint64_t prev_length;
 {
        MipMapLevel &m0 = mip_map_[0];
        uint64_t prev_length;

-       const uint8_t *src_ptr;

        uint8_t *dest_ptr;
        uint8_t *dest_ptr;

+       SegmentRawDataIterator* it;

        uint64_t accumulator;
        unsigned int diff_counter;
 
        uint64_t accumulator;
        unsigned int diff_counter;
 


@@ -204,24 +216,26 @@ void LogicSegment::append_payload_to_mipmap()
        dest_ptr = (uint8_t*)m0.data + prev_length * unit_size_;
 
        // Iterate through the samples to populate the first level mipmap
        dest_ptr = (uint8_t*)m0.data + prev_length * unit_size_;
 
        // Iterate through the samples to populate the first level mipmap

-       const uint8_t *const end_src_ptr = (uint8_t*)data_.data() +
-               m0.length * unit_size_ * MipMapScaleFactor;
-       for (src_ptr = (uint8_t*)data_.data() +
-                       prev_length * unit_size_ * MipMapScaleFactor;
-                       src_ptr < end_src_ptr;) {
+       uint64_t start_sample = prev_length * MipMapScaleFactor;
+       uint64_t end_sample = m0.length * MipMapScaleFactor;
+
+       it = begin_raw_sample_iteration(start_sample);
+       for (uint64_t i = start_sample; i < end_sample;) {

                // Accumulate transitions which have occurred in this sample
                accumulator = 0;
                diff_counter = MipMapScaleFactor;
                while (diff_counter-- > 0) {
                // Accumulate transitions which have occurred in this sample
                accumulator = 0;
                diff_counter = MipMapScaleFactor;
                while (diff_counter-- > 0) {

-                       const uint64_t sample = unpack_sample(src_ptr);
+                       const uint64_t sample = unpack_sample(it->value);

                        accumulator |= last_append_sample_ ^ sample;
                        last_append_sample_ = sample;
                        accumulator |= last_append_sample_ ^ sample;
                        last_append_sample_ = sample;

-                       src_ptr += unit_size_;
+                       continue_raw_sample_iteration(it, 1);
+                       i++;

                }
 
                pack_sample(dest_ptr, accumulator);
                dest_ptr += unit_size_;
        }
                }
 
                pack_sample(dest_ptr, accumulator);
                dest_ptr += unit_size_;
        }

+       end_raw_sample_iteration(it);

 
        // Compute higher level mipmaps
        for (unsigned int level = 1; level < ScaleStepCount; level++) {
 
        // Compute higher level mipmaps
        for (unsigned int level = 1; level < ScaleStepCount; level++) {


@@ -232,17 +246,18 @@ void LogicSegment::append_payload_to_mipmap()
                prev_length = m.length;
                m.length = ml.length / MipMapScaleFactor;
 
                prev_length = m.length;
                m.length = ml.length / MipMapScaleFactor;
 

-               // Break off if there are no more samples to computed
+               // Break off if there are no more samples to be computed

                if (m.length == prev_length)
                        break;
 
                reallocate_mipmap_level(m);
 
                if (m.length == prev_length)
                        break;
 
                reallocate_mipmap_level(m);
 

-               // Subsample the level lower level
-               src_ptr = (uint8_t*)ml.data +
+               // Subsample the lower level
+               const uint8_t* src_ptr = (uint8_t*)ml.data +

                        unit_size_ * prev_length * MipMapScaleFactor;
                const uint8_t *const end_dest_ptr =
                        (uint8_t*)m.data + unit_size_ * m.length;
                        unit_size_ * prev_length * MipMapScaleFactor;
                const uint8_t *const end_dest_ptr =
                        (uint8_t*)m.data + unit_size_ * m.length;

+

                for (dest_ptr = (uint8_t*)m.data +
                                unit_size_ * prev_length;
                                dest_ptr < end_dest_ptr;
                for (dest_ptr = (uint8_t*)m.data +
                                unit_size_ * prev_length;
                                dest_ptr < end_dest_ptr;


@@ -259,11 +274,15 @@ void LogicSegment::append_payload_to_mipmap()
        }
 }
 
        }
 }
 

-uint64_t LogicSegment::get_sample(uint64_t index) const
+uint64_t LogicSegment::get_unpacked_sample(uint64_t index) const

 {
        assert(index < sample_count_);
 
 {
        assert(index < sample_count_);
 

-       return unpack_sample((uint8_t*)data_.data() + index * unit_size_);
+       const uint8_t* data = get_raw_samples(index, 1);
+       uint64_t sample = unpack_sample(data);
+       delete[] data;
+
+       return sample;

 }
 
 void LogicSegment::get_subsampled_edges(
 }
 
 void LogicSegment::get_subsampled_edges(


@@ -290,7 +309,7 @@ void LogicSegment::get_subsampled_edges(
        const uint64_t sig_mask = 1ULL << sig_index;
 
        // Store the initial state
        const uint64_t sig_mask = 1ULL << sig_index;
 
        // Store the initial state

-       last_sample = (get_sample(start) & sig_mask) != 0;
+       last_sample = (get_unpacked_sample(start) & sig_mask) != 0;

        edges.push_back(pair<int64_t, bool>(index++, last_sample));
 
        while (index + block_length <= end) {
        edges.push_back(pair<int64_t, bool>(index++, last_sample));
 
        while (index + block_length <= end) {


@@ -311,7 +330,7 @@ void LogicSegment::get_subsampled_edges(
                                        (index & ~((uint64_t)(~0) << MipMapScalePower)) != 0;
                                        index++) {
                                const bool sample =
                                        (index & ~((uint64_t)(~0) << MipMapScalePower)) != 0;
                                        index++) {
                                const bool sample =

-                                       (get_sample(index) & sig_mask) != 0;
+                                       (get_unpacked_sample(index) & sig_mask) != 0;

 
                                // If there was a change we cannot fast forward
                                if (sample != last_sample) {
 
                                // If there was a change we cannot fast forward
                                if (sample != last_sample) {


@@ -331,7 +350,7 @@ void LogicSegment::get_subsampled_edges(
 
                        // We can fast forward only if there was no change
                        const bool sample =
 
                        // We can fast forward only if there was no change
                        const bool sample =

-                               (get_sample(index) & sig_mask) != 0;
+                               (get_unpacked_sample(index) & sig_mask) != 0;

                        if (last_sample != sample)
                                fast_forward = false;
                }
                        if (last_sample != sample)
                                fast_forward = false;
                }


@@ -409,7 +428,7 @@ void LogicSegment::get_subsampled_edges(
                        // block
                        if (min_length < MipMapScaleFactor) {
                                for (; index < end; index++) {
                        // block
                        if (min_length < MipMapScaleFactor) {
                                for (; index < end; index++) {

-                                       const bool sample = (get_sample(index) &
+                                       const bool sample = (get_unpacked_sample(index) &

                                                sig_mask) != 0;
                                        if (sample != last_sample)
                                                break;
                                                sig_mask) != 0;
                                        if (sample != last_sample)
                                                break;


@@ -426,7 +445,7 @@ void LogicSegment::get_subsampled_edges(
 
                // Store the final state
                const bool final_sample =
 
                // Store the final state
                const bool final_sample =

-                       (get_sample(final_index - 1) & sig_mask) != 0;
+                       (get_unpacked_sample(final_index - 1) & sig_mask) != 0;

                edges.push_back(pair<int64_t, bool>(index, final_sample));
 
                index = final_index;
                edges.push_back(pair<int64_t, bool>(index, final_sample));
 
                index = final_index;


@@ -434,7 +453,7 @@ void LogicSegment::get_subsampled_edges(
        }
 
        // Add the final state
        }
 
        // Add the final state

-       const bool end_sample = get_sample(end) & sig_mask;
+       const bool end_sample = get_unpacked_sample(end) & sig_mask;

        if (last_sample != end_sample)
                edges.push_back(pair<int64_t, bool>(end, end_sample));
        edges.push_back(pair<int64_t, bool>(end + 1, end_sample));
        if (last_sample != end_sample)
                edges.push_back(pair<int64_t, bool>(end, end_sample));
        edges.push_back(pair<int64_t, bool>(end + 1, end_sample));

diff --git a/pv/data/logicsegment.hpp b/pv/data/logicsegment.hpp
index 0931e2c46c9f6ed86a9c1d8f8706c8477cc56a26..43fd41126617bd0b9d76dbc0f8bbf7875c5569bd 100644 (file)
--- a/pv/data/logicsegment.hpp
+++ b/pv/data/logicsegment.hpp
@@ -40,6 +40,12 @@ struct LongPulses;
 namespace pv {
 namespace data {
 
 namespace pv {
 namespace data {
 

+typedef struct {
+       uint64_t sample_index, chunk_num, chunk_offs;
+       uint8_t* chunk;
+       uint8_t* value;
+} SegmentLogicDataIterator;
+

 class LogicSegment : public Segment
 {
 private:
 class LogicSegment : public Segment
 {
 private:


@@ -61,8 +67,7 @@ public:
        typedef std::pair<int64_t, bool> EdgePair;
 
 public:
        typedef std::pair<int64_t, bool> EdgePair;
 
 public:

-       LogicSegment(std::shared_ptr<sigrok::Logic> logic,
-               uint64_t samplerate, uint64_t expected_num_samples = 0);
+       LogicSegment(std::shared_ptr<sigrok::Logic> logic, uint64_t samplerate);

 
        virtual ~LogicSegment();
 
 
        virtual ~LogicSegment();
 


@@ -70,6 +75,10 @@ public:
 
        const uint8_t* get_samples(int64_t start_sample, int64_t end_sample) const;
 
 
        const uint8_t* get_samples(int64_t start_sample, int64_t end_sample) const;
 

+       SegmentLogicDataIterator* begin_sample_iteration(uint64_t start) const;
+       void continue_sample_iteration(SegmentLogicDataIterator* it, uint64_t increase) const;
+       void end_sample_iteration(SegmentLogicDataIterator* it) const;
+

 private:
        uint64_t unpack_sample(const uint8_t *ptr) const;
        void pack_sample(uint8_t *ptr, uint64_t value);
 private:
        uint64_t unpack_sample(const uint8_t *ptr) const;
        void pack_sample(uint8_t *ptr, uint64_t value);


@@ -78,7 +87,7 @@ private:
 
        void append_payload_to_mipmap();
 
 
        void append_payload_to_mipmap();
 

-       uint64_t get_sample(uint64_t index) const;
+       uint64_t get_unpacked_sample(uint64_t index) const;

 
 public:
        /**
 
 public:
        /**

diff --git a/pv/data/segment.cpp b/pv/data/segment.cpp
index 9afc8ea2d5b49a9f65ea31fb76f10c97069ac002..f635fc387a59dc4156273ca0dd4d88602ca35a27 100644 (file)
--- a/pv/data/segment.cpp
+++ b/pv/data/segment.cpp
@@ -1,6 +1,7 @@
 /*
  * This file is part of the PulseView project.
  *
 /*
  * This file is part of the PulseView project.
  *

+ * Copyright (C) 2017 Soeren Apel <soeren@apelpie.net>

  * Copyright (C) 2012 Joel Holdsworth <joel@airwebreathe.org.uk>
  *
  * This program is free software; you can redistribute it and/or modify
  * Copyright (C) 2012 Joel Holdsworth <joel@airwebreathe.org.uk>
  *
  * This program is free software; you can redistribute it and/or modify


@@ -23,8 +24,11 @@
 #include <stdlib.h>
 #include <string.h>
 
 #include <stdlib.h>
 #include <string.h>
 

+#include <vector>
+

 using std::lock_guard;
 using std::recursive_mutex;
 using std::lock_guard;
 using std::recursive_mutex;

+using std::vector;

 
 namespace pv {
 namespace data {
 
 namespace pv {
 namespace data {


@@ -33,16 +37,29 @@ Segment::Segment(uint64_t samplerate, unsigned int unit_size) :
        sample_count_(0),
        start_time_(0),
        samplerate_(samplerate),
        sample_count_(0),
        start_time_(0),
        samplerate_(samplerate),

-       capacity_(0),

        unit_size_(unit_size)
 {
        lock_guard<recursive_mutex> lock(mutex_);
        assert(unit_size_ > 0);
        unit_size_(unit_size)
 {
        lock_guard<recursive_mutex> lock(mutex_);
        assert(unit_size_ > 0);

+
+       // Determine the number of samples we can fit in one chunk
+       // without exceeding MaxChunkSize
+       chunk_size_ = std::min(MaxChunkSize,
+               (MaxChunkSize / unit_size_) * unit_size_);
+
+       // Create the initial chunk
+       current_chunk_ = new uint8_t[chunk_size_];
+       data_chunks_.push_back(current_chunk_);
+       used_samples_ = 0;
+       unused_samples_ = chunk_size_ / unit_size_;

 }
 
 Segment::~Segment()
 {
        lock_guard<recursive_mutex> lock(mutex_);
 }
 
 Segment::~Segment()
 {
        lock_guard<recursive_mutex> lock(mutex_);

+
+       for (uint8_t* chunk : data_chunks_)
+               delete[] chunk;

 }
 
 uint64_t Segment::get_sample_count() const
 }
 
 uint64_t Segment::get_sample_count() const


@@ -71,39 +88,139 @@ unsigned int Segment::unit_size() const
        return unit_size_;
 }
 
        return unit_size_;
 }
 

-void Segment::set_capacity(const uint64_t new_capacity)
+void Segment::append_single_sample(void *data)
+{
+       lock_guard<recursive_mutex> lock(mutex_);
+
+       // There will always be space for at least one sample in
+       // the current chunk, so we do not need to test for space
+
+       memcpy(current_chunk_ + (used_samples_ * unit_size_),
+               data, unit_size_);
+       used_samples_++;
+       unused_samples_--;
+
+       if (unused_samples_ == 0) {
+               current_chunk_ = new uint8_t[chunk_size_];
+               data_chunks_.push_back(current_chunk_);
+               used_samples_ = 0;
+               unused_samples_ = chunk_size_ / unit_size_;
+       }
+
+       sample_count_++;
+}
+
+void Segment::append_samples(void* data, uint64_t samples)

 {
        lock_guard<recursive_mutex> lock(mutex_);
 
 {
        lock_guard<recursive_mutex> lock(mutex_);
 

-       assert(capacity_ >= sample_count_);
-       if (new_capacity > capacity_) {
+       if (unused_samples_ >= samples) {
+               // All samples fit into the current chunk
+               memcpy(current_chunk_ + (used_samples_ * unit_size_),
+                       data, (samples * unit_size_));
+               used_samples_ += samples;
+               unused_samples_ -= samples;
+       } else {
+               // Only a part of the samples fit, split data up between chunks
+               memcpy(current_chunk_ + (used_samples_ * unit_size_),
+                       data, (unused_samples_ * unit_size_));
+               const uint64_t remaining_samples = samples - unused_samples_;
+

                // If we're out of memory, this will throw std::bad_alloc
                // If we're out of memory, this will throw std::bad_alloc

-               data_.resize((new_capacity * unit_size_) + sizeof(uint64_t));
-               capacity_ = new_capacity;
+               current_chunk_ = new uint8_t[chunk_size_];
+               data_chunks_.push_back(current_chunk_);
+               memcpy(current_chunk_, (uint8_t*)data + (unused_samples_ * unit_size_),
+                       (remaining_samples * unit_size_));
+
+               used_samples_ = remaining_samples;
+               unused_samples_ = (chunk_size_ / unit_size_) - remaining_samples;

        }
        }

+
+       if (unused_samples_ == 0) {
+               // If we're out of memory, this will throw std::bad_alloc
+               current_chunk_ = new uint8_t[chunk_size_];
+               data_chunks_.push_back(current_chunk_);
+               used_samples_ = 0;
+               unused_samples_ = chunk_size_ / unit_size_;
+       }
+
+       sample_count_ += samples;

 }
 
 }
 

-uint64_t Segment::capacity() const
+uint8_t* Segment::get_raw_samples(uint64_t start, uint64_t count) const

 {
 {

+       assert(start < sample_count_);
+       assert(start + count <= sample_count_);
+       assert(count > 0);
+

        lock_guard<recursive_mutex> lock(mutex_);
        lock_guard<recursive_mutex> lock(mutex_);

-       return data_.size();
+
+       uint8_t* dest = new uint8_t[count * unit_size_];
+       uint8_t* dest_ptr = dest;
+
+       uint64_t chunk_num = (start * unit_size_) / chunk_size_;
+       uint64_t chunk_offs = (start * unit_size_) % chunk_size_;
+
+       while (count > 0) {
+               const uint8_t* chunk = data_chunks_[chunk_num];
+
+               uint64_t copy_size = std::min(count * unit_size_,
+                       chunk_size_ - chunk_offs);
+
+               memcpy(dest_ptr, chunk + chunk_offs, copy_size);
+
+               dest_ptr += copy_size;
+               count -= (copy_size / unit_size_);
+
+               chunk_num++;
+               chunk_offs = 0;
+       }
+
+       return dest;

 }
 
 }
 

-void Segment::append_data(void *data, uint64_t samples)
+SegmentRawDataIterator* Segment::begin_raw_sample_iteration(uint64_t start) const
+{
+       SegmentRawDataIterator* it = new SegmentRawDataIterator;
+
+       assert(start < sample_count_);
+
+       it->sample_index = start;
+       it->chunk_num = (start * unit_size_) / chunk_size_;
+       it->chunk_offs = (start * unit_size_) % chunk_size_;
+       it->chunk = data_chunks_[it->chunk_num];
+       it->value = it->chunk + it->chunk_offs;
+
+       return it;
+}
+
+void Segment::continue_raw_sample_iteration(SegmentRawDataIterator* it, uint64_t increase) const

 {
        lock_guard<recursive_mutex> lock(mutex_);
 
 {
        lock_guard<recursive_mutex> lock(mutex_);
 

-       assert(capacity_ >= sample_count_);
+       if (it->sample_index > sample_count_)
+       {
+               // Fail gracefully if we are asked to deliver data we don't have
+               return;
+       } else {
+               it->sample_index += increase;
+               it->chunk_offs += (increase * unit_size_);
+       }
+
+       if (it->chunk_offs > (chunk_size_ - 1)) {
+               it->chunk_num++;
+               it->chunk_offs -= chunk_size_;
+               it->chunk = data_chunks_[it->chunk_num];
+       }

 
 

-       // Ensure there's enough capacity to copy.
-       const uint64_t free_space = capacity_ - sample_count_;
-       if (free_space < samples)
-               set_capacity(sample_count_ + samples);
+       it->value = it->chunk + it->chunk_offs;
+}

 
 

-       memcpy((uint8_t*)data_.data() + sample_count_ * unit_size_,
-               data, samples * unit_size_);
-       sample_count_ += samples;
+void Segment::end_raw_sample_iteration(SegmentRawDataIterator* it) const
+{
+       delete it;

 }
 
 }
 

+

 } // namespace data
 } // namespace pv
 } // namespace data
 } // namespace pv

diff --git a/pv/data/segment.hpp b/pv/data/segment.hpp
index db34b7687cdd098179b55a483be5ecab7c7c9650..513b7dbd9412fdc03883e7c4991083551923df60 100644 (file)
--- a/pv/data/segment.hpp
+++ b/pv/data/segment.hpp
@@ -1,6 +1,7 @@
 /*
  * This file is part of the PulseView project.
  *
 /*
  * This file is part of the PulseView project.
  *

+ * Copyright (C) 2017 Soeren Apel <soeren@apelpie.net>

  * Copyright (C) 2012 Joel Holdsworth <joel@airwebreathe.org.uk>
  *
  * This program is free software; you can redistribute it and/or modify
  * Copyright (C) 2012 Joel Holdsworth <joel@airwebreathe.org.uk>
  *
  * This program is free software; you can redistribute it and/or modify


@@ -26,11 +27,32 @@
 #include <mutex>
 #include <vector>
 
 #include <mutex>
 #include <vector>
 

+namespace SegmentTest {
+struct SmallSize8Single;
+struct MediumSize8Single;
+struct MaxSize8Single;
+struct MediumSize24Single;
+struct MediumSize32Single;
+struct MaxSize32Single;
+struct MediumSize32Multi;
+struct MaxSize32Multi;
+struct MaxSize32MultiIterated;
+}
+

 namespace pv {
 namespace data {
 
 namespace pv {
 namespace data {
 

+typedef struct {
+       uint64_t sample_index, chunk_num, chunk_offs;
+       uint8_t* chunk;
+       uint8_t* value;
+} SegmentRawDataIterator;
+

 class Segment
 {
 class Segment
 {

+private:
+       static const uint64_t MaxChunkSize = 10*1024*1024;  /* 10MiB */
+

 public:
        Segment(uint64_t samplerate, unsigned int unit_size);
 
 public:
        Segment(uint64_t samplerate, unsigned int unit_size);
 


@@ -45,44 +67,34 @@ public:
 
        unsigned int unit_size() const;
 
 
        unsigned int unit_size() const;
 

-       /**
-        * @brief Increase the capacity of the segment.
-        *
-        * Increasing the capacity allows samples to be appended without needing
-        * to reallocate memory.
-        *
-        * For the best efficiency @c set_capacity() should be called once before
-        * @c append_data() is called to set up the segment with the expected number
-        * of samples that will be appended in total.
-        *
-        * @note The capacity will automatically be increased when @c append_data()
-        * is called if there is not enough capacity in the buffer to store the samples.
-        *
-        * @param[in] new_capacity The new capacity of the segment. If this value is
-        *      smaller or equal than the current capacity then the method has no effect.
-        */
-       void set_capacity(uint64_t new_capacity);
-
-       /**
-        * @brief Get the current capacity of the segment.
-        *
-        * The capacity can be increased by calling @c set_capacity().
-        *
-        * @return The current capacity of the segment.
-        */
-       uint64_t capacity() const;
-

 protected:
 protected:

-       void append_data(void *data, uint64_t samples);
+       void append_single_sample(void *data);
+       void append_samples(void *data, uint64_t samples);
+       uint8_t* get_raw_samples(uint64_t start, uint64_t count) const;
+
+       SegmentRawDataIterator* begin_raw_sample_iteration(uint64_t start) const;
+       void continue_raw_sample_iteration(SegmentRawDataIterator* it, uint64_t increase) const;
+       void end_raw_sample_iteration(SegmentRawDataIterator* it) const;

 
 

-protected:

        mutable std::recursive_mutex mutex_;
        mutable std::recursive_mutex mutex_;

-       std::vector<uint8_t> data_;
+       std::vector<uint8_t*> data_chunks_;
+       uint8_t* current_chunk_;
+       uint64_t used_samples_, unused_samples_;

        uint64_t sample_count_;
        pv::util::Timestamp start_time_;
        double samplerate_;
        uint64_t sample_count_;
        pv::util::Timestamp start_time_;
        double samplerate_;

-       uint64_t capacity_;
+       uint64_t chunk_size_;

        unsigned int unit_size_;
        unsigned int unit_size_;

+
+       friend struct SegmentTest::SmallSize8Single;
+       friend struct SegmentTest::MediumSize8Single;
+       friend struct SegmentTest::MaxSize8Single;
+       friend struct SegmentTest::MediumSize24Single;
+       friend struct SegmentTest::MediumSize32Single;
+       friend struct SegmentTest::MaxSize32Single;
+       friend struct SegmentTest::MediumSize32Multi;
+       friend struct SegmentTest::MaxSize32Multi;
+       friend struct SegmentTest::MaxSize32MultiIterated;

 };
 
 } // namespace data
 };
 
 } // namespace data

diff --git a/pv/session.cpp b/pv/session.cpp
index fcd0e135b5a190972613e85ac79ada3f067b7023..97741ebf7b944a4c6507bd1a4e64c1d8128a5d5b 100644 (file)
--- a/pv/session.cpp
+++ b/pv/session.cpp
@@ -927,8 +927,6 @@ void Session::feed_in_logic(shared_ptr<Logic> logic)
 {
        lock_guard<recursive_mutex> lock(data_mutex_);
 
 {
        lock_guard<recursive_mutex> lock(data_mutex_);
 

-       const size_t sample_count = logic->data_length() / logic->unit_size();
-

        if (!logic_data_) {
                // The only reason logic_data_ would not have been created is
                // if it was not possible to determine the signals when the
        if (!logic_data_) {
                // The only reason logic_data_ would not have been created is
                // if it was not possible to determine the signals when the


@@ -942,8 +940,7 @@ void Session::feed_in_logic(shared_ptr<Logic> logic)
 
                // Create a new data segment
                cur_logic_segment_ = shared_ptr<data::LogicSegment>(
 
                // Create a new data segment
                cur_logic_segment_ = shared_ptr<data::LogicSegment>(

-                       new data::LogicSegment(
-                               logic, cur_samplerate_, sample_count));
+                       new data::LogicSegment(logic, cur_samplerate_));

                logic_data_->push_segment(cur_logic_segment_);
 
                // @todo Putting this here means that only listeners querying
                logic_data_->push_segment(cur_logic_segment_);
 
                // @todo Putting this here means that only listeners querying


@@ -988,8 +985,7 @@ void Session::feed_in_analog(shared_ptr<Analog> analog)
 
                        // Create a segment, keep it in the maps of channels
                        segment = shared_ptr<data::AnalogSegment>(
 
                        // Create a segment, keep it in the maps of channels
                        segment = shared_ptr<data::AnalogSegment>(

-                               new data::AnalogSegment(
-                                       cur_samplerate_, sample_count));
+                               new data::AnalogSegment(cur_samplerate_));

                        cur_analog_segments_[channel] = segment;
 
                        // Find the analog data associated with the channel
                        cur_analog_segments_[channel] = segment;
 
                        // Find the analog data associated with the channel

diff --git a/pv/view/analogsignal.cpp b/pv/view/analogsignal.cpp
index 3291c1615f4fb253048ae36468fc191a04c3840a..41f46dfcc75b414e47fbf8dd6bf47d2d7ff8502a 100644 (file)
--- a/pv/view/analogsignal.cpp
+++ b/pv/view/analogsignal.cpp
@@ -240,26 +240,25 @@ void AnalogSignal::paint_trace(QPainter &p,
        int y, int left, const int64_t start, const int64_t end,
        const double pixels_offset, const double samples_per_pixel)
 {
        int y, int left, const int64_t start, const int64_t end,
        const double pixels_offset, const double samples_per_pixel)
 {

-       const int64_t sample_count = end - start;
-
-       const float *const samples = segment->get_samples(start, end);
-       assert(samples);
-

        p.setPen(base_->colour());
 
        p.setPen(base_->colour());
 

-       QPointF *points = new QPointF[sample_count];
+       QPointF *points = new QPointF[end - start];

        QPointF *point = points;
 
        QPointF *point = points;
 

+       pv::data::SegmentAnalogDataIterator* it =
+               segment->begin_sample_iteration(start);
+

        for (int64_t sample = start; sample != end; sample++) {
                const float x = (sample / samples_per_pixel -
                        pixels_offset) + left;
        for (int64_t sample = start; sample != end; sample++) {
                const float x = (sample / samples_per_pixel -
                        pixels_offset) + left;

-               *point++ = QPointF(x,
-                       y - samples[sample - start] * scale_);
+
+               *point++ = QPointF(x, y - *((float*)it->value) * scale_);
+               segment->continue_sample_iteration(it, 1);

        }
        }

+       segment->end_sample_iteration(it);

 
        p.drawPolyline(points, point - points);
 
 
        p.drawPolyline(points, point - points);
 

-       delete[] samples;

        delete[] points;
 }
 
        delete[] points;
 }